Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2015-2018 Linaro Limited.
4 *
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 */
8
9#include <linux/bitops.h>
10#include <linux/err.h>
11#include <linux/kernel.h>
12#include <linux/log2.h>
13#include <linux/types.h>
14#include <linux/zalloc.h>
15
16#include <opencsd/ocsd_if_types.h>
17#include <stdlib.h>
18
19#include "auxtrace.h"
20#include "color.h"
21#include "cs-etm.h"
22#include "cs-etm-decoder/cs-etm-decoder.h"
23#include "debug.h"
24#include "dso.h"
25#include "evlist.h"
26#include "intlist.h"
27#include "machine.h"
28#include "map.h"
29#include "perf.h"
30#include "session.h"
31#include "map_symbol.h"
32#include "branch.h"
33#include "symbol.h"
34#include "tool.h"
35#include "thread.h"
36#include "thread-stack.h"
37#include <tools/libc_compat.h>
38#include "util/synthetic-events.h"
39
40#define MAX_TIMESTAMP (~0ULL)
41
42struct cs_etm_auxtrace {
43 struct auxtrace auxtrace;
44 struct auxtrace_queues queues;
45 struct auxtrace_heap heap;
46 struct itrace_synth_opts synth_opts;
47 struct perf_session *session;
48 struct machine *machine;
49 struct thread *unknown_thread;
50
51 u8 timeless_decoding;
52 u8 snapshot_mode;
53 u8 data_queued;
54 u8 sample_branches;
55 u8 sample_instructions;
56
57 int num_cpu;
58 u32 auxtrace_type;
59 u64 branches_sample_type;
60 u64 branches_id;
61 u64 instructions_sample_type;
62 u64 instructions_sample_period;
63 u64 instructions_id;
64 u64 **metadata;
65 u64 kernel_start;
66 unsigned int pmu_type;
67};
68
69struct cs_etm_traceid_queue {
70 u8 trace_chan_id;
71 pid_t pid, tid;
72 u64 period_instructions;
73 size_t last_branch_pos;
74 union perf_event *event_buf;
75 struct thread *thread;
76 struct branch_stack *last_branch;
77 struct branch_stack *last_branch_rb;
78 struct cs_etm_packet *prev_packet;
79 struct cs_etm_packet *packet;
80 struct cs_etm_packet_queue packet_queue;
81};
82
83struct cs_etm_queue {
84 struct cs_etm_auxtrace *etm;
85 struct cs_etm_decoder *decoder;
86 struct auxtrace_buffer *buffer;
87 unsigned int queue_nr;
88 u8 pending_timestamp;
89 u64 offset;
90 const unsigned char *buf;
91 size_t buf_len, buf_used;
92 /* Conversion between traceID and index in traceid_queues array */
93 struct intlist *traceid_queues_list;
94 struct cs_etm_traceid_queue **traceid_queues;
95};
96
97static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
98static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
99static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
100 pid_t tid);
101static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
102static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
103
104/* PTMs ETMIDR [11:8] set to b0011 */
105#define ETMIDR_PTM_VERSION 0x00000300
106
107/*
108 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
109 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
110 * encode the etm queue number as the upper 16 bit and the channel as
111 * the lower 16 bit.
112 */
113#define TO_CS_QUEUE_NR(queue_nr, trace_id_chan) \
114 (queue_nr << 16 | trace_chan_id)
115#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
116#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
117
118static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
119{
120 etmidr &= ETMIDR_PTM_VERSION;
121
122 if (etmidr == ETMIDR_PTM_VERSION)
123 return CS_ETM_PROTO_PTM;
124
125 return CS_ETM_PROTO_ETMV3;
126}
127
128static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
129{
130 struct int_node *inode;
131 u64 *metadata;
132
133 inode = intlist__find(traceid_list, trace_chan_id);
134 if (!inode)
135 return -EINVAL;
136
137 metadata = inode->priv;
138 *magic = metadata[CS_ETM_MAGIC];
139 return 0;
140}
141
142int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
143{
144 struct int_node *inode;
145 u64 *metadata;
146
147 inode = intlist__find(traceid_list, trace_chan_id);
148 if (!inode)
149 return -EINVAL;
150
151 metadata = inode->priv;
152 *cpu = (int)metadata[CS_ETM_CPU];
153 return 0;
154}
155
156void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
157 u8 trace_chan_id)
158{
159 /*
160 * Wnen a timestamp packet is encountered the backend code
161 * is stopped so that the front end has time to process packets
162 * that were accumulated in the traceID queue. Since there can
163 * be more than one channel per cs_etm_queue, we need to specify
164 * what traceID queue needs servicing.
165 */
166 etmq->pending_timestamp = trace_chan_id;
167}
168
169static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
170 u8 *trace_chan_id)
171{
172 struct cs_etm_packet_queue *packet_queue;
173
174 if (!etmq->pending_timestamp)
175 return 0;
176
177 if (trace_chan_id)
178 *trace_chan_id = etmq->pending_timestamp;
179
180 packet_queue = cs_etm__etmq_get_packet_queue(etmq,
181 etmq->pending_timestamp);
182 if (!packet_queue)
183 return 0;
184
185 /* Acknowledge pending status */
186 etmq->pending_timestamp = 0;
187
188 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
189 return packet_queue->timestamp;
190}
191
192static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
193{
194 int i;
195
196 queue->head = 0;
197 queue->tail = 0;
198 queue->packet_count = 0;
199 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
200 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
201 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
202 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
203 queue->packet_buffer[i].instr_count = 0;
204 queue->packet_buffer[i].last_instr_taken_branch = false;
205 queue->packet_buffer[i].last_instr_size = 0;
206 queue->packet_buffer[i].last_instr_type = 0;
207 queue->packet_buffer[i].last_instr_subtype = 0;
208 queue->packet_buffer[i].last_instr_cond = 0;
209 queue->packet_buffer[i].flags = 0;
210 queue->packet_buffer[i].exception_number = UINT32_MAX;
211 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
212 queue->packet_buffer[i].cpu = INT_MIN;
213 }
214}
215
216static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
217{
218 int idx;
219 struct int_node *inode;
220 struct cs_etm_traceid_queue *tidq;
221 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
222
223 intlist__for_each_entry(inode, traceid_queues_list) {
224 idx = (int)(intptr_t)inode->priv;
225 tidq = etmq->traceid_queues[idx];
226 cs_etm__clear_packet_queue(&tidq->packet_queue);
227 }
228}
229
230static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
231 struct cs_etm_traceid_queue *tidq,
232 u8 trace_chan_id)
233{
234 int rc = -ENOMEM;
235 struct auxtrace_queue *queue;
236 struct cs_etm_auxtrace *etm = etmq->etm;
237
238 cs_etm__clear_packet_queue(&tidq->packet_queue);
239
240 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
241 tidq->tid = queue->tid;
242 tidq->pid = -1;
243 tidq->trace_chan_id = trace_chan_id;
244
245 tidq->packet = zalloc(sizeof(struct cs_etm_packet));
246 if (!tidq->packet)
247 goto out;
248
249 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
250 if (!tidq->prev_packet)
251 goto out_free;
252
253 if (etm->synth_opts.last_branch) {
254 size_t sz = sizeof(struct branch_stack);
255
256 sz += etm->synth_opts.last_branch_sz *
257 sizeof(struct branch_entry);
258 tidq->last_branch = zalloc(sz);
259 if (!tidq->last_branch)
260 goto out_free;
261 tidq->last_branch_rb = zalloc(sz);
262 if (!tidq->last_branch_rb)
263 goto out_free;
264 }
265
266 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
267 if (!tidq->event_buf)
268 goto out_free;
269
270 return 0;
271
272out_free:
273 zfree(&tidq->last_branch_rb);
274 zfree(&tidq->last_branch);
275 zfree(&tidq->prev_packet);
276 zfree(&tidq->packet);
277out:
278 return rc;
279}
280
281static struct cs_etm_traceid_queue
282*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
283{
284 int idx;
285 struct int_node *inode;
286 struct intlist *traceid_queues_list;
287 struct cs_etm_traceid_queue *tidq, **traceid_queues;
288 struct cs_etm_auxtrace *etm = etmq->etm;
289
290 if (etm->timeless_decoding)
291 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
292
293 traceid_queues_list = etmq->traceid_queues_list;
294
295 /*
296 * Check if the traceid_queue exist for this traceID by looking
297 * in the queue list.
298 */
299 inode = intlist__find(traceid_queues_list, trace_chan_id);
300 if (inode) {
301 idx = (int)(intptr_t)inode->priv;
302 return etmq->traceid_queues[idx];
303 }
304
305 /* We couldn't find a traceid_queue for this traceID, allocate one */
306 tidq = malloc(sizeof(*tidq));
307 if (!tidq)
308 return NULL;
309
310 memset(tidq, 0, sizeof(*tidq));
311
312 /* Get a valid index for the new traceid_queue */
313 idx = intlist__nr_entries(traceid_queues_list);
314 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
315 inode = intlist__findnew(traceid_queues_list, trace_chan_id);
316 if (!inode)
317 goto out_free;
318
319 /* Associate this traceID with this index */
320 inode->priv = (void *)(intptr_t)idx;
321
322 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
323 goto out_free;
324
325 /* Grow the traceid_queues array by one unit */
326 traceid_queues = etmq->traceid_queues;
327 traceid_queues = reallocarray(traceid_queues,
328 idx + 1,
329 sizeof(*traceid_queues));
330
331 /*
332 * On failure reallocarray() returns NULL and the original block of
333 * memory is left untouched.
334 */
335 if (!traceid_queues)
336 goto out_free;
337
338 traceid_queues[idx] = tidq;
339 etmq->traceid_queues = traceid_queues;
340
341 return etmq->traceid_queues[idx];
342
343out_free:
344 /*
345 * Function intlist__remove() removes the inode from the list
346 * and delete the memory associated to it.
347 */
348 intlist__remove(traceid_queues_list, inode);
349 free(tidq);
350
351 return NULL;
352}
353
354struct cs_etm_packet_queue
355*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
356{
357 struct cs_etm_traceid_queue *tidq;
358
359 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
360 if (tidq)
361 return &tidq->packet_queue;
362
363 return NULL;
364}
365
366static void cs_etm__packet_dump(const char *pkt_string)
367{
368 const char *color = PERF_COLOR_BLUE;
369 int len = strlen(pkt_string);
370
371 if (len && (pkt_string[len-1] == '\n'))
372 color_fprintf(stdout, color, " %s", pkt_string);
373 else
374 color_fprintf(stdout, color, " %s\n", pkt_string);
375
376 fflush(stdout);
377}
378
379static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
380 struct cs_etm_auxtrace *etm, int idx,
381 u32 etmidr)
382{
383 u64 **metadata = etm->metadata;
384
385 t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
386 t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
387 t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
388}
389
390static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
391 struct cs_etm_auxtrace *etm, int idx)
392{
393 u64 **metadata = etm->metadata;
394
395 t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
396 t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
397 t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
398 t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
399 t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
400 t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
401 t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
402}
403
404static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
405 struct cs_etm_auxtrace *etm)
406{
407 int i;
408 u32 etmidr;
409 u64 architecture;
410
411 for (i = 0; i < etm->num_cpu; i++) {
412 architecture = etm->metadata[i][CS_ETM_MAGIC];
413
414 switch (architecture) {
415 case __perf_cs_etmv3_magic:
416 etmidr = etm->metadata[i][CS_ETM_ETMIDR];
417 cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
418 break;
419 case __perf_cs_etmv4_magic:
420 cs_etm__set_trace_param_etmv4(t_params, etm, i);
421 break;
422 default:
423 return -EINVAL;
424 }
425 }
426
427 return 0;
428}
429
430static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
431 struct cs_etm_queue *etmq,
432 enum cs_etm_decoder_operation mode)
433{
434 int ret = -EINVAL;
435
436 if (!(mode < CS_ETM_OPERATION_MAX))
437 goto out;
438
439 d_params->packet_printer = cs_etm__packet_dump;
440 d_params->operation = mode;
441 d_params->data = etmq;
442 d_params->formatted = true;
443 d_params->fsyncs = false;
444 d_params->hsyncs = false;
445 d_params->frame_aligned = true;
446
447 ret = 0;
448out:
449 return ret;
450}
451
452static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
453 struct auxtrace_buffer *buffer)
454{
455 int ret;
456 const char *color = PERF_COLOR_BLUE;
457 struct cs_etm_decoder_params d_params;
458 struct cs_etm_trace_params *t_params;
459 struct cs_etm_decoder *decoder;
460 size_t buffer_used = 0;
461
462 fprintf(stdout, "\n");
463 color_fprintf(stdout, color,
464 ". ... CoreSight ETM Trace data: size %zu bytes\n",
465 buffer->size);
466
467 /* Use metadata to fill in trace parameters for trace decoder */
468 t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
469
470 if (!t_params)
471 return;
472
473 if (cs_etm__init_trace_params(t_params, etm))
474 goto out_free;
475
476 /* Set decoder parameters to simply print the trace packets */
477 if (cs_etm__init_decoder_params(&d_params, NULL,
478 CS_ETM_OPERATION_PRINT))
479 goto out_free;
480
481 decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
482
483 if (!decoder)
484 goto out_free;
485 do {
486 size_t consumed;
487
488 ret = cs_etm_decoder__process_data_block(
489 decoder, buffer->offset,
490 &((u8 *)buffer->data)[buffer_used],
491 buffer->size - buffer_used, &consumed);
492 if (ret)
493 break;
494
495 buffer_used += consumed;
496 } while (buffer_used < buffer->size);
497
498 cs_etm_decoder__free(decoder);
499
500out_free:
501 zfree(&t_params);
502}
503
504static int cs_etm__flush_events(struct perf_session *session,
505 struct perf_tool *tool)
506{
507 int ret;
508 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
509 struct cs_etm_auxtrace,
510 auxtrace);
511 if (dump_trace)
512 return 0;
513
514 if (!tool->ordered_events)
515 return -EINVAL;
516
517 ret = cs_etm__update_queues(etm);
518
519 if (ret < 0)
520 return ret;
521
522 if (etm->timeless_decoding)
523 return cs_etm__process_timeless_queues(etm, -1);
524
525 return cs_etm__process_queues(etm);
526}
527
528static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
529{
530 int idx;
531 uintptr_t priv;
532 struct int_node *inode, *tmp;
533 struct cs_etm_traceid_queue *tidq;
534 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
535
536 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
537 priv = (uintptr_t)inode->priv;
538 idx = priv;
539
540 /* Free this traceid_queue from the array */
541 tidq = etmq->traceid_queues[idx];
542 thread__zput(tidq->thread);
543 zfree(&tidq->event_buf);
544 zfree(&tidq->last_branch);
545 zfree(&tidq->last_branch_rb);
546 zfree(&tidq->prev_packet);
547 zfree(&tidq->packet);
548 zfree(&tidq);
549
550 /*
551 * Function intlist__remove() removes the inode from the list
552 * and delete the memory associated to it.
553 */
554 intlist__remove(traceid_queues_list, inode);
555 }
556
557 /* Then the RB tree itself */
558 intlist__delete(traceid_queues_list);
559 etmq->traceid_queues_list = NULL;
560
561 /* finally free the traceid_queues array */
562 zfree(&etmq->traceid_queues);
563}
564
565static void cs_etm__free_queue(void *priv)
566{
567 struct cs_etm_queue *etmq = priv;
568
569 if (!etmq)
570 return;
571
572 cs_etm_decoder__free(etmq->decoder);
573 cs_etm__free_traceid_queues(etmq);
574 free(etmq);
575}
576
577static void cs_etm__free_events(struct perf_session *session)
578{
579 unsigned int i;
580 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
581 struct cs_etm_auxtrace,
582 auxtrace);
583 struct auxtrace_queues *queues = &aux->queues;
584
585 for (i = 0; i < queues->nr_queues; i++) {
586 cs_etm__free_queue(queues->queue_array[i].priv);
587 queues->queue_array[i].priv = NULL;
588 }
589
590 auxtrace_queues__free(queues);
591}
592
593static void cs_etm__free(struct perf_session *session)
594{
595 int i;
596 struct int_node *inode, *tmp;
597 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
598 struct cs_etm_auxtrace,
599 auxtrace);
600 cs_etm__free_events(session);
601 session->auxtrace = NULL;
602
603 /* First remove all traceID/metadata nodes for the RB tree */
604 intlist__for_each_entry_safe(inode, tmp, traceid_list)
605 intlist__remove(traceid_list, inode);
606 /* Then the RB tree itself */
607 intlist__delete(traceid_list);
608
609 for (i = 0; i < aux->num_cpu; i++)
610 zfree(&aux->metadata[i]);
611
612 thread__zput(aux->unknown_thread);
613 zfree(&aux->metadata);
614 zfree(&aux);
615}
616
617static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
618{
619 struct machine *machine;
620
621 machine = etmq->etm->machine;
622
623 if (address >= etmq->etm->kernel_start) {
624 if (machine__is_host(machine))
625 return PERF_RECORD_MISC_KERNEL;
626 else
627 return PERF_RECORD_MISC_GUEST_KERNEL;
628 } else {
629 if (machine__is_host(machine))
630 return PERF_RECORD_MISC_USER;
631 else if (perf_guest)
632 return PERF_RECORD_MISC_GUEST_USER;
633 else
634 return PERF_RECORD_MISC_HYPERVISOR;
635 }
636}
637
638static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
639 u64 address, size_t size, u8 *buffer)
640{
641 u8 cpumode;
642 u64 offset;
643 int len;
644 struct thread *thread;
645 struct machine *machine;
646 struct addr_location al;
647 struct cs_etm_traceid_queue *tidq;
648
649 if (!etmq)
650 return 0;
651
652 machine = etmq->etm->machine;
653 cpumode = cs_etm__cpu_mode(etmq, address);
654 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
655 if (!tidq)
656 return 0;
657
658 thread = tidq->thread;
659 if (!thread) {
660 if (cpumode != PERF_RECORD_MISC_KERNEL)
661 return 0;
662 thread = etmq->etm->unknown_thread;
663 }
664
665 if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
666 return 0;
667
668 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
669 dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
670 return 0;
671
672 offset = al.map->map_ip(al.map, address);
673
674 map__load(al.map);
675
676 len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
677
678 if (len <= 0)
679 return 0;
680
681 return len;
682}
683
684static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
685{
686 struct cs_etm_decoder_params d_params;
687 struct cs_etm_trace_params *t_params = NULL;
688 struct cs_etm_queue *etmq;
689
690 etmq = zalloc(sizeof(*etmq));
691 if (!etmq)
692 return NULL;
693
694 etmq->traceid_queues_list = intlist__new(NULL);
695 if (!etmq->traceid_queues_list)
696 goto out_free;
697
698 /* Use metadata to fill in trace parameters for trace decoder */
699 t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
700
701 if (!t_params)
702 goto out_free;
703
704 if (cs_etm__init_trace_params(t_params, etm))
705 goto out_free;
706
707 /* Set decoder parameters to decode trace packets */
708 if (cs_etm__init_decoder_params(&d_params, etmq,
709 CS_ETM_OPERATION_DECODE))
710 goto out_free;
711
712 etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
713
714 if (!etmq->decoder)
715 goto out_free;
716
717 /*
718 * Register a function to handle all memory accesses required by
719 * the trace decoder library.
720 */
721 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
722 0x0L, ((u64) -1L),
723 cs_etm__mem_access))
724 goto out_free_decoder;
725
726 zfree(&t_params);
727 return etmq;
728
729out_free_decoder:
730 cs_etm_decoder__free(etmq->decoder);
731out_free:
732 intlist__delete(etmq->traceid_queues_list);
733 free(etmq);
734
735 return NULL;
736}
737
738static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
739 struct auxtrace_queue *queue,
740 unsigned int queue_nr)
741{
742 int ret = 0;
743 unsigned int cs_queue_nr;
744 u8 trace_chan_id;
745 u64 timestamp;
746 struct cs_etm_queue *etmq = queue->priv;
747
748 if (list_empty(&queue->head) || etmq)
749 goto out;
750
751 etmq = cs_etm__alloc_queue(etm);
752
753 if (!etmq) {
754 ret = -ENOMEM;
755 goto out;
756 }
757
758 queue->priv = etmq;
759 etmq->etm = etm;
760 etmq->queue_nr = queue_nr;
761 etmq->offset = 0;
762
763 if (etm->timeless_decoding)
764 goto out;
765
766 /*
767 * We are under a CPU-wide trace scenario. As such we need to know
768 * when the code that generated the traces started to execute so that
769 * it can be correlated with execution on other CPUs. So we get a
770 * handle on the beginning of traces and decode until we find a
771 * timestamp. The timestamp is then added to the auxtrace min heap
772 * in order to know what nibble (of all the etmqs) to decode first.
773 */
774 while (1) {
775 /*
776 * Fetch an aux_buffer from this etmq. Bail if no more
777 * blocks or an error has been encountered.
778 */
779 ret = cs_etm__get_data_block(etmq);
780 if (ret <= 0)
781 goto out;
782
783 /*
784 * Run decoder on the trace block. The decoder will stop when
785 * encountering a timestamp, a full packet queue or the end of
786 * trace for that block.
787 */
788 ret = cs_etm__decode_data_block(etmq);
789 if (ret)
790 goto out;
791
792 /*
793 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
794 * the timestamp calculation for us.
795 */
796 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
797
798 /* We found a timestamp, no need to continue. */
799 if (timestamp)
800 break;
801
802 /*
803 * We didn't find a timestamp so empty all the traceid packet
804 * queues before looking for another timestamp packet, either
805 * in the current data block or a new one. Packets that were
806 * just decoded are useless since no timestamp has been
807 * associated with them. As such simply discard them.
808 */
809 cs_etm__clear_all_packet_queues(etmq);
810 }
811
812 /*
813 * We have a timestamp. Add it to the min heap to reflect when
814 * instructions conveyed by the range packets of this traceID queue
815 * started to execute. Once the same has been done for all the traceID
816 * queues of each etmq, redenring and decoding can start in
817 * chronological order.
818 *
819 * Note that packets decoded above are still in the traceID's packet
820 * queue and will be processed in cs_etm__process_queues().
821 */
822 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_id_chan);
823 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
824out:
825 return ret;
826}
827
828static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
829{
830 unsigned int i;
831 int ret;
832
833 if (!etm->kernel_start)
834 etm->kernel_start = machine__kernel_start(etm->machine);
835
836 for (i = 0; i < etm->queues.nr_queues; i++) {
837 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
838 if (ret)
839 return ret;
840 }
841
842 return 0;
843}
844
845static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
846{
847 if (etm->queues.new_data) {
848 etm->queues.new_data = false;
849 return cs_etm__setup_queues(etm);
850 }
851
852 return 0;
853}
854
855static inline
856void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
857 struct cs_etm_traceid_queue *tidq)
858{
859 struct branch_stack *bs_src = tidq->last_branch_rb;
860 struct branch_stack *bs_dst = tidq->last_branch;
861 size_t nr = 0;
862
863 /*
864 * Set the number of records before early exit: ->nr is used to
865 * determine how many branches to copy from ->entries.
866 */
867 bs_dst->nr = bs_src->nr;
868
869 /*
870 * Early exit when there is nothing to copy.
871 */
872 if (!bs_src->nr)
873 return;
874
875 /*
876 * As bs_src->entries is a circular buffer, we need to copy from it in
877 * two steps. First, copy the branches from the most recently inserted
878 * branch ->last_branch_pos until the end of bs_src->entries buffer.
879 */
880 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
881 memcpy(&bs_dst->entries[0],
882 &bs_src->entries[tidq->last_branch_pos],
883 sizeof(struct branch_entry) * nr);
884
885 /*
886 * If we wrapped around at least once, the branches from the beginning
887 * of the bs_src->entries buffer and until the ->last_branch_pos element
888 * are older valid branches: copy them over. The total number of
889 * branches copied over will be equal to the number of branches asked by
890 * the user in last_branch_sz.
891 */
892 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
893 memcpy(&bs_dst->entries[nr],
894 &bs_src->entries[0],
895 sizeof(struct branch_entry) * tidq->last_branch_pos);
896 }
897}
898
899static inline
900void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
901{
902 tidq->last_branch_pos = 0;
903 tidq->last_branch_rb->nr = 0;
904}
905
906static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
907 u8 trace_chan_id, u64 addr)
908{
909 u8 instrBytes[2];
910
911 cs_etm__mem_access(etmq, trace_chan_id, addr,
912 ARRAY_SIZE(instrBytes), instrBytes);
913 /*
914 * T32 instruction size is indicated by bits[15:11] of the first
915 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
916 * denote a 32-bit instruction.
917 */
918 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
919}
920
921static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
922{
923 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
924 if (packet->sample_type == CS_ETM_DISCONTINUITY)
925 return 0;
926
927 return packet->start_addr;
928}
929
930static inline
931u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
932{
933 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
934 if (packet->sample_type == CS_ETM_DISCONTINUITY)
935 return 0;
936
937 return packet->end_addr - packet->last_instr_size;
938}
939
940static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
941 u64 trace_chan_id,
942 const struct cs_etm_packet *packet,
943 u64 offset)
944{
945 if (packet->isa == CS_ETM_ISA_T32) {
946 u64 addr = packet->start_addr;
947
948 while (offset > 0) {
949 addr += cs_etm__t32_instr_size(etmq,
950 trace_chan_id, addr);
951 offset--;
952 }
953 return addr;
954 }
955
956 /* Assume a 4 byte instruction size (A32/A64) */
957 return packet->start_addr + offset * 4;
958}
959
960static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
961 struct cs_etm_traceid_queue *tidq)
962{
963 struct branch_stack *bs = tidq->last_branch_rb;
964 struct branch_entry *be;
965
966 /*
967 * The branches are recorded in a circular buffer in reverse
968 * chronological order: we start recording from the last element of the
969 * buffer down. After writing the first element of the stack, move the
970 * insert position back to the end of the buffer.
971 */
972 if (!tidq->last_branch_pos)
973 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
974
975 tidq->last_branch_pos -= 1;
976
977 be = &bs->entries[tidq->last_branch_pos];
978 be->from = cs_etm__last_executed_instr(tidq->prev_packet);
979 be->to = cs_etm__first_executed_instr(tidq->packet);
980 /* No support for mispredict */
981 be->flags.mispred = 0;
982 be->flags.predicted = 1;
983
984 /*
985 * Increment bs->nr until reaching the number of last branches asked by
986 * the user on the command line.
987 */
988 if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
989 bs->nr += 1;
990}
991
992static int cs_etm__inject_event(union perf_event *event,
993 struct perf_sample *sample, u64 type)
994{
995 event->header.size = perf_event__sample_event_size(sample, type, 0);
996 return perf_event__synthesize_sample(event, type, 0, sample);
997}
998
999
1000static int
1001cs_etm__get_trace(struct cs_etm_queue *etmq)
1002{
1003 struct auxtrace_buffer *aux_buffer = etmq->buffer;
1004 struct auxtrace_buffer *old_buffer = aux_buffer;
1005 struct auxtrace_queue *queue;
1006
1007 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1008
1009 aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1010
1011 /* If no more data, drop the previous auxtrace_buffer and return */
1012 if (!aux_buffer) {
1013 if (old_buffer)
1014 auxtrace_buffer__drop_data(old_buffer);
1015 etmq->buf_len = 0;
1016 return 0;
1017 }
1018
1019 etmq->buffer = aux_buffer;
1020
1021 /* If the aux_buffer doesn't have data associated, try to load it */
1022 if (!aux_buffer->data) {
1023 /* get the file desc associated with the perf data file */
1024 int fd = perf_data__fd(etmq->etm->session->data);
1025
1026 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1027 if (!aux_buffer->data)
1028 return -ENOMEM;
1029 }
1030
1031 /* If valid, drop the previous buffer */
1032 if (old_buffer)
1033 auxtrace_buffer__drop_data(old_buffer);
1034
1035 etmq->buf_used = 0;
1036 etmq->buf_len = aux_buffer->size;
1037 etmq->buf = aux_buffer->data;
1038
1039 return etmq->buf_len;
1040}
1041
1042static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1043 struct cs_etm_traceid_queue *tidq)
1044{
1045 if ((!tidq->thread) && (tidq->tid != -1))
1046 tidq->thread = machine__find_thread(etm->machine, -1,
1047 tidq->tid);
1048
1049 if (tidq->thread)
1050 tidq->pid = tidq->thread->pid_;
1051}
1052
1053int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1054 pid_t tid, u8 trace_chan_id)
1055{
1056 int cpu, err = -EINVAL;
1057 struct cs_etm_auxtrace *etm = etmq->etm;
1058 struct cs_etm_traceid_queue *tidq;
1059
1060 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1061 if (!tidq)
1062 return err;
1063
1064 if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1065 return err;
1066
1067 err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1068 if (err)
1069 return err;
1070
1071 tidq->tid = tid;
1072 thread__zput(tidq->thread);
1073
1074 cs_etm__set_pid_tid_cpu(etm, tidq);
1075 return 0;
1076}
1077
1078bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1079{
1080 return !!etmq->etm->timeless_decoding;
1081}
1082
1083static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1084 u64 trace_chan_id,
1085 const struct cs_etm_packet *packet,
1086 struct perf_sample *sample)
1087{
1088 /*
1089 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1090 * packet, so directly bail out with 'insn_len' = 0.
1091 */
1092 if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1093 sample->insn_len = 0;
1094 return;
1095 }
1096
1097 /*
1098 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1099 * cs_etm__t32_instr_size().
1100 */
1101 if (packet->isa == CS_ETM_ISA_T32)
1102 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1103 sample->ip);
1104 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1105 else
1106 sample->insn_len = 4;
1107
1108 cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1109 sample->insn_len, (void *)sample->insn);
1110}
1111
1112static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1113 struct cs_etm_traceid_queue *tidq,
1114 u64 addr, u64 period)
1115{
1116 int ret = 0;
1117 struct cs_etm_auxtrace *etm = etmq->etm;
1118 union perf_event *event = tidq->event_buf;
1119 struct perf_sample sample = {.ip = 0,};
1120
1121 event->sample.header.type = PERF_RECORD_SAMPLE;
1122 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1123 event->sample.header.size = sizeof(struct perf_event_header);
1124
1125 sample.ip = addr;
1126 sample.pid = tidq->pid;
1127 sample.tid = tidq->tid;
1128 sample.id = etmq->etm->instructions_id;
1129 sample.stream_id = etmq->etm->instructions_id;
1130 sample.period = period;
1131 sample.cpu = tidq->packet->cpu;
1132 sample.flags = tidq->prev_packet->flags;
1133 sample.cpumode = event->sample.header.misc;
1134
1135 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1136
1137 if (etm->synth_opts.last_branch) {
1138 cs_etm__copy_last_branch_rb(etmq, tidq);
1139 sample.branch_stack = tidq->last_branch;
1140 }
1141
1142 if (etm->synth_opts.inject) {
1143 ret = cs_etm__inject_event(event, &sample,
1144 etm->instructions_sample_type);
1145 if (ret)
1146 return ret;
1147 }
1148
1149 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1150
1151 if (ret)
1152 pr_err(
1153 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1154 ret);
1155
1156 if (etm->synth_opts.last_branch)
1157 cs_etm__reset_last_branch_rb(tidq);
1158
1159 return ret;
1160}
1161
1162/*
1163 * The cs etm packet encodes an instruction range between a branch target
1164 * and the next taken branch. Generate sample accordingly.
1165 */
1166static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1167 struct cs_etm_traceid_queue *tidq)
1168{
1169 int ret = 0;
1170 struct cs_etm_auxtrace *etm = etmq->etm;
1171 struct perf_sample sample = {.ip = 0,};
1172 union perf_event *event = tidq->event_buf;
1173 struct dummy_branch_stack {
1174 u64 nr;
1175 struct branch_entry entries;
1176 } dummy_bs;
1177 u64 ip;
1178
1179 ip = cs_etm__last_executed_instr(tidq->prev_packet);
1180
1181 event->sample.header.type = PERF_RECORD_SAMPLE;
1182 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1183 event->sample.header.size = sizeof(struct perf_event_header);
1184
1185 sample.ip = ip;
1186 sample.pid = tidq->pid;
1187 sample.tid = tidq->tid;
1188 sample.addr = cs_etm__first_executed_instr(tidq->packet);
1189 sample.id = etmq->etm->branches_id;
1190 sample.stream_id = etmq->etm->branches_id;
1191 sample.period = 1;
1192 sample.cpu = tidq->packet->cpu;
1193 sample.flags = tidq->prev_packet->flags;
1194 sample.cpumode = event->sample.header.misc;
1195
1196 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1197 &sample);
1198
1199 /*
1200 * perf report cannot handle events without a branch stack
1201 */
1202 if (etm->synth_opts.last_branch) {
1203 dummy_bs = (struct dummy_branch_stack){
1204 .nr = 1,
1205 .entries = {
1206 .from = sample.ip,
1207 .to = sample.addr,
1208 },
1209 };
1210 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1211 }
1212
1213 if (etm->synth_opts.inject) {
1214 ret = cs_etm__inject_event(event, &sample,
1215 etm->branches_sample_type);
1216 if (ret)
1217 return ret;
1218 }
1219
1220 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1221
1222 if (ret)
1223 pr_err(
1224 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1225 ret);
1226
1227 return ret;
1228}
1229
1230struct cs_etm_synth {
1231 struct perf_tool dummy_tool;
1232 struct perf_session *session;
1233};
1234
1235static int cs_etm__event_synth(struct perf_tool *tool,
1236 union perf_event *event,
1237 struct perf_sample *sample __maybe_unused,
1238 struct machine *machine __maybe_unused)
1239{
1240 struct cs_etm_synth *cs_etm_synth =
1241 container_of(tool, struct cs_etm_synth, dummy_tool);
1242
1243 return perf_session__deliver_synth_event(cs_etm_synth->session,
1244 event, NULL);
1245}
1246
1247static int cs_etm__synth_event(struct perf_session *session,
1248 struct perf_event_attr *attr, u64 id)
1249{
1250 struct cs_etm_synth cs_etm_synth;
1251
1252 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1253 cs_etm_synth.session = session;
1254
1255 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1256 &id, cs_etm__event_synth);
1257}
1258
1259static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1260 struct perf_session *session)
1261{
1262 struct evlist *evlist = session->evlist;
1263 struct evsel *evsel;
1264 struct perf_event_attr attr;
1265 bool found = false;
1266 u64 id;
1267 int err;
1268
1269 evlist__for_each_entry(evlist, evsel) {
1270 if (evsel->core.attr.type == etm->pmu_type) {
1271 found = true;
1272 break;
1273 }
1274 }
1275
1276 if (!found) {
1277 pr_debug("No selected events with CoreSight Trace data\n");
1278 return 0;
1279 }
1280
1281 memset(&attr, 0, sizeof(struct perf_event_attr));
1282 attr.size = sizeof(struct perf_event_attr);
1283 attr.type = PERF_TYPE_HARDWARE;
1284 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1285 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1286 PERF_SAMPLE_PERIOD;
1287 if (etm->timeless_decoding)
1288 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1289 else
1290 attr.sample_type |= PERF_SAMPLE_TIME;
1291
1292 attr.exclude_user = evsel->core.attr.exclude_user;
1293 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1294 attr.exclude_hv = evsel->core.attr.exclude_hv;
1295 attr.exclude_host = evsel->core.attr.exclude_host;
1296 attr.exclude_guest = evsel->core.attr.exclude_guest;
1297 attr.sample_id_all = evsel->core.attr.sample_id_all;
1298 attr.read_format = evsel->core.attr.read_format;
1299
1300 /* create new id val to be a fixed offset from evsel id */
1301 id = evsel->core.id[0] + 1000000000;
1302
1303 if (!id)
1304 id = 1;
1305
1306 if (etm->synth_opts.branches) {
1307 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1308 attr.sample_period = 1;
1309 attr.sample_type |= PERF_SAMPLE_ADDR;
1310 err = cs_etm__synth_event(session, &attr, id);
1311 if (err)
1312 return err;
1313 etm->sample_branches = true;
1314 etm->branches_sample_type = attr.sample_type;
1315 etm->branches_id = id;
1316 id += 1;
1317 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1318 }
1319
1320 if (etm->synth_opts.last_branch)
1321 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1322
1323 if (etm->synth_opts.instructions) {
1324 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1325 attr.sample_period = etm->synth_opts.period;
1326 etm->instructions_sample_period = attr.sample_period;
1327 err = cs_etm__synth_event(session, &attr, id);
1328 if (err)
1329 return err;
1330 etm->sample_instructions = true;
1331 etm->instructions_sample_type = attr.sample_type;
1332 etm->instructions_id = id;
1333 id += 1;
1334 }
1335
1336 return 0;
1337}
1338
1339static int cs_etm__sample(struct cs_etm_queue *etmq,
1340 struct cs_etm_traceid_queue *tidq)
1341{
1342 struct cs_etm_auxtrace *etm = etmq->etm;
1343 struct cs_etm_packet *tmp;
1344 int ret;
1345 u8 trace_chan_id = tidq->trace_chan_id;
1346 u64 instrs_executed = tidq->packet->instr_count;
1347
1348 tidq->period_instructions += instrs_executed;
1349
1350 /*
1351 * Record a branch when the last instruction in
1352 * PREV_PACKET is a branch.
1353 */
1354 if (etm->synth_opts.last_branch &&
1355 tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1356 tidq->prev_packet->last_instr_taken_branch)
1357 cs_etm__update_last_branch_rb(etmq, tidq);
1358
1359 if (etm->sample_instructions &&
1360 tidq->period_instructions >= etm->instructions_sample_period) {
1361 /*
1362 * Emit instruction sample periodically
1363 * TODO: allow period to be defined in cycles and clock time
1364 */
1365
1366 /* Get number of instructions executed after the sample point */
1367 u64 instrs_over = tidq->period_instructions -
1368 etm->instructions_sample_period;
1369
1370 /*
1371 * Calculate the address of the sampled instruction (-1 as
1372 * sample is reported as though instruction has just been
1373 * executed, but PC has not advanced to next instruction)
1374 */
1375 u64 offset = (instrs_executed - instrs_over - 1);
1376 u64 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1377 tidq->packet, offset);
1378
1379 ret = cs_etm__synth_instruction_sample(
1380 etmq, tidq, addr, etm->instructions_sample_period);
1381 if (ret)
1382 return ret;
1383
1384 /* Carry remaining instructions into next sample period */
1385 tidq->period_instructions = instrs_over;
1386 }
1387
1388 if (etm->sample_branches) {
1389 bool generate_sample = false;
1390
1391 /* Generate sample for tracing on packet */
1392 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1393 generate_sample = true;
1394
1395 /* Generate sample for branch taken packet */
1396 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1397 tidq->prev_packet->last_instr_taken_branch)
1398 generate_sample = true;
1399
1400 if (generate_sample) {
1401 ret = cs_etm__synth_branch_sample(etmq, tidq);
1402 if (ret)
1403 return ret;
1404 }
1405 }
1406
1407 if (etm->sample_branches || etm->synth_opts.last_branch) {
1408 /*
1409 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1410 * the next incoming packet.
1411 */
1412 tmp = tidq->packet;
1413 tidq->packet = tidq->prev_packet;
1414 tidq->prev_packet = tmp;
1415 }
1416
1417 return 0;
1418}
1419
1420static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1421{
1422 /*
1423 * When the exception packet is inserted, whether the last instruction
1424 * in previous range packet is taken branch or not, we need to force
1425 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1426 * to generate branch sample for the instruction range before the
1427 * exception is trapped to kernel or before the exception returning.
1428 *
1429 * The exception packet includes the dummy address values, so don't
1430 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1431 * for generating instruction and branch samples.
1432 */
1433 if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1434 tidq->prev_packet->last_instr_taken_branch = true;
1435
1436 return 0;
1437}
1438
1439static int cs_etm__flush(struct cs_etm_queue *etmq,
1440 struct cs_etm_traceid_queue *tidq)
1441{
1442 int err = 0;
1443 struct cs_etm_auxtrace *etm = etmq->etm;
1444 struct cs_etm_packet *tmp;
1445
1446 /* Handle start tracing packet */
1447 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1448 goto swap_packet;
1449
1450 if (etmq->etm->synth_opts.last_branch &&
1451 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1452 /*
1453 * Generate a last branch event for the branches left in the
1454 * circular buffer at the end of the trace.
1455 *
1456 * Use the address of the end of the last reported execution
1457 * range
1458 */
1459 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1460
1461 err = cs_etm__synth_instruction_sample(
1462 etmq, tidq, addr,
1463 tidq->period_instructions);
1464 if (err)
1465 return err;
1466
1467 tidq->period_instructions = 0;
1468
1469 }
1470
1471 if (etm->sample_branches &&
1472 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1473 err = cs_etm__synth_branch_sample(etmq, tidq);
1474 if (err)
1475 return err;
1476 }
1477
1478swap_packet:
1479 if (etm->sample_branches || etm->synth_opts.last_branch) {
1480 /*
1481 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1482 * the next incoming packet.
1483 */
1484 tmp = tidq->packet;
1485 tidq->packet = tidq->prev_packet;
1486 tidq->prev_packet = tmp;
1487 }
1488
1489 return err;
1490}
1491
1492static int cs_etm__end_block(struct cs_etm_queue *etmq,
1493 struct cs_etm_traceid_queue *tidq)
1494{
1495 int err;
1496
1497 /*
1498 * It has no new packet coming and 'etmq->packet' contains the stale
1499 * packet which was set at the previous time with packets swapping;
1500 * so skip to generate branch sample to avoid stale packet.
1501 *
1502 * For this case only flush branch stack and generate a last branch
1503 * event for the branches left in the circular buffer at the end of
1504 * the trace.
1505 */
1506 if (etmq->etm->synth_opts.last_branch &&
1507 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1508 /*
1509 * Use the address of the end of the last reported execution
1510 * range.
1511 */
1512 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1513
1514 err = cs_etm__synth_instruction_sample(
1515 etmq, tidq, addr,
1516 tidq->period_instructions);
1517 if (err)
1518 return err;
1519
1520 tidq->period_instructions = 0;
1521 }
1522
1523 return 0;
1524}
1525/*
1526 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1527 * if need be.
1528 * Returns: < 0 if error
1529 * = 0 if no more auxtrace_buffer to read
1530 * > 0 if the current buffer isn't empty yet
1531 */
1532static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1533{
1534 int ret;
1535
1536 if (!etmq->buf_len) {
1537 ret = cs_etm__get_trace(etmq);
1538 if (ret <= 0)
1539 return ret;
1540 /*
1541 * We cannot assume consecutive blocks in the data file
1542 * are contiguous, reset the decoder to force re-sync.
1543 */
1544 ret = cs_etm_decoder__reset(etmq->decoder);
1545 if (ret)
1546 return ret;
1547 }
1548
1549 return etmq->buf_len;
1550}
1551
1552static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1553 struct cs_etm_packet *packet,
1554 u64 end_addr)
1555{
1556 /* Initialise to keep compiler happy */
1557 u16 instr16 = 0;
1558 u32 instr32 = 0;
1559 u64 addr;
1560
1561 switch (packet->isa) {
1562 case CS_ETM_ISA_T32:
1563 /*
1564 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1565 *
1566 * b'15 b'8
1567 * +-----------------+--------+
1568 * | 1 1 0 1 1 1 1 1 | imm8 |
1569 * +-----------------+--------+
1570 *
1571 * According to the specifiction, it only defines SVC for T32
1572 * with 16 bits instruction and has no definition for 32bits;
1573 * so below only read 2 bytes as instruction size for T32.
1574 */
1575 addr = end_addr - 2;
1576 cs_etm__mem_access(etmq, trace_chan_id, addr,
1577 sizeof(instr16), (u8 *)&instr16);
1578 if ((instr16 & 0xFF00) == 0xDF00)
1579 return true;
1580
1581 break;
1582 case CS_ETM_ISA_A32:
1583 /*
1584 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1585 *
1586 * b'31 b'28 b'27 b'24
1587 * +---------+---------+-------------------------+
1588 * | !1111 | 1 1 1 1 | imm24 |
1589 * +---------+---------+-------------------------+
1590 */
1591 addr = end_addr - 4;
1592 cs_etm__mem_access(etmq, trace_chan_id, addr,
1593 sizeof(instr32), (u8 *)&instr32);
1594 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1595 (instr32 & 0xF0000000) != 0xF0000000)
1596 return true;
1597
1598 break;
1599 case CS_ETM_ISA_A64:
1600 /*
1601 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1602 *
1603 * b'31 b'21 b'4 b'0
1604 * +-----------------------+---------+-----------+
1605 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
1606 * +-----------------------+---------+-----------+
1607 */
1608 addr = end_addr - 4;
1609 cs_etm__mem_access(etmq, trace_chan_id, addr,
1610 sizeof(instr32), (u8 *)&instr32);
1611 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1612 return true;
1613
1614 break;
1615 case CS_ETM_ISA_UNKNOWN:
1616 default:
1617 break;
1618 }
1619
1620 return false;
1621}
1622
1623static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1624 struct cs_etm_traceid_queue *tidq, u64 magic)
1625{
1626 u8 trace_chan_id = tidq->trace_chan_id;
1627 struct cs_etm_packet *packet = tidq->packet;
1628 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1629
1630 if (magic == __perf_cs_etmv3_magic)
1631 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1632 return true;
1633
1634 /*
1635 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1636 * HVC cases; need to check if it's SVC instruction based on
1637 * packet address.
1638 */
1639 if (magic == __perf_cs_etmv4_magic) {
1640 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1641 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1642 prev_packet->end_addr))
1643 return true;
1644 }
1645
1646 return false;
1647}
1648
1649static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1650 u64 magic)
1651{
1652 struct cs_etm_packet *packet = tidq->packet;
1653
1654 if (magic == __perf_cs_etmv3_magic)
1655 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1656 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1657 packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1658 packet->exception_number == CS_ETMV3_EXC_IRQ ||
1659 packet->exception_number == CS_ETMV3_EXC_FIQ)
1660 return true;
1661
1662 if (magic == __perf_cs_etmv4_magic)
1663 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1664 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1665 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1666 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1667 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1668 packet->exception_number == CS_ETMV4_EXC_IRQ ||
1669 packet->exception_number == CS_ETMV4_EXC_FIQ)
1670 return true;
1671
1672 return false;
1673}
1674
1675static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1676 struct cs_etm_traceid_queue *tidq,
1677 u64 magic)
1678{
1679 u8 trace_chan_id = tidq->trace_chan_id;
1680 struct cs_etm_packet *packet = tidq->packet;
1681 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1682
1683 if (magic == __perf_cs_etmv3_magic)
1684 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1685 packet->exception_number == CS_ETMV3_EXC_HYP ||
1686 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1687 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1688 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1689 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1690 packet->exception_number == CS_ETMV3_EXC_GENERIC)
1691 return true;
1692
1693 if (magic == __perf_cs_etmv4_magic) {
1694 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1695 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1696 packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1697 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1698 return true;
1699
1700 /*
1701 * For CS_ETMV4_EXC_CALL, except SVC other instructions
1702 * (SMC, HVC) are taken as sync exceptions.
1703 */
1704 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1705 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1706 prev_packet->end_addr))
1707 return true;
1708
1709 /*
1710 * ETMv4 has 5 bits for exception number; if the numbers
1711 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1712 * they are implementation defined exceptions.
1713 *
1714 * For this case, simply take it as sync exception.
1715 */
1716 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1717 packet->exception_number <= CS_ETMV4_EXC_END)
1718 return true;
1719 }
1720
1721 return false;
1722}
1723
1724static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1725 struct cs_etm_traceid_queue *tidq)
1726{
1727 struct cs_etm_packet *packet = tidq->packet;
1728 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1729 u8 trace_chan_id = tidq->trace_chan_id;
1730 u64 magic;
1731 int ret;
1732
1733 switch (packet->sample_type) {
1734 case CS_ETM_RANGE:
1735 /*
1736 * Immediate branch instruction without neither link nor
1737 * return flag, it's normal branch instruction within
1738 * the function.
1739 */
1740 if (packet->last_instr_type == OCSD_INSTR_BR &&
1741 packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1742 packet->flags = PERF_IP_FLAG_BRANCH;
1743
1744 if (packet->last_instr_cond)
1745 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1746 }
1747
1748 /*
1749 * Immediate branch instruction with link (e.g. BL), this is
1750 * branch instruction for function call.
1751 */
1752 if (packet->last_instr_type == OCSD_INSTR_BR &&
1753 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1754 packet->flags = PERF_IP_FLAG_BRANCH |
1755 PERF_IP_FLAG_CALL;
1756
1757 /*
1758 * Indirect branch instruction with link (e.g. BLR), this is
1759 * branch instruction for function call.
1760 */
1761 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1762 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1763 packet->flags = PERF_IP_FLAG_BRANCH |
1764 PERF_IP_FLAG_CALL;
1765
1766 /*
1767 * Indirect branch instruction with subtype of
1768 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1769 * function return for A32/T32.
1770 */
1771 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1772 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1773 packet->flags = PERF_IP_FLAG_BRANCH |
1774 PERF_IP_FLAG_RETURN;
1775
1776 /*
1777 * Indirect branch instruction without link (e.g. BR), usually
1778 * this is used for function return, especially for functions
1779 * within dynamic link lib.
1780 */
1781 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1782 packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1783 packet->flags = PERF_IP_FLAG_BRANCH |
1784 PERF_IP_FLAG_RETURN;
1785
1786 /* Return instruction for function return. */
1787 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1788 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1789 packet->flags = PERF_IP_FLAG_BRANCH |
1790 PERF_IP_FLAG_RETURN;
1791
1792 /*
1793 * Decoder might insert a discontinuity in the middle of
1794 * instruction packets, fixup prev_packet with flag
1795 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1796 */
1797 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1798 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1799 PERF_IP_FLAG_TRACE_BEGIN;
1800
1801 /*
1802 * If the previous packet is an exception return packet
1803 * and the return address just follows SVC instuction,
1804 * it needs to calibrate the previous packet sample flags
1805 * as PERF_IP_FLAG_SYSCALLRET.
1806 */
1807 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1808 PERF_IP_FLAG_RETURN |
1809 PERF_IP_FLAG_INTERRUPT) &&
1810 cs_etm__is_svc_instr(etmq, trace_chan_id,
1811 packet, packet->start_addr))
1812 prev_packet->flags = PERF_IP_FLAG_BRANCH |
1813 PERF_IP_FLAG_RETURN |
1814 PERF_IP_FLAG_SYSCALLRET;
1815 break;
1816 case CS_ETM_DISCONTINUITY:
1817 /*
1818 * The trace is discontinuous, if the previous packet is
1819 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1820 * for previous packet.
1821 */
1822 if (prev_packet->sample_type == CS_ETM_RANGE)
1823 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1824 PERF_IP_FLAG_TRACE_END;
1825 break;
1826 case CS_ETM_EXCEPTION:
1827 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1828 if (ret)
1829 return ret;
1830
1831 /* The exception is for system call. */
1832 if (cs_etm__is_syscall(etmq, tidq, magic))
1833 packet->flags = PERF_IP_FLAG_BRANCH |
1834 PERF_IP_FLAG_CALL |
1835 PERF_IP_FLAG_SYSCALLRET;
1836 /*
1837 * The exceptions are triggered by external signals from bus,
1838 * interrupt controller, debug module, PE reset or halt.
1839 */
1840 else if (cs_etm__is_async_exception(tidq, magic))
1841 packet->flags = PERF_IP_FLAG_BRANCH |
1842 PERF_IP_FLAG_CALL |
1843 PERF_IP_FLAG_ASYNC |
1844 PERF_IP_FLAG_INTERRUPT;
1845 /*
1846 * Otherwise, exception is caused by trap, instruction &
1847 * data fault, or alignment errors.
1848 */
1849 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1850 packet->flags = PERF_IP_FLAG_BRANCH |
1851 PERF_IP_FLAG_CALL |
1852 PERF_IP_FLAG_INTERRUPT;
1853
1854 /*
1855 * When the exception packet is inserted, since exception
1856 * packet is not used standalone for generating samples
1857 * and it's affiliation to the previous instruction range
1858 * packet; so set previous range packet flags to tell perf
1859 * it is an exception taken branch.
1860 */
1861 if (prev_packet->sample_type == CS_ETM_RANGE)
1862 prev_packet->flags = packet->flags;
1863 break;
1864 case CS_ETM_EXCEPTION_RET:
1865 /*
1866 * When the exception return packet is inserted, since
1867 * exception return packet is not used standalone for
1868 * generating samples and it's affiliation to the previous
1869 * instruction range packet; so set previous range packet
1870 * flags to tell perf it is an exception return branch.
1871 *
1872 * The exception return can be for either system call or
1873 * other exception types; unfortunately the packet doesn't
1874 * contain exception type related info so we cannot decide
1875 * the exception type purely based on exception return packet.
1876 * If we record the exception number from exception packet and
1877 * reuse it for excpetion return packet, this is not reliable
1878 * due the trace can be discontinuity or the interrupt can
1879 * be nested, thus the recorded exception number cannot be
1880 * used for exception return packet for these two cases.
1881 *
1882 * For exception return packet, we only need to distinguish the
1883 * packet is for system call or for other types. Thus the
1884 * decision can be deferred when receive the next packet which
1885 * contains the return address, based on the return address we
1886 * can read out the previous instruction and check if it's a
1887 * system call instruction and then calibrate the sample flag
1888 * as needed.
1889 */
1890 if (prev_packet->sample_type == CS_ETM_RANGE)
1891 prev_packet->flags = PERF_IP_FLAG_BRANCH |
1892 PERF_IP_FLAG_RETURN |
1893 PERF_IP_FLAG_INTERRUPT;
1894 break;
1895 case CS_ETM_EMPTY:
1896 default:
1897 break;
1898 }
1899
1900 return 0;
1901}
1902
1903static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1904{
1905 int ret = 0;
1906 size_t processed = 0;
1907
1908 /*
1909 * Packets are decoded and added to the decoder's packet queue
1910 * until the decoder packet processing callback has requested that
1911 * processing stops or there is nothing left in the buffer. Normal
1912 * operations that stop processing are a timestamp packet or a full
1913 * decoder buffer queue.
1914 */
1915 ret = cs_etm_decoder__process_data_block(etmq->decoder,
1916 etmq->offset,
1917 &etmq->buf[etmq->buf_used],
1918 etmq->buf_len,
1919 &processed);
1920 if (ret)
1921 goto out;
1922
1923 etmq->offset += processed;
1924 etmq->buf_used += processed;
1925 etmq->buf_len -= processed;
1926
1927out:
1928 return ret;
1929}
1930
1931static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1932 struct cs_etm_traceid_queue *tidq)
1933{
1934 int ret;
1935 struct cs_etm_packet_queue *packet_queue;
1936
1937 packet_queue = &tidq->packet_queue;
1938
1939 /* Process each packet in this chunk */
1940 while (1) {
1941 ret = cs_etm_decoder__get_packet(packet_queue,
1942 tidq->packet);
1943 if (ret <= 0)
1944 /*
1945 * Stop processing this chunk on
1946 * end of data or error
1947 */
1948 break;
1949
1950 /*
1951 * Since packet addresses are swapped in packet
1952 * handling within below switch() statements,
1953 * thus setting sample flags must be called
1954 * prior to switch() statement to use address
1955 * information before packets swapping.
1956 */
1957 ret = cs_etm__set_sample_flags(etmq, tidq);
1958 if (ret < 0)
1959 break;
1960
1961 switch (tidq->packet->sample_type) {
1962 case CS_ETM_RANGE:
1963 /*
1964 * If the packet contains an instruction
1965 * range, generate instruction sequence
1966 * events.
1967 */
1968 cs_etm__sample(etmq, tidq);
1969 break;
1970 case CS_ETM_EXCEPTION:
1971 case CS_ETM_EXCEPTION_RET:
1972 /*
1973 * If the exception packet is coming,
1974 * make sure the previous instruction
1975 * range packet to be handled properly.
1976 */
1977 cs_etm__exception(tidq);
1978 break;
1979 case CS_ETM_DISCONTINUITY:
1980 /*
1981 * Discontinuity in trace, flush
1982 * previous branch stack
1983 */
1984 cs_etm__flush(etmq, tidq);
1985 break;
1986 case CS_ETM_EMPTY:
1987 /*
1988 * Should not receive empty packet,
1989 * report error.
1990 */
1991 pr_err("CS ETM Trace: empty packet\n");
1992 return -EINVAL;
1993 default:
1994 break;
1995 }
1996 }
1997
1998 return ret;
1999}
2000
2001static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2002{
2003 int idx;
2004 struct int_node *inode;
2005 struct cs_etm_traceid_queue *tidq;
2006 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2007
2008 intlist__for_each_entry(inode, traceid_queues_list) {
2009 idx = (int)(intptr_t)inode->priv;
2010 tidq = etmq->traceid_queues[idx];
2011
2012 /* Ignore return value */
2013 cs_etm__process_traceid_queue(etmq, tidq);
2014
2015 /*
2016 * Generate an instruction sample with the remaining
2017 * branchstack entries.
2018 */
2019 cs_etm__flush(etmq, tidq);
2020 }
2021}
2022
2023static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
2024{
2025 int err = 0;
2026 struct cs_etm_traceid_queue *tidq;
2027
2028 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2029 if (!tidq)
2030 return -EINVAL;
2031
2032 /* Go through each buffer in the queue and decode them one by one */
2033 while (1) {
2034 err = cs_etm__get_data_block(etmq);
2035 if (err <= 0)
2036 return err;
2037
2038 /* Run trace decoder until buffer consumed or end of trace */
2039 do {
2040 err = cs_etm__decode_data_block(etmq);
2041 if (err)
2042 return err;
2043
2044 /*
2045 * Process each packet in this chunk, nothing to do if
2046 * an error occurs other than hoping the next one will
2047 * be better.
2048 */
2049 err = cs_etm__process_traceid_queue(etmq, tidq);
2050
2051 } while (etmq->buf_len);
2052
2053 if (err == 0)
2054 /* Flush any remaining branch stack entries */
2055 err = cs_etm__end_block(etmq, tidq);
2056 }
2057
2058 return err;
2059}
2060
2061static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2062 pid_t tid)
2063{
2064 unsigned int i;
2065 struct auxtrace_queues *queues = &etm->queues;
2066
2067 for (i = 0; i < queues->nr_queues; i++) {
2068 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2069 struct cs_etm_queue *etmq = queue->priv;
2070 struct cs_etm_traceid_queue *tidq;
2071
2072 if (!etmq)
2073 continue;
2074
2075 tidq = cs_etm__etmq_get_traceid_queue(etmq,
2076 CS_ETM_PER_THREAD_TRACEID);
2077
2078 if (!tidq)
2079 continue;
2080
2081 if ((tid == -1) || (tidq->tid == tid)) {
2082 cs_etm__set_pid_tid_cpu(etm, tidq);
2083 cs_etm__run_decoder(etmq);
2084 }
2085 }
2086
2087 return 0;
2088}
2089
2090static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
2091{
2092 int ret = 0;
2093 unsigned int cs_queue_nr, queue_nr;
2094 u8 trace_chan_id;
2095 u64 timestamp;
2096 struct auxtrace_queue *queue;
2097 struct cs_etm_queue *etmq;
2098 struct cs_etm_traceid_queue *tidq;
2099
2100 while (1) {
2101 if (!etm->heap.heap_cnt)
2102 goto out;
2103
2104 /* Take the entry at the top of the min heap */
2105 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2106 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2107 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2108 queue = &etm->queues.queue_array[queue_nr];
2109 etmq = queue->priv;
2110
2111 /*
2112 * Remove the top entry from the heap since we are about
2113 * to process it.
2114 */
2115 auxtrace_heap__pop(&etm->heap);
2116
2117 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2118 if (!tidq) {
2119 /*
2120 * No traceID queue has been allocated for this traceID,
2121 * which means something somewhere went very wrong. No
2122 * other choice than simply exit.
2123 */
2124 ret = -EINVAL;
2125 goto out;
2126 }
2127
2128 /*
2129 * Packets associated with this timestamp are already in
2130 * the etmq's traceID queue, so process them.
2131 */
2132 ret = cs_etm__process_traceid_queue(etmq, tidq);
2133 if (ret < 0)
2134 goto out;
2135
2136 /*
2137 * Packets for this timestamp have been processed, time to
2138 * move on to the next timestamp, fetching a new auxtrace_buffer
2139 * if need be.
2140 */
2141refetch:
2142 ret = cs_etm__get_data_block(etmq);
2143 if (ret < 0)
2144 goto out;
2145
2146 /*
2147 * No more auxtrace_buffers to process in this etmq, simply
2148 * move on to another entry in the auxtrace_heap.
2149 */
2150 if (!ret)
2151 continue;
2152
2153 ret = cs_etm__decode_data_block(etmq);
2154 if (ret)
2155 goto out;
2156
2157 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2158
2159 if (!timestamp) {
2160 /*
2161 * Function cs_etm__decode_data_block() returns when
2162 * there is no more traces to decode in the current
2163 * auxtrace_buffer OR when a timestamp has been
2164 * encountered on any of the traceID queues. Since we
2165 * did not get a timestamp, there is no more traces to
2166 * process in this auxtrace_buffer. As such empty and
2167 * flush all traceID queues.
2168 */
2169 cs_etm__clear_all_traceid_queues(etmq);
2170
2171 /* Fetch another auxtrace_buffer for this etmq */
2172 goto refetch;
2173 }
2174
2175 /*
2176 * Add to the min heap the timestamp for packets that have
2177 * just been decoded. They will be processed and synthesized
2178 * during the next call to cs_etm__process_traceid_queue() for
2179 * this queue/traceID.
2180 */
2181 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2182 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
2183 }
2184
2185out:
2186 return ret;
2187}
2188
2189static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2190 union perf_event *event)
2191{
2192 struct thread *th;
2193
2194 if (etm->timeless_decoding)
2195 return 0;
2196
2197 /*
2198 * Add the tid/pid to the log so that we can get a match when
2199 * we get a contextID from the decoder.
2200 */
2201 th = machine__findnew_thread(etm->machine,
2202 event->itrace_start.pid,
2203 event->itrace_start.tid);
2204 if (!th)
2205 return -ENOMEM;
2206
2207 thread__put(th);
2208
2209 return 0;
2210}
2211
2212static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2213 union perf_event *event)
2214{
2215 struct thread *th;
2216 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2217
2218 /*
2219 * Context switch in per-thread mode are irrelevant since perf
2220 * will start/stop tracing as the process is scheduled.
2221 */
2222 if (etm->timeless_decoding)
2223 return 0;
2224
2225 /*
2226 * SWITCH_IN events carry the next process to be switched out while
2227 * SWITCH_OUT events carry the process to be switched in. As such
2228 * we don't care about IN events.
2229 */
2230 if (!out)
2231 return 0;
2232
2233 /*
2234 * Add the tid/pid to the log so that we can get a match when
2235 * we get a contextID from the decoder.
2236 */
2237 th = machine__findnew_thread(etm->machine,
2238 event->context_switch.next_prev_pid,
2239 event->context_switch.next_prev_tid);
2240 if (!th)
2241 return -ENOMEM;
2242
2243 thread__put(th);
2244
2245 return 0;
2246}
2247
2248static int cs_etm__process_event(struct perf_session *session,
2249 union perf_event *event,
2250 struct perf_sample *sample,
2251 struct perf_tool *tool)
2252{
2253 int err = 0;
2254 u64 timestamp;
2255 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2256 struct cs_etm_auxtrace,
2257 auxtrace);
2258
2259 if (dump_trace)
2260 return 0;
2261
2262 if (!tool->ordered_events) {
2263 pr_err("CoreSight ETM Trace requires ordered events\n");
2264 return -EINVAL;
2265 }
2266
2267 if (sample->time && (sample->time != (u64) -1))
2268 timestamp = sample->time;
2269 else
2270 timestamp = 0;
2271
2272 if (timestamp || etm->timeless_decoding) {
2273 err = cs_etm__update_queues(etm);
2274 if (err)
2275 return err;
2276 }
2277
2278 if (etm->timeless_decoding &&
2279 event->header.type == PERF_RECORD_EXIT)
2280 return cs_etm__process_timeless_queues(etm,
2281 event->fork.tid);
2282
2283 if (event->header.type == PERF_RECORD_ITRACE_START)
2284 return cs_etm__process_itrace_start(etm, event);
2285 else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2286 return cs_etm__process_switch_cpu_wide(etm, event);
2287
2288 if (!etm->timeless_decoding &&
2289 event->header.type == PERF_RECORD_AUX)
2290 return cs_etm__process_queues(etm);
2291
2292 return 0;
2293}
2294
2295static int cs_etm__process_auxtrace_event(struct perf_session *session,
2296 union perf_event *event,
2297 struct perf_tool *tool __maybe_unused)
2298{
2299 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2300 struct cs_etm_auxtrace,
2301 auxtrace);
2302 if (!etm->data_queued) {
2303 struct auxtrace_buffer *buffer;
2304 off_t data_offset;
2305 int fd = perf_data__fd(session->data);
2306 bool is_pipe = perf_data__is_pipe(session->data);
2307 int err;
2308
2309 if (is_pipe)
2310 data_offset = 0;
2311 else {
2312 data_offset = lseek(fd, 0, SEEK_CUR);
2313 if (data_offset == -1)
2314 return -errno;
2315 }
2316
2317 err = auxtrace_queues__add_event(&etm->queues, session,
2318 event, data_offset, &buffer);
2319 if (err)
2320 return err;
2321
2322 if (dump_trace)
2323 if (auxtrace_buffer__get_data(buffer, fd)) {
2324 cs_etm__dump_event(etm, buffer);
2325 auxtrace_buffer__put_data(buffer);
2326 }
2327 }
2328
2329 return 0;
2330}
2331
2332static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
2333{
2334 struct evsel *evsel;
2335 struct evlist *evlist = etm->session->evlist;
2336 bool timeless_decoding = true;
2337
2338 /*
2339 * Circle through the list of event and complain if we find one
2340 * with the time bit set.
2341 */
2342 evlist__for_each_entry(evlist, evsel) {
2343 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
2344 timeless_decoding = false;
2345 }
2346
2347 return timeless_decoding;
2348}
2349
2350static const char * const cs_etm_global_header_fmts[] = {
2351 [CS_HEADER_VERSION_0] = " Header version %llx\n",
2352 [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n",
2353 [CS_ETM_SNAPSHOT] = " Snapshot %llx\n",
2354};
2355
2356static const char * const cs_etm_priv_fmts[] = {
2357 [CS_ETM_MAGIC] = " Magic number %llx\n",
2358 [CS_ETM_CPU] = " CPU %lld\n",
2359 [CS_ETM_ETMCR] = " ETMCR %llx\n",
2360 [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n",
2361 [CS_ETM_ETMCCER] = " ETMCCER %llx\n",
2362 [CS_ETM_ETMIDR] = " ETMIDR %llx\n",
2363};
2364
2365static const char * const cs_etmv4_priv_fmts[] = {
2366 [CS_ETM_MAGIC] = " Magic number %llx\n",
2367 [CS_ETM_CPU] = " CPU %lld\n",
2368 [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n",
2369 [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n",
2370 [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n",
2371 [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n",
2372 [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n",
2373 [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n",
2374 [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n",
2375};
2376
2377static void cs_etm__print_auxtrace_info(__u64 *val, int num)
2378{
2379 int i, j, cpu = 0;
2380
2381 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2382 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
2383
2384 for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
2385 if (val[i] == __perf_cs_etmv3_magic)
2386 for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
2387 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
2388 else if (val[i] == __perf_cs_etmv4_magic)
2389 for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
2390 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
2391 else
2392 /* failure.. return */
2393 return;
2394 }
2395}
2396
2397int cs_etm__process_auxtrace_info(union perf_event *event,
2398 struct perf_session *session)
2399{
2400 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
2401 struct cs_etm_auxtrace *etm = NULL;
2402 struct int_node *inode;
2403 unsigned int pmu_type;
2404 int event_header_size = sizeof(struct perf_event_header);
2405 int info_header_size;
2406 int total_size = auxtrace_info->header.size;
2407 int priv_size = 0;
2408 int num_cpu;
2409 int err = 0, idx = -1;
2410 int i, j, k;
2411 u64 *ptr, *hdr = NULL;
2412 u64 **metadata = NULL;
2413
2414 /*
2415 * sizeof(auxtrace_info_event::type) +
2416 * sizeof(auxtrace_info_event::reserved) == 8
2417 */
2418 info_header_size = 8;
2419
2420 if (total_size < (event_header_size + info_header_size))
2421 return -EINVAL;
2422
2423 priv_size = total_size - event_header_size - info_header_size;
2424
2425 /* First the global part */
2426 ptr = (u64 *) auxtrace_info->priv;
2427
2428 /* Look for version '0' of the header */
2429 if (ptr[0] != 0)
2430 return -EINVAL;
2431
2432 hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2433 if (!hdr)
2434 return -ENOMEM;
2435
2436 /* Extract header information - see cs-etm.h for format */
2437 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2438 hdr[i] = ptr[i];
2439 num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2440 pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2441 0xffffffff);
2442
2443 /*
2444 * Create an RB tree for traceID-metadata tuple. Since the conversion
2445 * has to be made for each packet that gets decoded, optimizing access
2446 * in anything other than a sequential array is worth doing.
2447 */
2448 traceid_list = intlist__new(NULL);
2449 if (!traceid_list) {
2450 err = -ENOMEM;
2451 goto err_free_hdr;
2452 }
2453
2454 metadata = zalloc(sizeof(*metadata) * num_cpu);
2455 if (!metadata) {
2456 err = -ENOMEM;
2457 goto err_free_traceid_list;
2458 }
2459
2460 /*
2461 * The metadata is stored in the auxtrace_info section and encodes
2462 * the configuration of the ARM embedded trace macrocell which is
2463 * required by the trace decoder to properly decode the trace due
2464 * to its highly compressed nature.
2465 */
2466 for (j = 0; j < num_cpu; j++) {
2467 if (ptr[i] == __perf_cs_etmv3_magic) {
2468 metadata[j] = zalloc(sizeof(*metadata[j]) *
2469 CS_ETM_PRIV_MAX);
2470 if (!metadata[j]) {
2471 err = -ENOMEM;
2472 goto err_free_metadata;
2473 }
2474 for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2475 metadata[j][k] = ptr[i + k];
2476
2477 /* The traceID is our handle */
2478 idx = metadata[j][CS_ETM_ETMTRACEIDR];
2479 i += CS_ETM_PRIV_MAX;
2480 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2481 metadata[j] = zalloc(sizeof(*metadata[j]) *
2482 CS_ETMV4_PRIV_MAX);
2483 if (!metadata[j]) {
2484 err = -ENOMEM;
2485 goto err_free_metadata;
2486 }
2487 for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2488 metadata[j][k] = ptr[i + k];
2489
2490 /* The traceID is our handle */
2491 idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2492 i += CS_ETMV4_PRIV_MAX;
2493 }
2494
2495 /* Get an RB node for this CPU */
2496 inode = intlist__findnew(traceid_list, idx);
2497
2498 /* Something went wrong, no need to continue */
2499 if (!inode) {
2500 err = -ENOMEM;
2501 goto err_free_metadata;
2502 }
2503
2504 /*
2505 * The node for that CPU should not be taken.
2506 * Back out if that's the case.
2507 */
2508 if (inode->priv) {
2509 err = -EINVAL;
2510 goto err_free_metadata;
2511 }
2512 /* All good, associate the traceID with the metadata pointer */
2513 inode->priv = metadata[j];
2514 }
2515
2516 /*
2517 * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2518 * CS_ETMV4_PRIV_MAX mark how many double words are in the
2519 * global metadata, and each cpu's metadata respectively.
2520 * The following tests if the correct number of double words was
2521 * present in the auxtrace info section.
2522 */
2523 if (i * 8 != priv_size) {
2524 err = -EINVAL;
2525 goto err_free_metadata;
2526 }
2527
2528 etm = zalloc(sizeof(*etm));
2529
2530 if (!etm) {
2531 err = -ENOMEM;
2532 goto err_free_metadata;
2533 }
2534
2535 err = auxtrace_queues__init(&etm->queues);
2536 if (err)
2537 goto err_free_etm;
2538
2539 etm->session = session;
2540 etm->machine = &session->machines.host;
2541
2542 etm->num_cpu = num_cpu;
2543 etm->pmu_type = pmu_type;
2544 etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2545 etm->metadata = metadata;
2546 etm->auxtrace_type = auxtrace_info->type;
2547 etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2548
2549 etm->auxtrace.process_event = cs_etm__process_event;
2550 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2551 etm->auxtrace.flush_events = cs_etm__flush_events;
2552 etm->auxtrace.free_events = cs_etm__free_events;
2553 etm->auxtrace.free = cs_etm__free;
2554 session->auxtrace = &etm->auxtrace;
2555
2556 etm->unknown_thread = thread__new(999999999, 999999999);
2557 if (!etm->unknown_thread) {
2558 err = -ENOMEM;
2559 goto err_free_queues;
2560 }
2561
2562 /*
2563 * Initialize list node so that at thread__zput() we can avoid
2564 * segmentation fault at list_del_init().
2565 */
2566 INIT_LIST_HEAD(&etm->unknown_thread->node);
2567
2568 err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2569 if (err)
2570 goto err_delete_thread;
2571
2572 if (thread__init_map_groups(etm->unknown_thread, etm->machine)) {
2573 err = -ENOMEM;
2574 goto err_delete_thread;
2575 }
2576
2577 if (dump_trace) {
2578 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2579 return 0;
2580 }
2581
2582 if (session->itrace_synth_opts->set) {
2583 etm->synth_opts = *session->itrace_synth_opts;
2584 } else {
2585 itrace_synth_opts__set_default(&etm->synth_opts,
2586 session->itrace_synth_opts->default_no_sample);
2587 etm->synth_opts.callchain = false;
2588 }
2589
2590 err = cs_etm__synth_events(etm, session);
2591 if (err)
2592 goto err_delete_thread;
2593
2594 err = auxtrace_queues__process_index(&etm->queues, session);
2595 if (err)
2596 goto err_delete_thread;
2597
2598 etm->data_queued = etm->queues.populated;
2599
2600 return 0;
2601
2602err_delete_thread:
2603 thread__zput(etm->unknown_thread);
2604err_free_queues:
2605 auxtrace_queues__free(&etm->queues);
2606 session->auxtrace = NULL;
2607err_free_etm:
2608 zfree(&etm);
2609err_free_metadata:
2610 /* No need to check @metadata[j], free(NULL) is supported */
2611 for (j = 0; j < num_cpu; j++)
2612 zfree(&metadata[j]);
2613 zfree(&metadata);
2614err_free_traceid_list:
2615 intlist__delete(traceid_list);
2616err_free_hdr:
2617 zfree(&hdr);
2618
2619 return err;
2620}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2015-2018 Linaro Limited.
4 *
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 */
8
9#include <linux/kernel.h>
10#include <linux/bitfield.h>
11#include <linux/bitops.h>
12#include <linux/coresight-pmu.h>
13#include <linux/err.h>
14#include <linux/log2.h>
15#include <linux/types.h>
16#include <linux/zalloc.h>
17
18#include <stdlib.h>
19
20#include "auxtrace.h"
21#include "color.h"
22#include "cs-etm.h"
23#include "cs-etm-decoder/cs-etm-decoder.h"
24#include "debug.h"
25#include "dso.h"
26#include "evlist.h"
27#include "intlist.h"
28#include "machine.h"
29#include "map.h"
30#include "perf.h"
31#include "session.h"
32#include "map_symbol.h"
33#include "branch.h"
34#include "symbol.h"
35#include "tool.h"
36#include "thread.h"
37#include "thread-stack.h"
38#include "tsc.h"
39#include <tools/libc_compat.h>
40#include "util/synthetic-events.h"
41#include "util/util.h"
42
43struct cs_etm_auxtrace {
44 struct auxtrace auxtrace;
45 struct auxtrace_queues queues;
46 struct auxtrace_heap heap;
47 struct itrace_synth_opts synth_opts;
48 struct perf_session *session;
49 struct perf_tsc_conversion tc;
50
51 /*
52 * Timeless has no timestamps in the trace so overlapping mmap lookups
53 * are less accurate but produces smaller trace data. We use context IDs
54 * in the trace instead of matching timestamps with fork records so
55 * they're not really needed in the general case. Overlapping mmaps
56 * happen in cases like between a fork and an exec.
57 */
58 bool timeless_decoding;
59
60 /*
61 * Per-thread ignores the trace channel ID and instead assumes that
62 * everything in a buffer comes from the same process regardless of
63 * which CPU it ran on. It also implies no context IDs so the TID is
64 * taken from the auxtrace buffer.
65 */
66 bool per_thread_decoding;
67 bool snapshot_mode;
68 bool data_queued;
69 bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
70
71 int num_cpu;
72 u64 latest_kernel_timestamp;
73 u32 auxtrace_type;
74 u64 branches_sample_type;
75 u64 branches_id;
76 u64 instructions_sample_type;
77 u64 instructions_sample_period;
78 u64 instructions_id;
79 u64 **metadata;
80 unsigned int pmu_type;
81 enum cs_etm_pid_fmt pid_fmt;
82};
83
84struct cs_etm_traceid_queue {
85 u8 trace_chan_id;
86 u64 period_instructions;
87 size_t last_branch_pos;
88 union perf_event *event_buf;
89 struct thread *thread;
90 struct thread *prev_packet_thread;
91 ocsd_ex_level prev_packet_el;
92 ocsd_ex_level el;
93 struct branch_stack *last_branch;
94 struct branch_stack *last_branch_rb;
95 struct cs_etm_packet *prev_packet;
96 struct cs_etm_packet *packet;
97 struct cs_etm_packet_queue packet_queue;
98};
99
100enum cs_etm_format {
101 UNSET,
102 FORMATTED,
103 UNFORMATTED
104};
105
106struct cs_etm_queue {
107 struct cs_etm_auxtrace *etm;
108 struct cs_etm_decoder *decoder;
109 struct auxtrace_buffer *buffer;
110 unsigned int queue_nr;
111 u8 pending_timestamp_chan_id;
112 enum cs_etm_format format;
113 u64 offset;
114 const unsigned char *buf;
115 size_t buf_len, buf_used;
116 /* Conversion between traceID and index in traceid_queues array */
117 struct intlist *traceid_queues_list;
118 struct cs_etm_traceid_queue **traceid_queues;
119 /* Conversion between traceID and metadata pointers */
120 struct intlist *traceid_list;
121 /*
122 * Same as traceid_list, but traceid_list may be a reference to another
123 * queue's which has a matching sink ID.
124 */
125 struct intlist *own_traceid_list;
126 u32 sink_id;
127};
128
129static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
130static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
131 pid_t tid);
132static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
133static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
134static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata);
135static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu);
136static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata);
137
138/* PTMs ETMIDR [11:8] set to b0011 */
139#define ETMIDR_PTM_VERSION 0x00000300
140
141/*
142 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
143 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
144 * encode the etm queue number as the upper 16 bit and the channel as
145 * the lower 16 bit.
146 */
147#define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
148 (queue_nr << 16 | trace_chan_id)
149#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
150#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
151#define SINK_UNSET ((u32) -1)
152
153static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
154{
155 etmidr &= ETMIDR_PTM_VERSION;
156
157 if (etmidr == ETMIDR_PTM_VERSION)
158 return CS_ETM_PROTO_PTM;
159
160 return CS_ETM_PROTO_ETMV3;
161}
162
163static int cs_etm__get_magic(struct cs_etm_queue *etmq, u8 trace_chan_id, u64 *magic)
164{
165 struct int_node *inode;
166 u64 *metadata;
167
168 inode = intlist__find(etmq->traceid_list, trace_chan_id);
169 if (!inode)
170 return -EINVAL;
171
172 metadata = inode->priv;
173 *magic = metadata[CS_ETM_MAGIC];
174 return 0;
175}
176
177int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu)
178{
179 struct int_node *inode;
180 u64 *metadata;
181
182 inode = intlist__find(etmq->traceid_list, trace_chan_id);
183 if (!inode)
184 return -EINVAL;
185
186 metadata = inode->priv;
187 *cpu = (int)metadata[CS_ETM_CPU];
188 return 0;
189}
190
191/*
192 * The returned PID format is presented as an enum:
193 *
194 * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced.
195 * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced.
196 * CS_ETM_PIDFMT_NONE: No context IDs
197 *
198 * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
199 * are enabled at the same time when the session runs on an EL2 kernel.
200 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
201 * recorded in the trace data, the tool will selectively use
202 * CONTEXTIDR_EL2 as PID.
203 *
204 * The result is cached in etm->pid_fmt so this function only needs to be called
205 * when processing the aux info.
206 */
207static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata)
208{
209 u64 val;
210
211 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
212 val = metadata[CS_ETM_ETMCR];
213 /* CONTEXTIDR is traced */
214 if (val & BIT(ETM_OPT_CTXTID))
215 return CS_ETM_PIDFMT_CTXTID;
216 } else {
217 val = metadata[CS_ETMV4_TRCCONFIGR];
218 /* CONTEXTIDR_EL2 is traced */
219 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
220 return CS_ETM_PIDFMT_CTXTID2;
221 /* CONTEXTIDR_EL1 is traced */
222 else if (val & BIT(ETM4_CFG_BIT_CTXTID))
223 return CS_ETM_PIDFMT_CTXTID;
224 }
225
226 return CS_ETM_PIDFMT_NONE;
227}
228
229enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq)
230{
231 return etmq->etm->pid_fmt;
232}
233
234static int cs_etm__insert_trace_id_node(struct cs_etm_queue *etmq,
235 u8 trace_chan_id, u64 *cpu_metadata)
236{
237 /* Get an RB node for this CPU */
238 struct int_node *inode = intlist__findnew(etmq->traceid_list, trace_chan_id);
239
240 /* Something went wrong, no need to continue */
241 if (!inode)
242 return -ENOMEM;
243
244 /* Disallow re-mapping a different traceID to metadata pair. */
245 if (inode->priv) {
246 u64 *curr_cpu_data = inode->priv;
247 u8 curr_chan_id;
248 int err;
249
250 if (curr_cpu_data[CS_ETM_CPU] != cpu_metadata[CS_ETM_CPU]) {
251 /*
252 * With > CORESIGHT_TRACE_IDS_MAX ETMs, overlapping IDs
253 * are expected (but not supported) in per-thread mode,
254 * rather than signifying an error.
255 */
256 if (etmq->etm->per_thread_decoding)
257 pr_err("CS_ETM: overlapping Trace IDs aren't currently supported in per-thread mode\n");
258 else
259 pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
260
261 return -EINVAL;
262 }
263
264 /* check that the mapped ID matches */
265 err = cs_etm__metadata_get_trace_id(&curr_chan_id, curr_cpu_data);
266 if (err)
267 return err;
268
269 if (curr_chan_id != trace_chan_id) {
270 pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
271 return -EINVAL;
272 }
273
274 /* Skip re-adding the same mappings if everything matched */
275 return 0;
276 }
277
278 /* Not one we've seen before, associate the traceID with the metadata pointer */
279 inode->priv = cpu_metadata;
280
281 return 0;
282}
283
284static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int cpu)
285{
286 if (etm->per_thread_decoding)
287 return etm->queues.queue_array[0].priv;
288 else
289 return etm->queues.queue_array[cpu].priv;
290}
291
292static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id,
293 u64 *cpu_metadata)
294{
295 struct cs_etm_queue *etmq;
296
297 /*
298 * If the queue is unformatted then only save one mapping in the
299 * queue associated with that CPU so only one decoder is made.
300 */
301 etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]);
302 if (etmq->format == UNFORMATTED)
303 return cs_etm__insert_trace_id_node(etmq, trace_chan_id,
304 cpu_metadata);
305
306 /*
307 * Otherwise, version 0 trace IDs are global so save them into every
308 * queue.
309 */
310 for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) {
311 int ret;
312
313 etmq = etm->queues.queue_array[i].priv;
314 ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id,
315 cpu_metadata);
316 if (ret)
317 return ret;
318 }
319
320 return 0;
321}
322
323static int cs_etm__process_trace_id_v0(struct cs_etm_auxtrace *etm, int cpu,
324 u64 hw_id)
325{
326 int err;
327 u64 *cpu_data;
328 u8 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
329
330 cpu_data = get_cpu_data(etm, cpu);
331 if (cpu_data == NULL)
332 return -EINVAL;
333
334 err = cs_etm__map_trace_id_v0(etm, trace_chan_id, cpu_data);
335 if (err)
336 return err;
337
338 /*
339 * if we are picking up the association from the packet, need to plug
340 * the correct trace ID into the metadata for setting up decoders later.
341 */
342 return cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
343}
344
345static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu,
346 u64 hw_id)
347{
348 struct cs_etm_queue *etmq = cs_etm__get_queue(etm, cpu);
349 int ret;
350 u64 *cpu_data;
351 u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id);
352 u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
353
354 /*
355 * Check sink id hasn't changed in per-cpu mode. In per-thread mode,
356 * let it pass for now until an actual overlapping trace ID is hit. In
357 * most cases IDs won't overlap even if the sink changes.
358 */
359 if (!etmq->etm->per_thread_decoding && etmq->sink_id != SINK_UNSET &&
360 etmq->sink_id != sink_id) {
361 pr_err("CS_ETM: mismatch between sink IDs\n");
362 return -EINVAL;
363 }
364
365 etmq->sink_id = sink_id;
366
367 /* Find which other queues use this sink and link their ID maps */
368 for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) {
369 struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv;
370
371 /* Different sinks, skip */
372 if (other_etmq->sink_id != etmq->sink_id)
373 continue;
374
375 /* Already linked, skip */
376 if (other_etmq->traceid_list == etmq->traceid_list)
377 continue;
378
379 /* At the point of first linking, this one should be empty */
380 if (!intlist__empty(etmq->traceid_list)) {
381 pr_err("CS_ETM: Can't link populated trace ID lists\n");
382 return -EINVAL;
383 }
384
385 etmq->own_traceid_list = NULL;
386 intlist__delete(etmq->traceid_list);
387 etmq->traceid_list = other_etmq->traceid_list;
388 break;
389 }
390
391 cpu_data = get_cpu_data(etm, cpu);
392 ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data);
393 if (ret)
394 return ret;
395
396 ret = cs_etm__metadata_set_trace_id(trace_id, cpu_data);
397 if (ret)
398 return ret;
399
400 return 0;
401}
402
403static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
404{
405 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
406
407 switch (cs_etm_magic) {
408 case __perf_cs_etmv3_magic:
409 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
410 CORESIGHT_TRACE_ID_VAL_MASK);
411 break;
412 case __perf_cs_etmv4_magic:
413 case __perf_cs_ete_magic:
414 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
415 CORESIGHT_TRACE_ID_VAL_MASK);
416 break;
417 default:
418 return -EINVAL;
419 }
420 return 0;
421}
422
423/*
424 * update metadata trace ID from the value found in the AUX_HW_INFO packet.
425 */
426static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
427{
428 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
429
430 switch (cs_etm_magic) {
431 case __perf_cs_etmv3_magic:
432 cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
433 break;
434 case __perf_cs_etmv4_magic:
435 case __perf_cs_ete_magic:
436 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
437 break;
438
439 default:
440 return -EINVAL;
441 }
442 return 0;
443}
444
445/*
446 * Get a metadata index for a specific cpu from an array.
447 *
448 */
449static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu)
450{
451 int i;
452
453 for (i = 0; i < etm->num_cpu; i++) {
454 if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) {
455 return i;
456 }
457 }
458
459 return -1;
460}
461
462/*
463 * Get a metadata for a specific cpu from an array.
464 *
465 */
466static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu)
467{
468 int idx = get_cpu_data_idx(etm, cpu);
469
470 return (idx != -1) ? etm->metadata[idx] : NULL;
471}
472
473/*
474 * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
475 *
476 * The payload associates the Trace ID and the CPU.
477 * The routine is tolerant of seeing multiple packets with the same association,
478 * but a CPU / Trace ID association changing during a session is an error.
479 */
480static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
481 union perf_event *event)
482{
483 struct cs_etm_auxtrace *etm;
484 struct perf_sample sample;
485 struct evsel *evsel;
486 u64 hw_id;
487 int cpu, version, err;
488
489 /* extract and parse the HW ID */
490 hw_id = event->aux_output_hw_id.hw_id;
491 version = FIELD_GET(CS_AUX_HW_ID_MAJOR_VERSION_MASK, hw_id);
492
493 /* check that we can handle this version */
494 if (version > CS_AUX_HW_ID_MAJOR_VERSION) {
495 pr_err("CS ETM Trace: PERF_RECORD_AUX_OUTPUT_HW_ID version %d not supported. Please update Perf.\n",
496 version);
497 return -EINVAL;
498 }
499
500 /* get access to the etm metadata */
501 etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
502 if (!etm || !etm->metadata)
503 return -EINVAL;
504
505 /* parse the sample to get the CPU */
506 evsel = evlist__event2evsel(session->evlist, event);
507 if (!evsel)
508 return -EINVAL;
509 err = evsel__parse_sample(evsel, event, &sample);
510 if (err)
511 return err;
512 cpu = sample.cpu;
513 if (cpu == -1) {
514 /* no CPU in the sample - possibly recorded with an old version of perf */
515 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
516 return -EINVAL;
517 }
518
519 if (FIELD_GET(CS_AUX_HW_ID_MINOR_VERSION_MASK, hw_id) == 0)
520 return cs_etm__process_trace_id_v0(etm, cpu, hw_id);
521
522 return cs_etm__process_trace_id_v0_1(etm, cpu, hw_id);
523}
524
525void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
526 u8 trace_chan_id)
527{
528 /*
529 * When a timestamp packet is encountered the backend code
530 * is stopped so that the front end has time to process packets
531 * that were accumulated in the traceID queue. Since there can
532 * be more than one channel per cs_etm_queue, we need to specify
533 * what traceID queue needs servicing.
534 */
535 etmq->pending_timestamp_chan_id = trace_chan_id;
536}
537
538static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
539 u8 *trace_chan_id)
540{
541 struct cs_etm_packet_queue *packet_queue;
542
543 if (!etmq->pending_timestamp_chan_id)
544 return 0;
545
546 if (trace_chan_id)
547 *trace_chan_id = etmq->pending_timestamp_chan_id;
548
549 packet_queue = cs_etm__etmq_get_packet_queue(etmq,
550 etmq->pending_timestamp_chan_id);
551 if (!packet_queue)
552 return 0;
553
554 /* Acknowledge pending status */
555 etmq->pending_timestamp_chan_id = 0;
556
557 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
558 return packet_queue->cs_timestamp;
559}
560
561static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
562{
563 int i;
564
565 queue->head = 0;
566 queue->tail = 0;
567 queue->packet_count = 0;
568 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
569 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
570 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
571 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
572 queue->packet_buffer[i].instr_count = 0;
573 queue->packet_buffer[i].last_instr_taken_branch = false;
574 queue->packet_buffer[i].last_instr_size = 0;
575 queue->packet_buffer[i].last_instr_type = 0;
576 queue->packet_buffer[i].last_instr_subtype = 0;
577 queue->packet_buffer[i].last_instr_cond = 0;
578 queue->packet_buffer[i].flags = 0;
579 queue->packet_buffer[i].exception_number = UINT32_MAX;
580 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
581 queue->packet_buffer[i].cpu = INT_MIN;
582 }
583}
584
585static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
586{
587 int idx;
588 struct int_node *inode;
589 struct cs_etm_traceid_queue *tidq;
590 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
591
592 intlist__for_each_entry(inode, traceid_queues_list) {
593 idx = (int)(intptr_t)inode->priv;
594 tidq = etmq->traceid_queues[idx];
595 cs_etm__clear_packet_queue(&tidq->packet_queue);
596 }
597}
598
599static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
600 struct cs_etm_traceid_queue *tidq,
601 u8 trace_chan_id)
602{
603 int rc = -ENOMEM;
604 struct auxtrace_queue *queue;
605 struct cs_etm_auxtrace *etm = etmq->etm;
606
607 cs_etm__clear_packet_queue(&tidq->packet_queue);
608
609 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
610 tidq->trace_chan_id = trace_chan_id;
611 tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;
612 tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,
613 queue->tid);
614 tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);
615
616 tidq->packet = zalloc(sizeof(struct cs_etm_packet));
617 if (!tidq->packet)
618 goto out;
619
620 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
621 if (!tidq->prev_packet)
622 goto out_free;
623
624 if (etm->synth_opts.last_branch) {
625 size_t sz = sizeof(struct branch_stack);
626
627 sz += etm->synth_opts.last_branch_sz *
628 sizeof(struct branch_entry);
629 tidq->last_branch = zalloc(sz);
630 if (!tidq->last_branch)
631 goto out_free;
632 tidq->last_branch_rb = zalloc(sz);
633 if (!tidq->last_branch_rb)
634 goto out_free;
635 }
636
637 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
638 if (!tidq->event_buf)
639 goto out_free;
640
641 return 0;
642
643out_free:
644 zfree(&tidq->last_branch_rb);
645 zfree(&tidq->last_branch);
646 zfree(&tidq->prev_packet);
647 zfree(&tidq->packet);
648out:
649 return rc;
650}
651
652static struct cs_etm_traceid_queue
653*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
654{
655 int idx;
656 struct int_node *inode;
657 struct intlist *traceid_queues_list;
658 struct cs_etm_traceid_queue *tidq, **traceid_queues;
659 struct cs_etm_auxtrace *etm = etmq->etm;
660
661 if (etm->per_thread_decoding)
662 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
663
664 traceid_queues_list = etmq->traceid_queues_list;
665
666 /*
667 * Check if the traceid_queue exist for this traceID by looking
668 * in the queue list.
669 */
670 inode = intlist__find(traceid_queues_list, trace_chan_id);
671 if (inode) {
672 idx = (int)(intptr_t)inode->priv;
673 return etmq->traceid_queues[idx];
674 }
675
676 /* We couldn't find a traceid_queue for this traceID, allocate one */
677 tidq = malloc(sizeof(*tidq));
678 if (!tidq)
679 return NULL;
680
681 memset(tidq, 0, sizeof(*tidq));
682
683 /* Get a valid index for the new traceid_queue */
684 idx = intlist__nr_entries(traceid_queues_list);
685 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
686 inode = intlist__findnew(traceid_queues_list, trace_chan_id);
687 if (!inode)
688 goto out_free;
689
690 /* Associate this traceID with this index */
691 inode->priv = (void *)(intptr_t)idx;
692
693 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
694 goto out_free;
695
696 /* Grow the traceid_queues array by one unit */
697 traceid_queues = etmq->traceid_queues;
698 traceid_queues = reallocarray(traceid_queues,
699 idx + 1,
700 sizeof(*traceid_queues));
701
702 /*
703 * On failure reallocarray() returns NULL and the original block of
704 * memory is left untouched.
705 */
706 if (!traceid_queues)
707 goto out_free;
708
709 traceid_queues[idx] = tidq;
710 etmq->traceid_queues = traceid_queues;
711
712 return etmq->traceid_queues[idx];
713
714out_free:
715 /*
716 * Function intlist__remove() removes the inode from the list
717 * and delete the memory associated to it.
718 */
719 intlist__remove(traceid_queues_list, inode);
720 free(tidq);
721
722 return NULL;
723}
724
725struct cs_etm_packet_queue
726*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
727{
728 struct cs_etm_traceid_queue *tidq;
729
730 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
731 if (tidq)
732 return &tidq->packet_queue;
733
734 return NULL;
735}
736
737static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
738 struct cs_etm_traceid_queue *tidq)
739{
740 struct cs_etm_packet *tmp;
741
742 if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
743 etm->synth_opts.instructions) {
744 /*
745 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
746 * the next incoming packet.
747 *
748 * Threads and exception levels are also tracked for both the
749 * previous and current packets. This is because the previous
750 * packet is used for the 'from' IP for branch samples, so the
751 * thread at that time must also be assigned to that sample.
752 * Across discontinuity packets the thread can change, so by
753 * tracking the thread for the previous packet the branch sample
754 * will have the correct info.
755 */
756 tmp = tidq->packet;
757 tidq->packet = tidq->prev_packet;
758 tidq->prev_packet = tmp;
759 tidq->prev_packet_el = tidq->el;
760 thread__put(tidq->prev_packet_thread);
761 tidq->prev_packet_thread = thread__get(tidq->thread);
762 }
763}
764
765static void cs_etm__packet_dump(const char *pkt_string, void *data)
766{
767 const char *color = PERF_COLOR_BLUE;
768 int len = strlen(pkt_string);
769 struct cs_etm_queue *etmq = data;
770 char queue_nr[64];
771
772 if (verbose)
773 snprintf(queue_nr, sizeof(queue_nr), "Qnr:%d; ", etmq->queue_nr);
774 else
775 queue_nr[0] = '\0';
776
777 if (len && (pkt_string[len-1] == '\n'))
778 color_fprintf(stdout, color, " %s%s", queue_nr, pkt_string);
779 else
780 color_fprintf(stdout, color, " %s%s\n", queue_nr, pkt_string);
781
782 fflush(stdout);
783}
784
785static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
786 u64 *metadata, u32 etmidr)
787{
788 t_params->protocol = cs_etm__get_v7_protocol_version(etmidr);
789 t_params->etmv3.reg_ctrl = metadata[CS_ETM_ETMCR];
790 t_params->etmv3.reg_trc_id = metadata[CS_ETM_ETMTRACEIDR];
791}
792
793static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
794 u64 *metadata)
795{
796 t_params->protocol = CS_ETM_PROTO_ETMV4i;
797 t_params->etmv4.reg_idr0 = metadata[CS_ETMV4_TRCIDR0];
798 t_params->etmv4.reg_idr1 = metadata[CS_ETMV4_TRCIDR1];
799 t_params->etmv4.reg_idr2 = metadata[CS_ETMV4_TRCIDR2];
800 t_params->etmv4.reg_idr8 = metadata[CS_ETMV4_TRCIDR8];
801 t_params->etmv4.reg_configr = metadata[CS_ETMV4_TRCCONFIGR];
802 t_params->etmv4.reg_traceidr = metadata[CS_ETMV4_TRCTRACEIDR];
803}
804
805static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
806 u64 *metadata)
807{
808 t_params->protocol = CS_ETM_PROTO_ETE;
809 t_params->ete.reg_idr0 = metadata[CS_ETE_TRCIDR0];
810 t_params->ete.reg_idr1 = metadata[CS_ETE_TRCIDR1];
811 t_params->ete.reg_idr2 = metadata[CS_ETE_TRCIDR2];
812 t_params->ete.reg_idr8 = metadata[CS_ETE_TRCIDR8];
813 t_params->ete.reg_configr = metadata[CS_ETE_TRCCONFIGR];
814 t_params->ete.reg_traceidr = metadata[CS_ETE_TRCTRACEIDR];
815 t_params->ete.reg_devarch = metadata[CS_ETE_TRCDEVARCH];
816}
817
818static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
819 struct cs_etm_queue *etmq)
820{
821 struct int_node *inode;
822
823 intlist__for_each_entry(inode, etmq->traceid_list) {
824 u64 *metadata = inode->priv;
825 u64 architecture = metadata[CS_ETM_MAGIC];
826 u32 etmidr;
827
828 switch (architecture) {
829 case __perf_cs_etmv3_magic:
830 etmidr = metadata[CS_ETM_ETMIDR];
831 cs_etm__set_trace_param_etmv3(t_params++, metadata, etmidr);
832 break;
833 case __perf_cs_etmv4_magic:
834 cs_etm__set_trace_param_etmv4(t_params++, metadata);
835 break;
836 case __perf_cs_ete_magic:
837 cs_etm__set_trace_param_ete(t_params++, metadata);
838 break;
839 default:
840 return -EINVAL;
841 }
842 }
843
844 return 0;
845}
846
847static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
848 struct cs_etm_queue *etmq,
849 enum cs_etm_decoder_operation mode)
850{
851 int ret = -EINVAL;
852
853 if (!(mode < CS_ETM_OPERATION_MAX))
854 goto out;
855
856 d_params->packet_printer = cs_etm__packet_dump;
857 d_params->operation = mode;
858 d_params->data = etmq;
859 d_params->formatted = etmq->format == FORMATTED;
860 d_params->fsyncs = false;
861 d_params->hsyncs = false;
862 d_params->frame_aligned = true;
863
864 ret = 0;
865out:
866 return ret;
867}
868
869static void cs_etm__dump_event(struct cs_etm_queue *etmq,
870 struct auxtrace_buffer *buffer)
871{
872 int ret;
873 const char *color = PERF_COLOR_BLUE;
874 size_t buffer_used = 0;
875
876 fprintf(stdout, "\n");
877 color_fprintf(stdout, color,
878 ". ... CoreSight %s Trace data: size %#zx bytes\n",
879 cs_etm_decoder__get_name(etmq->decoder), buffer->size);
880
881 do {
882 size_t consumed;
883
884 ret = cs_etm_decoder__process_data_block(
885 etmq->decoder, buffer->offset,
886 &((u8 *)buffer->data)[buffer_used],
887 buffer->size - buffer_used, &consumed);
888 if (ret)
889 break;
890
891 buffer_used += consumed;
892 } while (buffer_used < buffer->size);
893
894 cs_etm_decoder__reset(etmq->decoder);
895}
896
897static int cs_etm__flush_events(struct perf_session *session,
898 const struct perf_tool *tool)
899{
900 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
901 struct cs_etm_auxtrace,
902 auxtrace);
903 if (dump_trace)
904 return 0;
905
906 if (!tool->ordered_events)
907 return -EINVAL;
908
909 if (etm->timeless_decoding) {
910 /*
911 * Pass tid = -1 to process all queues. But likely they will have
912 * already been processed on PERF_RECORD_EXIT anyway.
913 */
914 return cs_etm__process_timeless_queues(etm, -1);
915 }
916
917 return cs_etm__process_timestamped_queues(etm);
918}
919
920static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
921{
922 int idx;
923 uintptr_t priv;
924 struct int_node *inode, *tmp;
925 struct cs_etm_traceid_queue *tidq;
926 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
927
928 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
929 priv = (uintptr_t)inode->priv;
930 idx = priv;
931
932 /* Free this traceid_queue from the array */
933 tidq = etmq->traceid_queues[idx];
934 thread__zput(tidq->thread);
935 thread__zput(tidq->prev_packet_thread);
936 zfree(&tidq->event_buf);
937 zfree(&tidq->last_branch);
938 zfree(&tidq->last_branch_rb);
939 zfree(&tidq->prev_packet);
940 zfree(&tidq->packet);
941 zfree(&tidq);
942
943 /*
944 * Function intlist__remove() removes the inode from the list
945 * and delete the memory associated to it.
946 */
947 intlist__remove(traceid_queues_list, inode);
948 }
949
950 /* Then the RB tree itself */
951 intlist__delete(traceid_queues_list);
952 etmq->traceid_queues_list = NULL;
953
954 /* finally free the traceid_queues array */
955 zfree(&etmq->traceid_queues);
956}
957
958static void cs_etm__free_queue(void *priv)
959{
960 struct int_node *inode, *tmp;
961 struct cs_etm_queue *etmq = priv;
962
963 if (!etmq)
964 return;
965
966 cs_etm_decoder__free(etmq->decoder);
967 cs_etm__free_traceid_queues(etmq);
968
969 if (etmq->own_traceid_list) {
970 /* First remove all traceID/metadata nodes for the RB tree */
971 intlist__for_each_entry_safe(inode, tmp, etmq->own_traceid_list)
972 intlist__remove(etmq->own_traceid_list, inode);
973
974 /* Then the RB tree itself */
975 intlist__delete(etmq->own_traceid_list);
976 }
977
978 free(etmq);
979}
980
981static void cs_etm__free_events(struct perf_session *session)
982{
983 unsigned int i;
984 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
985 struct cs_etm_auxtrace,
986 auxtrace);
987 struct auxtrace_queues *queues = &aux->queues;
988
989 for (i = 0; i < queues->nr_queues; i++) {
990 cs_etm__free_queue(queues->queue_array[i].priv);
991 queues->queue_array[i].priv = NULL;
992 }
993
994 auxtrace_queues__free(queues);
995}
996
997static void cs_etm__free(struct perf_session *session)
998{
999 int i;
1000 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
1001 struct cs_etm_auxtrace,
1002 auxtrace);
1003 cs_etm__free_events(session);
1004 session->auxtrace = NULL;
1005
1006 for (i = 0; i < aux->num_cpu; i++)
1007 zfree(&aux->metadata[i]);
1008
1009 zfree(&aux->metadata);
1010 zfree(&aux);
1011}
1012
1013static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
1014 struct evsel *evsel)
1015{
1016 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
1017 struct cs_etm_auxtrace,
1018 auxtrace);
1019
1020 return evsel->core.attr.type == aux->pmu_type;
1021}
1022
1023static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq,
1024 ocsd_ex_level el)
1025{
1026 enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq);
1027
1028 /*
1029 * For any virtualisation based on nVHE (e.g. pKVM), or host kernels
1030 * running at EL1 assume everything is the host.
1031 */
1032 if (pid_fmt == CS_ETM_PIDFMT_CTXTID)
1033 return &etmq->etm->session->machines.host;
1034
1035 /*
1036 * Not perfect, but otherwise assume anything in EL1 is the default
1037 * guest, and everything else is the host. Distinguishing between guest
1038 * and host userspaces isn't currently supported either. Neither is
1039 * multiple guest support. All this does is reduce the likeliness of
1040 * decode errors where we look into the host kernel maps when it should
1041 * have been the guest maps.
1042 */
1043 switch (el) {
1044 case ocsd_EL1:
1045 return machines__find_guest(&etmq->etm->session->machines,
1046 DEFAULT_GUEST_KERNEL_ID);
1047 case ocsd_EL3:
1048 case ocsd_EL2:
1049 case ocsd_EL0:
1050 case ocsd_EL_unknown:
1051 default:
1052 return &etmq->etm->session->machines.host;
1053 }
1054}
1055
1056static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address,
1057 ocsd_ex_level el)
1058{
1059 struct machine *machine = cs_etm__get_machine(etmq, el);
1060
1061 if (address >= machine__kernel_start(machine)) {
1062 if (machine__is_host(machine))
1063 return PERF_RECORD_MISC_KERNEL;
1064 else
1065 return PERF_RECORD_MISC_GUEST_KERNEL;
1066 } else {
1067 if (machine__is_host(machine))
1068 return PERF_RECORD_MISC_USER;
1069 else {
1070 /*
1071 * Can't really happen at the moment because
1072 * cs_etm__get_machine() will always return
1073 * machines.host for any non EL1 trace.
1074 */
1075 return PERF_RECORD_MISC_GUEST_USER;
1076 }
1077 }
1078}
1079
1080static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
1081 u64 address, size_t size, u8 *buffer,
1082 const ocsd_mem_space_acc_t mem_space)
1083{
1084 u8 cpumode;
1085 u64 offset;
1086 int len;
1087 struct addr_location al;
1088 struct dso *dso;
1089 struct cs_etm_traceid_queue *tidq;
1090 int ret = 0;
1091
1092 if (!etmq)
1093 return 0;
1094
1095 addr_location__init(&al);
1096 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1097 if (!tidq)
1098 goto out;
1099
1100 /*
1101 * We've already tracked EL along side the PID in cs_etm__set_thread()
1102 * so double check that it matches what OpenCSD thinks as well. It
1103 * doesn't distinguish between EL0 and EL1 for this mem access callback
1104 * so we had to do the extra tracking. Skip validation if it's any of
1105 * the 'any' values.
1106 */
1107 if (!(mem_space == OCSD_MEM_SPACE_ANY ||
1108 mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) {
1109 if (mem_space & OCSD_MEM_SPACE_EL1N) {
1110 /* Includes both non secure EL1 and EL0 */
1111 assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);
1112 } else if (mem_space & OCSD_MEM_SPACE_EL2)
1113 assert(tidq->el == ocsd_EL2);
1114 else if (mem_space & OCSD_MEM_SPACE_EL3)
1115 assert(tidq->el == ocsd_EL3);
1116 }
1117
1118 cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);
1119
1120 if (!thread__find_map(tidq->thread, cpumode, address, &al))
1121 goto out;
1122
1123 dso = map__dso(al.map);
1124 if (!dso)
1125 goto out;
1126
1127 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR &&
1128 dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
1129 goto out;
1130
1131 offset = map__map_ip(al.map, address);
1132
1133 map__load(al.map);
1134
1135 len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),
1136 offset, buffer, size);
1137
1138 if (len <= 0) {
1139 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
1140 " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
1141 if (!dso__auxtrace_warned(dso)) {
1142 pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
1143 address,
1144 dso__long_name(dso) ? dso__long_name(dso) : "Unknown");
1145 dso__set_auxtrace_warned(dso);
1146 }
1147 goto out;
1148 }
1149 ret = len;
1150out:
1151 addr_location__exit(&al);
1152 return ret;
1153}
1154
1155static struct cs_etm_queue *cs_etm__alloc_queue(void)
1156{
1157 struct cs_etm_queue *etmq = zalloc(sizeof(*etmq));
1158 if (!etmq)
1159 return NULL;
1160
1161 etmq->traceid_queues_list = intlist__new(NULL);
1162 if (!etmq->traceid_queues_list)
1163 goto out_free;
1164
1165 /*
1166 * Create an RB tree for traceID-metadata tuple. Since the conversion
1167 * has to be made for each packet that gets decoded, optimizing access
1168 * in anything other than a sequential array is worth doing.
1169 */
1170 etmq->traceid_list = etmq->own_traceid_list = intlist__new(NULL);
1171 if (!etmq->traceid_list)
1172 goto out_free;
1173
1174 return etmq;
1175
1176out_free:
1177 intlist__delete(etmq->traceid_queues_list);
1178 free(etmq);
1179
1180 return NULL;
1181}
1182
1183static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
1184 struct auxtrace_queue *queue,
1185 unsigned int queue_nr)
1186{
1187 struct cs_etm_queue *etmq = queue->priv;
1188
1189 if (etmq)
1190 return 0;
1191
1192 etmq = cs_etm__alloc_queue();
1193
1194 if (!etmq)
1195 return -ENOMEM;
1196
1197 queue->priv = etmq;
1198 etmq->etm = etm;
1199 etmq->queue_nr = queue_nr;
1200 queue->cpu = queue_nr; /* Placeholder, may be reset to -1 in per-thread mode */
1201 etmq->offset = 0;
1202 etmq->sink_id = SINK_UNSET;
1203
1204 return 0;
1205}
1206
1207static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1208 struct cs_etm_queue *etmq,
1209 unsigned int queue_nr)
1210{
1211 int ret = 0;
1212 unsigned int cs_queue_nr;
1213 u8 trace_chan_id;
1214 u64 cs_timestamp;
1215
1216 /*
1217 * We are under a CPU-wide trace scenario. As such we need to know
1218 * when the code that generated the traces started to execute so that
1219 * it can be correlated with execution on other CPUs. So we get a
1220 * handle on the beginning of traces and decode until we find a
1221 * timestamp. The timestamp is then added to the auxtrace min heap
1222 * in order to know what nibble (of all the etmqs) to decode first.
1223 */
1224 while (1) {
1225 /*
1226 * Fetch an aux_buffer from this etmq. Bail if no more
1227 * blocks or an error has been encountered.
1228 */
1229 ret = cs_etm__get_data_block(etmq);
1230 if (ret <= 0)
1231 goto out;
1232
1233 /*
1234 * Run decoder on the trace block. The decoder will stop when
1235 * encountering a CS timestamp, a full packet queue or the end of
1236 * trace for that block.
1237 */
1238 ret = cs_etm__decode_data_block(etmq);
1239 if (ret)
1240 goto out;
1241
1242 /*
1243 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1244 * the timestamp calculation for us.
1245 */
1246 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1247
1248 /* We found a timestamp, no need to continue. */
1249 if (cs_timestamp)
1250 break;
1251
1252 /*
1253 * We didn't find a timestamp so empty all the traceid packet
1254 * queues before looking for another timestamp packet, either
1255 * in the current data block or a new one. Packets that were
1256 * just decoded are useless since no timestamp has been
1257 * associated with them. As such simply discard them.
1258 */
1259 cs_etm__clear_all_packet_queues(etmq);
1260 }
1261
1262 /*
1263 * We have a timestamp. Add it to the min heap to reflect when
1264 * instructions conveyed by the range packets of this traceID queue
1265 * started to execute. Once the same has been done for all the traceID
1266 * queues of each etmq, redenring and decoding can start in
1267 * chronological order.
1268 *
1269 * Note that packets decoded above are still in the traceID's packet
1270 * queue and will be processed in cs_etm__process_timestamped_queues().
1271 */
1272 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1273 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1274out:
1275 return ret;
1276}
1277
1278static inline
1279void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1280 struct cs_etm_traceid_queue *tidq)
1281{
1282 struct branch_stack *bs_src = tidq->last_branch_rb;
1283 struct branch_stack *bs_dst = tidq->last_branch;
1284 size_t nr = 0;
1285
1286 /*
1287 * Set the number of records before early exit: ->nr is used to
1288 * determine how many branches to copy from ->entries.
1289 */
1290 bs_dst->nr = bs_src->nr;
1291
1292 /*
1293 * Early exit when there is nothing to copy.
1294 */
1295 if (!bs_src->nr)
1296 return;
1297
1298 /*
1299 * As bs_src->entries is a circular buffer, we need to copy from it in
1300 * two steps. First, copy the branches from the most recently inserted
1301 * branch ->last_branch_pos until the end of bs_src->entries buffer.
1302 */
1303 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1304 memcpy(&bs_dst->entries[0],
1305 &bs_src->entries[tidq->last_branch_pos],
1306 sizeof(struct branch_entry) * nr);
1307
1308 /*
1309 * If we wrapped around at least once, the branches from the beginning
1310 * of the bs_src->entries buffer and until the ->last_branch_pos element
1311 * are older valid branches: copy them over. The total number of
1312 * branches copied over will be equal to the number of branches asked by
1313 * the user in last_branch_sz.
1314 */
1315 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1316 memcpy(&bs_dst->entries[nr],
1317 &bs_src->entries[0],
1318 sizeof(struct branch_entry) * tidq->last_branch_pos);
1319 }
1320}
1321
1322static inline
1323void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1324{
1325 tidq->last_branch_pos = 0;
1326 tidq->last_branch_rb->nr = 0;
1327}
1328
1329static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1330 u8 trace_chan_id, u64 addr)
1331{
1332 u8 instrBytes[2];
1333
1334 cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes),
1335 instrBytes, 0);
1336 /*
1337 * T32 instruction size is indicated by bits[15:11] of the first
1338 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1339 * denote a 32-bit instruction.
1340 */
1341 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1342}
1343
1344static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1345{
1346 /*
1347 * Return 0 for packets that have no addresses so that CS_ETM_INVAL_ADDR doesn't
1348 * appear in samples.
1349 */
1350 if (packet->sample_type == CS_ETM_DISCONTINUITY ||
1351 packet->sample_type == CS_ETM_EXCEPTION)
1352 return 0;
1353
1354 return packet->start_addr;
1355}
1356
1357static inline
1358u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1359{
1360 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1361 if (packet->sample_type == CS_ETM_DISCONTINUITY)
1362 return 0;
1363
1364 return packet->end_addr - packet->last_instr_size;
1365}
1366
1367static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1368 u64 trace_chan_id,
1369 const struct cs_etm_packet *packet,
1370 u64 offset)
1371{
1372 if (packet->isa == CS_ETM_ISA_T32) {
1373 u64 addr = packet->start_addr;
1374
1375 while (offset) {
1376 addr += cs_etm__t32_instr_size(etmq,
1377 trace_chan_id, addr);
1378 offset--;
1379 }
1380 return addr;
1381 }
1382
1383 /* Assume a 4 byte instruction size (A32/A64) */
1384 return packet->start_addr + offset * 4;
1385}
1386
1387static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1388 struct cs_etm_traceid_queue *tidq)
1389{
1390 struct branch_stack *bs = tidq->last_branch_rb;
1391 struct branch_entry *be;
1392
1393 /*
1394 * The branches are recorded in a circular buffer in reverse
1395 * chronological order: we start recording from the last element of the
1396 * buffer down. After writing the first element of the stack, move the
1397 * insert position back to the end of the buffer.
1398 */
1399 if (!tidq->last_branch_pos)
1400 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1401
1402 tidq->last_branch_pos -= 1;
1403
1404 be = &bs->entries[tidq->last_branch_pos];
1405 be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1406 be->to = cs_etm__first_executed_instr(tidq->packet);
1407 /* No support for mispredict */
1408 be->flags.mispred = 0;
1409 be->flags.predicted = 1;
1410
1411 /*
1412 * Increment bs->nr until reaching the number of last branches asked by
1413 * the user on the command line.
1414 */
1415 if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1416 bs->nr += 1;
1417}
1418
1419static int cs_etm__inject_event(union perf_event *event,
1420 struct perf_sample *sample, u64 type)
1421{
1422 event->header.size = perf_event__sample_event_size(sample, type, 0);
1423 return perf_event__synthesize_sample(event, type, 0, sample);
1424}
1425
1426
1427static int
1428cs_etm__get_trace(struct cs_etm_queue *etmq)
1429{
1430 struct auxtrace_buffer *aux_buffer = etmq->buffer;
1431 struct auxtrace_buffer *old_buffer = aux_buffer;
1432 struct auxtrace_queue *queue;
1433
1434 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1435
1436 aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1437
1438 /* If no more data, drop the previous auxtrace_buffer and return */
1439 if (!aux_buffer) {
1440 if (old_buffer)
1441 auxtrace_buffer__drop_data(old_buffer);
1442 etmq->buf_len = 0;
1443 return 0;
1444 }
1445
1446 etmq->buffer = aux_buffer;
1447
1448 /* If the aux_buffer doesn't have data associated, try to load it */
1449 if (!aux_buffer->data) {
1450 /* get the file desc associated with the perf data file */
1451 int fd = perf_data__fd(etmq->etm->session->data);
1452
1453 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1454 if (!aux_buffer->data)
1455 return -ENOMEM;
1456 }
1457
1458 /* If valid, drop the previous buffer */
1459 if (old_buffer)
1460 auxtrace_buffer__drop_data(old_buffer);
1461
1462 etmq->buf_used = 0;
1463 etmq->buf_len = aux_buffer->size;
1464 etmq->buf = aux_buffer->data;
1465
1466 return etmq->buf_len;
1467}
1468
1469static void cs_etm__set_thread(struct cs_etm_queue *etmq,
1470 struct cs_etm_traceid_queue *tidq, pid_t tid,
1471 ocsd_ex_level el)
1472{
1473 struct machine *machine = cs_etm__get_machine(etmq, el);
1474
1475 if (tid != -1) {
1476 thread__zput(tidq->thread);
1477 tidq->thread = machine__find_thread(machine, -1, tid);
1478 }
1479
1480 /* Couldn't find a known thread */
1481 if (!tidq->thread)
1482 tidq->thread = machine__idle_thread(machine);
1483
1484 tidq->el = el;
1485}
1486
1487int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
1488 u8 trace_chan_id, ocsd_ex_level el)
1489{
1490 struct cs_etm_traceid_queue *tidq;
1491
1492 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1493 if (!tidq)
1494 return -EINVAL;
1495
1496 cs_etm__set_thread(etmq, tidq, tid, el);
1497 return 0;
1498}
1499
1500bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1501{
1502 return !!etmq->etm->timeless_decoding;
1503}
1504
1505static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1506 u64 trace_chan_id,
1507 const struct cs_etm_packet *packet,
1508 struct perf_sample *sample)
1509{
1510 /*
1511 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1512 * packet, so directly bail out with 'insn_len' = 0.
1513 */
1514 if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1515 sample->insn_len = 0;
1516 return;
1517 }
1518
1519 /*
1520 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1521 * cs_etm__t32_instr_size().
1522 */
1523 if (packet->isa == CS_ETM_ISA_T32)
1524 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1525 sample->ip);
1526 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1527 else
1528 sample->insn_len = 4;
1529
1530 cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len,
1531 (void *)sample->insn, 0);
1532}
1533
1534u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1535{
1536 struct cs_etm_auxtrace *etm = etmq->etm;
1537
1538 if (etm->has_virtual_ts)
1539 return tsc_to_perf_time(cs_timestamp, &etm->tc);
1540 else
1541 return cs_timestamp;
1542}
1543
1544static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1545 struct cs_etm_traceid_queue *tidq)
1546{
1547 struct cs_etm_auxtrace *etm = etmq->etm;
1548 struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1549
1550 if (!etm->timeless_decoding && etm->has_virtual_ts)
1551 return packet_queue->cs_timestamp;
1552 else
1553 return etm->latest_kernel_timestamp;
1554}
1555
1556static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1557 struct cs_etm_traceid_queue *tidq,
1558 u64 addr, u64 period)
1559{
1560 int ret = 0;
1561 struct cs_etm_auxtrace *etm = etmq->etm;
1562 union perf_event *event = tidq->event_buf;
1563 struct perf_sample sample = {.ip = 0,};
1564
1565 event->sample.header.type = PERF_RECORD_SAMPLE;
1566 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);
1567 event->sample.header.size = sizeof(struct perf_event_header);
1568
1569 /* Set time field based on etm auxtrace config. */
1570 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1571
1572 sample.ip = addr;
1573 sample.pid = thread__pid(tidq->thread);
1574 sample.tid = thread__tid(tidq->thread);
1575 sample.id = etmq->etm->instructions_id;
1576 sample.stream_id = etmq->etm->instructions_id;
1577 sample.period = period;
1578 sample.cpu = tidq->packet->cpu;
1579 sample.flags = tidq->prev_packet->flags;
1580 sample.cpumode = event->sample.header.misc;
1581
1582 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1583
1584 if (etm->synth_opts.last_branch)
1585 sample.branch_stack = tidq->last_branch;
1586
1587 if (etm->synth_opts.inject) {
1588 ret = cs_etm__inject_event(event, &sample,
1589 etm->instructions_sample_type);
1590 if (ret)
1591 return ret;
1592 }
1593
1594 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1595
1596 if (ret)
1597 pr_err(
1598 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1599 ret);
1600
1601 return ret;
1602}
1603
1604/*
1605 * The cs etm packet encodes an instruction range between a branch target
1606 * and the next taken branch. Generate sample accordingly.
1607 */
1608static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1609 struct cs_etm_traceid_queue *tidq)
1610{
1611 int ret = 0;
1612 struct cs_etm_auxtrace *etm = etmq->etm;
1613 struct perf_sample sample = {.ip = 0,};
1614 union perf_event *event = tidq->event_buf;
1615 struct dummy_branch_stack {
1616 u64 nr;
1617 u64 hw_idx;
1618 struct branch_entry entries;
1619 } dummy_bs;
1620 u64 ip;
1621
1622 ip = cs_etm__last_executed_instr(tidq->prev_packet);
1623
1624 event->sample.header.type = PERF_RECORD_SAMPLE;
1625 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip,
1626 tidq->prev_packet_el);
1627 event->sample.header.size = sizeof(struct perf_event_header);
1628
1629 /* Set time field based on etm auxtrace config. */
1630 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1631
1632 sample.ip = ip;
1633 sample.pid = thread__pid(tidq->prev_packet_thread);
1634 sample.tid = thread__tid(tidq->prev_packet_thread);
1635 sample.addr = cs_etm__first_executed_instr(tidq->packet);
1636 sample.id = etmq->etm->branches_id;
1637 sample.stream_id = etmq->etm->branches_id;
1638 sample.period = 1;
1639 sample.cpu = tidq->packet->cpu;
1640 sample.flags = tidq->prev_packet->flags;
1641 sample.cpumode = event->sample.header.misc;
1642
1643 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1644 &sample);
1645
1646 /*
1647 * perf report cannot handle events without a branch stack
1648 */
1649 if (etm->synth_opts.last_branch) {
1650 dummy_bs = (struct dummy_branch_stack){
1651 .nr = 1,
1652 .hw_idx = -1ULL,
1653 .entries = {
1654 .from = sample.ip,
1655 .to = sample.addr,
1656 },
1657 };
1658 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1659 }
1660
1661 if (etm->synth_opts.inject) {
1662 ret = cs_etm__inject_event(event, &sample,
1663 etm->branches_sample_type);
1664 if (ret)
1665 return ret;
1666 }
1667
1668 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1669
1670 if (ret)
1671 pr_err(
1672 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1673 ret);
1674
1675 return ret;
1676}
1677
1678static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1679 struct perf_session *session)
1680{
1681 struct evlist *evlist = session->evlist;
1682 struct evsel *evsel;
1683 struct perf_event_attr attr;
1684 bool found = false;
1685 u64 id;
1686 int err;
1687
1688 evlist__for_each_entry(evlist, evsel) {
1689 if (evsel->core.attr.type == etm->pmu_type) {
1690 found = true;
1691 break;
1692 }
1693 }
1694
1695 if (!found) {
1696 pr_debug("No selected events with CoreSight Trace data\n");
1697 return 0;
1698 }
1699
1700 memset(&attr, 0, sizeof(struct perf_event_attr));
1701 attr.size = sizeof(struct perf_event_attr);
1702 attr.type = PERF_TYPE_HARDWARE;
1703 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1704 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1705 PERF_SAMPLE_PERIOD;
1706 if (etm->timeless_decoding)
1707 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1708 else
1709 attr.sample_type |= PERF_SAMPLE_TIME;
1710
1711 attr.exclude_user = evsel->core.attr.exclude_user;
1712 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1713 attr.exclude_hv = evsel->core.attr.exclude_hv;
1714 attr.exclude_host = evsel->core.attr.exclude_host;
1715 attr.exclude_guest = evsel->core.attr.exclude_guest;
1716 attr.sample_id_all = evsel->core.attr.sample_id_all;
1717 attr.read_format = evsel->core.attr.read_format;
1718
1719 /* create new id val to be a fixed offset from evsel id */
1720 id = evsel->core.id[0] + 1000000000;
1721
1722 if (!id)
1723 id = 1;
1724
1725 if (etm->synth_opts.branches) {
1726 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1727 attr.sample_period = 1;
1728 attr.sample_type |= PERF_SAMPLE_ADDR;
1729 err = perf_session__deliver_synth_attr_event(session, &attr, id);
1730 if (err)
1731 return err;
1732 etm->branches_sample_type = attr.sample_type;
1733 etm->branches_id = id;
1734 id += 1;
1735 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1736 }
1737
1738 if (etm->synth_opts.last_branch) {
1739 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1740 /*
1741 * We don't use the hardware index, but the sample generation
1742 * code uses the new format branch_stack with this field,
1743 * so the event attributes must indicate that it's present.
1744 */
1745 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1746 }
1747
1748 if (etm->synth_opts.instructions) {
1749 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1750 attr.sample_period = etm->synth_opts.period;
1751 etm->instructions_sample_period = attr.sample_period;
1752 err = perf_session__deliver_synth_attr_event(session, &attr, id);
1753 if (err)
1754 return err;
1755 etm->instructions_sample_type = attr.sample_type;
1756 etm->instructions_id = id;
1757 id += 1;
1758 }
1759
1760 return 0;
1761}
1762
1763static int cs_etm__sample(struct cs_etm_queue *etmq,
1764 struct cs_etm_traceid_queue *tidq)
1765{
1766 struct cs_etm_auxtrace *etm = etmq->etm;
1767 int ret;
1768 u8 trace_chan_id = tidq->trace_chan_id;
1769 u64 instrs_prev;
1770
1771 /* Get instructions remainder from previous packet */
1772 instrs_prev = tidq->period_instructions;
1773
1774 tidq->period_instructions += tidq->packet->instr_count;
1775
1776 /*
1777 * Record a branch when the last instruction in
1778 * PREV_PACKET is a branch.
1779 */
1780 if (etm->synth_opts.last_branch &&
1781 tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1782 tidq->prev_packet->last_instr_taken_branch)
1783 cs_etm__update_last_branch_rb(etmq, tidq);
1784
1785 if (etm->synth_opts.instructions &&
1786 tidq->period_instructions >= etm->instructions_sample_period) {
1787 /*
1788 * Emit instruction sample periodically
1789 * TODO: allow period to be defined in cycles and clock time
1790 */
1791
1792 /*
1793 * Below diagram demonstrates the instruction samples
1794 * generation flows:
1795 *
1796 * Instrs Instrs Instrs Instrs
1797 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3)
1798 * | | | |
1799 * V V V V
1800 * --------------------------------------------------
1801 * ^ ^
1802 * | |
1803 * Period Period
1804 * instructions(Pi) instructions(Pi')
1805 *
1806 * | |
1807 * \---------------- -----------------/
1808 * V
1809 * tidq->packet->instr_count
1810 *
1811 * Instrs Sample(n...) are the synthesised samples occurring
1812 * every etm->instructions_sample_period instructions - as
1813 * defined on the perf command line. Sample(n) is being the
1814 * last sample before the current etm packet, n+1 to n+3
1815 * samples are generated from the current etm packet.
1816 *
1817 * tidq->packet->instr_count represents the number of
1818 * instructions in the current etm packet.
1819 *
1820 * Period instructions (Pi) contains the number of
1821 * instructions executed after the sample point(n) from the
1822 * previous etm packet. This will always be less than
1823 * etm->instructions_sample_period.
1824 *
1825 * When generate new samples, it combines with two parts
1826 * instructions, one is the tail of the old packet and another
1827 * is the head of the new coming packet, to generate
1828 * sample(n+1); sample(n+2) and sample(n+3) consume the
1829 * instructions with sample period. After sample(n+3), the rest
1830 * instructions will be used by later packet and it is assigned
1831 * to tidq->period_instructions for next round calculation.
1832 */
1833
1834 /*
1835 * Get the initial offset into the current packet instructions;
1836 * entry conditions ensure that instrs_prev is less than
1837 * etm->instructions_sample_period.
1838 */
1839 u64 offset = etm->instructions_sample_period - instrs_prev;
1840 u64 addr;
1841
1842 /* Prepare last branches for instruction sample */
1843 if (etm->synth_opts.last_branch)
1844 cs_etm__copy_last_branch_rb(etmq, tidq);
1845
1846 while (tidq->period_instructions >=
1847 etm->instructions_sample_period) {
1848 /*
1849 * Calculate the address of the sampled instruction (-1
1850 * as sample is reported as though instruction has just
1851 * been executed, but PC has not advanced to next
1852 * instruction)
1853 */
1854 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1855 tidq->packet, offset - 1);
1856 ret = cs_etm__synth_instruction_sample(
1857 etmq, tidq, addr,
1858 etm->instructions_sample_period);
1859 if (ret)
1860 return ret;
1861
1862 offset += etm->instructions_sample_period;
1863 tidq->period_instructions -=
1864 etm->instructions_sample_period;
1865 }
1866 }
1867
1868 if (etm->synth_opts.branches) {
1869 bool generate_sample = false;
1870
1871 /* Generate sample for tracing on packet */
1872 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1873 generate_sample = true;
1874
1875 /* Generate sample for branch taken packet */
1876 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1877 tidq->prev_packet->last_instr_taken_branch)
1878 generate_sample = true;
1879
1880 if (generate_sample) {
1881 ret = cs_etm__synth_branch_sample(etmq, tidq);
1882 if (ret)
1883 return ret;
1884 }
1885 }
1886
1887 cs_etm__packet_swap(etm, tidq);
1888
1889 return 0;
1890}
1891
1892static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1893{
1894 /*
1895 * When the exception packet is inserted, whether the last instruction
1896 * in previous range packet is taken branch or not, we need to force
1897 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1898 * to generate branch sample for the instruction range before the
1899 * exception is trapped to kernel or before the exception returning.
1900 *
1901 * The exception packet includes the dummy address values, so don't
1902 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1903 * for generating instruction and branch samples.
1904 */
1905 if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1906 tidq->prev_packet->last_instr_taken_branch = true;
1907
1908 return 0;
1909}
1910
1911static int cs_etm__flush(struct cs_etm_queue *etmq,
1912 struct cs_etm_traceid_queue *tidq)
1913{
1914 int err = 0;
1915 struct cs_etm_auxtrace *etm = etmq->etm;
1916
1917 /* Handle start tracing packet */
1918 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1919 goto swap_packet;
1920
1921 if (etmq->etm->synth_opts.last_branch &&
1922 etmq->etm->synth_opts.instructions &&
1923 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1924 u64 addr;
1925
1926 /* Prepare last branches for instruction sample */
1927 cs_etm__copy_last_branch_rb(etmq, tidq);
1928
1929 /*
1930 * Generate a last branch event for the branches left in the
1931 * circular buffer at the end of the trace.
1932 *
1933 * Use the address of the end of the last reported execution
1934 * range
1935 */
1936 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1937
1938 err = cs_etm__synth_instruction_sample(
1939 etmq, tidq, addr,
1940 tidq->period_instructions);
1941 if (err)
1942 return err;
1943
1944 tidq->period_instructions = 0;
1945
1946 }
1947
1948 if (etm->synth_opts.branches &&
1949 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1950 err = cs_etm__synth_branch_sample(etmq, tidq);
1951 if (err)
1952 return err;
1953 }
1954
1955swap_packet:
1956 cs_etm__packet_swap(etm, tidq);
1957
1958 /* Reset last branches after flush the trace */
1959 if (etm->synth_opts.last_branch)
1960 cs_etm__reset_last_branch_rb(tidq);
1961
1962 return err;
1963}
1964
1965static int cs_etm__end_block(struct cs_etm_queue *etmq,
1966 struct cs_etm_traceid_queue *tidq)
1967{
1968 int err;
1969
1970 /*
1971 * It has no new packet coming and 'etmq->packet' contains the stale
1972 * packet which was set at the previous time with packets swapping;
1973 * so skip to generate branch sample to avoid stale packet.
1974 *
1975 * For this case only flush branch stack and generate a last branch
1976 * event for the branches left in the circular buffer at the end of
1977 * the trace.
1978 */
1979 if (etmq->etm->synth_opts.last_branch &&
1980 etmq->etm->synth_opts.instructions &&
1981 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1982 u64 addr;
1983
1984 /* Prepare last branches for instruction sample */
1985 cs_etm__copy_last_branch_rb(etmq, tidq);
1986
1987 /*
1988 * Use the address of the end of the last reported execution
1989 * range.
1990 */
1991 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1992
1993 err = cs_etm__synth_instruction_sample(
1994 etmq, tidq, addr,
1995 tidq->period_instructions);
1996 if (err)
1997 return err;
1998
1999 tidq->period_instructions = 0;
2000 }
2001
2002 return 0;
2003}
2004/*
2005 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
2006 * if need be.
2007 * Returns: < 0 if error
2008 * = 0 if no more auxtrace_buffer to read
2009 * > 0 if the current buffer isn't empty yet
2010 */
2011static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
2012{
2013 int ret;
2014
2015 if (!etmq->buf_len) {
2016 ret = cs_etm__get_trace(etmq);
2017 if (ret <= 0)
2018 return ret;
2019 /*
2020 * We cannot assume consecutive blocks in the data file
2021 * are contiguous, reset the decoder to force re-sync.
2022 */
2023 ret = cs_etm_decoder__reset(etmq->decoder);
2024 if (ret)
2025 return ret;
2026 }
2027
2028 return etmq->buf_len;
2029}
2030
2031static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
2032 struct cs_etm_packet *packet,
2033 u64 end_addr)
2034{
2035 /* Initialise to keep compiler happy */
2036 u16 instr16 = 0;
2037 u32 instr32 = 0;
2038 u64 addr;
2039
2040 switch (packet->isa) {
2041 case CS_ETM_ISA_T32:
2042 /*
2043 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
2044 *
2045 * b'15 b'8
2046 * +-----------------+--------+
2047 * | 1 1 0 1 1 1 1 1 | imm8 |
2048 * +-----------------+--------+
2049 *
2050 * According to the specification, it only defines SVC for T32
2051 * with 16 bits instruction and has no definition for 32bits;
2052 * so below only read 2 bytes as instruction size for T32.
2053 */
2054 addr = end_addr - 2;
2055 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16),
2056 (u8 *)&instr16, 0);
2057 if ((instr16 & 0xFF00) == 0xDF00)
2058 return true;
2059
2060 break;
2061 case CS_ETM_ISA_A32:
2062 /*
2063 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
2064 *
2065 * b'31 b'28 b'27 b'24
2066 * +---------+---------+-------------------------+
2067 * | !1111 | 1 1 1 1 | imm24 |
2068 * +---------+---------+-------------------------+
2069 */
2070 addr = end_addr - 4;
2071 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2072 (u8 *)&instr32, 0);
2073 if ((instr32 & 0x0F000000) == 0x0F000000 &&
2074 (instr32 & 0xF0000000) != 0xF0000000)
2075 return true;
2076
2077 break;
2078 case CS_ETM_ISA_A64:
2079 /*
2080 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
2081 *
2082 * b'31 b'21 b'4 b'0
2083 * +-----------------------+---------+-----------+
2084 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
2085 * +-----------------------+---------+-----------+
2086 */
2087 addr = end_addr - 4;
2088 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2089 (u8 *)&instr32, 0);
2090 if ((instr32 & 0xFFE0001F) == 0xd4000001)
2091 return true;
2092
2093 break;
2094 case CS_ETM_ISA_UNKNOWN:
2095 default:
2096 break;
2097 }
2098
2099 return false;
2100}
2101
2102static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
2103 struct cs_etm_traceid_queue *tidq, u64 magic)
2104{
2105 u8 trace_chan_id = tidq->trace_chan_id;
2106 struct cs_etm_packet *packet = tidq->packet;
2107 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2108
2109 if (magic == __perf_cs_etmv3_magic)
2110 if (packet->exception_number == CS_ETMV3_EXC_SVC)
2111 return true;
2112
2113 /*
2114 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
2115 * HVC cases; need to check if it's SVC instruction based on
2116 * packet address.
2117 */
2118 if (magic == __perf_cs_etmv4_magic) {
2119 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2120 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2121 prev_packet->end_addr))
2122 return true;
2123 }
2124
2125 return false;
2126}
2127
2128static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
2129 u64 magic)
2130{
2131 struct cs_etm_packet *packet = tidq->packet;
2132
2133 if (magic == __perf_cs_etmv3_magic)
2134 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
2135 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
2136 packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
2137 packet->exception_number == CS_ETMV3_EXC_IRQ ||
2138 packet->exception_number == CS_ETMV3_EXC_FIQ)
2139 return true;
2140
2141 if (magic == __perf_cs_etmv4_magic)
2142 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
2143 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
2144 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
2145 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
2146 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
2147 packet->exception_number == CS_ETMV4_EXC_IRQ ||
2148 packet->exception_number == CS_ETMV4_EXC_FIQ)
2149 return true;
2150
2151 return false;
2152}
2153
2154static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
2155 struct cs_etm_traceid_queue *tidq,
2156 u64 magic)
2157{
2158 u8 trace_chan_id = tidq->trace_chan_id;
2159 struct cs_etm_packet *packet = tidq->packet;
2160 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2161
2162 if (magic == __perf_cs_etmv3_magic)
2163 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
2164 packet->exception_number == CS_ETMV3_EXC_HYP ||
2165 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
2166 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
2167 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
2168 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
2169 packet->exception_number == CS_ETMV3_EXC_GENERIC)
2170 return true;
2171
2172 if (magic == __perf_cs_etmv4_magic) {
2173 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2174 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2175 packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2176 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2177 return true;
2178
2179 /*
2180 * For CS_ETMV4_EXC_CALL, except SVC other instructions
2181 * (SMC, HVC) are taken as sync exceptions.
2182 */
2183 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2184 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2185 prev_packet->end_addr))
2186 return true;
2187
2188 /*
2189 * ETMv4 has 5 bits for exception number; if the numbers
2190 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2191 * they are implementation defined exceptions.
2192 *
2193 * For this case, simply take it as sync exception.
2194 */
2195 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2196 packet->exception_number <= CS_ETMV4_EXC_END)
2197 return true;
2198 }
2199
2200 return false;
2201}
2202
2203static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2204 struct cs_etm_traceid_queue *tidq)
2205{
2206 struct cs_etm_packet *packet = tidq->packet;
2207 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2208 u8 trace_chan_id = tidq->trace_chan_id;
2209 u64 magic;
2210 int ret;
2211
2212 switch (packet->sample_type) {
2213 case CS_ETM_RANGE:
2214 /*
2215 * Immediate branch instruction without neither link nor
2216 * return flag, it's normal branch instruction within
2217 * the function.
2218 */
2219 if (packet->last_instr_type == OCSD_INSTR_BR &&
2220 packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2221 packet->flags = PERF_IP_FLAG_BRANCH;
2222
2223 if (packet->last_instr_cond)
2224 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2225 }
2226
2227 /*
2228 * Immediate branch instruction with link (e.g. BL), this is
2229 * branch instruction for function call.
2230 */
2231 if (packet->last_instr_type == OCSD_INSTR_BR &&
2232 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2233 packet->flags = PERF_IP_FLAG_BRANCH |
2234 PERF_IP_FLAG_CALL;
2235
2236 /*
2237 * Indirect branch instruction with link (e.g. BLR), this is
2238 * branch instruction for function call.
2239 */
2240 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2241 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2242 packet->flags = PERF_IP_FLAG_BRANCH |
2243 PERF_IP_FLAG_CALL;
2244
2245 /*
2246 * Indirect branch instruction with subtype of
2247 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2248 * function return for A32/T32.
2249 */
2250 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2251 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2252 packet->flags = PERF_IP_FLAG_BRANCH |
2253 PERF_IP_FLAG_RETURN;
2254
2255 /*
2256 * Indirect branch instruction without link (e.g. BR), usually
2257 * this is used for function return, especially for functions
2258 * within dynamic link lib.
2259 */
2260 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2261 packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2262 packet->flags = PERF_IP_FLAG_BRANCH |
2263 PERF_IP_FLAG_RETURN;
2264
2265 /* Return instruction for function return. */
2266 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2267 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2268 packet->flags = PERF_IP_FLAG_BRANCH |
2269 PERF_IP_FLAG_RETURN;
2270
2271 /*
2272 * Decoder might insert a discontinuity in the middle of
2273 * instruction packets, fixup prev_packet with flag
2274 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2275 */
2276 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2277 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2278 PERF_IP_FLAG_TRACE_BEGIN;
2279
2280 /*
2281 * If the previous packet is an exception return packet
2282 * and the return address just follows SVC instruction,
2283 * it needs to calibrate the previous packet sample flags
2284 * as PERF_IP_FLAG_SYSCALLRET.
2285 */
2286 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2287 PERF_IP_FLAG_RETURN |
2288 PERF_IP_FLAG_INTERRUPT) &&
2289 cs_etm__is_svc_instr(etmq, trace_chan_id,
2290 packet, packet->start_addr))
2291 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2292 PERF_IP_FLAG_RETURN |
2293 PERF_IP_FLAG_SYSCALLRET;
2294 break;
2295 case CS_ETM_DISCONTINUITY:
2296 /*
2297 * The trace is discontinuous, if the previous packet is
2298 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2299 * for previous packet.
2300 */
2301 if (prev_packet->sample_type == CS_ETM_RANGE)
2302 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2303 PERF_IP_FLAG_TRACE_END;
2304 break;
2305 case CS_ETM_EXCEPTION:
2306 ret = cs_etm__get_magic(etmq, packet->trace_chan_id, &magic);
2307 if (ret)
2308 return ret;
2309
2310 /* The exception is for system call. */
2311 if (cs_etm__is_syscall(etmq, tidq, magic))
2312 packet->flags = PERF_IP_FLAG_BRANCH |
2313 PERF_IP_FLAG_CALL |
2314 PERF_IP_FLAG_SYSCALLRET;
2315 /*
2316 * The exceptions are triggered by external signals from bus,
2317 * interrupt controller, debug module, PE reset or halt.
2318 */
2319 else if (cs_etm__is_async_exception(tidq, magic))
2320 packet->flags = PERF_IP_FLAG_BRANCH |
2321 PERF_IP_FLAG_CALL |
2322 PERF_IP_FLAG_ASYNC |
2323 PERF_IP_FLAG_INTERRUPT;
2324 /*
2325 * Otherwise, exception is caused by trap, instruction &
2326 * data fault, or alignment errors.
2327 */
2328 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2329 packet->flags = PERF_IP_FLAG_BRANCH |
2330 PERF_IP_FLAG_CALL |
2331 PERF_IP_FLAG_INTERRUPT;
2332
2333 /*
2334 * When the exception packet is inserted, since exception
2335 * packet is not used standalone for generating samples
2336 * and it's affiliation to the previous instruction range
2337 * packet; so set previous range packet flags to tell perf
2338 * it is an exception taken branch.
2339 */
2340 if (prev_packet->sample_type == CS_ETM_RANGE)
2341 prev_packet->flags = packet->flags;
2342 break;
2343 case CS_ETM_EXCEPTION_RET:
2344 /*
2345 * When the exception return packet is inserted, since
2346 * exception return packet is not used standalone for
2347 * generating samples and it's affiliation to the previous
2348 * instruction range packet; so set previous range packet
2349 * flags to tell perf it is an exception return branch.
2350 *
2351 * The exception return can be for either system call or
2352 * other exception types; unfortunately the packet doesn't
2353 * contain exception type related info so we cannot decide
2354 * the exception type purely based on exception return packet.
2355 * If we record the exception number from exception packet and
2356 * reuse it for exception return packet, this is not reliable
2357 * due the trace can be discontinuity or the interrupt can
2358 * be nested, thus the recorded exception number cannot be
2359 * used for exception return packet for these two cases.
2360 *
2361 * For exception return packet, we only need to distinguish the
2362 * packet is for system call or for other types. Thus the
2363 * decision can be deferred when receive the next packet which
2364 * contains the return address, based on the return address we
2365 * can read out the previous instruction and check if it's a
2366 * system call instruction and then calibrate the sample flag
2367 * as needed.
2368 */
2369 if (prev_packet->sample_type == CS_ETM_RANGE)
2370 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2371 PERF_IP_FLAG_RETURN |
2372 PERF_IP_FLAG_INTERRUPT;
2373 break;
2374 case CS_ETM_EMPTY:
2375 default:
2376 break;
2377 }
2378
2379 return 0;
2380}
2381
2382static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2383{
2384 int ret = 0;
2385 size_t processed = 0;
2386
2387 /*
2388 * Packets are decoded and added to the decoder's packet queue
2389 * until the decoder packet processing callback has requested that
2390 * processing stops or there is nothing left in the buffer. Normal
2391 * operations that stop processing are a timestamp packet or a full
2392 * decoder buffer queue.
2393 */
2394 ret = cs_etm_decoder__process_data_block(etmq->decoder,
2395 etmq->offset,
2396 &etmq->buf[etmq->buf_used],
2397 etmq->buf_len,
2398 &processed);
2399 if (ret)
2400 goto out;
2401
2402 etmq->offset += processed;
2403 etmq->buf_used += processed;
2404 etmq->buf_len -= processed;
2405
2406out:
2407 return ret;
2408}
2409
2410static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2411 struct cs_etm_traceid_queue *tidq)
2412{
2413 int ret;
2414 struct cs_etm_packet_queue *packet_queue;
2415
2416 packet_queue = &tidq->packet_queue;
2417
2418 /* Process each packet in this chunk */
2419 while (1) {
2420 ret = cs_etm_decoder__get_packet(packet_queue,
2421 tidq->packet);
2422 if (ret <= 0)
2423 /*
2424 * Stop processing this chunk on
2425 * end of data or error
2426 */
2427 break;
2428
2429 /*
2430 * Since packet addresses are swapped in packet
2431 * handling within below switch() statements,
2432 * thus setting sample flags must be called
2433 * prior to switch() statement to use address
2434 * information before packets swapping.
2435 */
2436 ret = cs_etm__set_sample_flags(etmq, tidq);
2437 if (ret < 0)
2438 break;
2439
2440 switch (tidq->packet->sample_type) {
2441 case CS_ETM_RANGE:
2442 /*
2443 * If the packet contains an instruction
2444 * range, generate instruction sequence
2445 * events.
2446 */
2447 cs_etm__sample(etmq, tidq);
2448 break;
2449 case CS_ETM_EXCEPTION:
2450 case CS_ETM_EXCEPTION_RET:
2451 /*
2452 * If the exception packet is coming,
2453 * make sure the previous instruction
2454 * range packet to be handled properly.
2455 */
2456 cs_etm__exception(tidq);
2457 break;
2458 case CS_ETM_DISCONTINUITY:
2459 /*
2460 * Discontinuity in trace, flush
2461 * previous branch stack
2462 */
2463 cs_etm__flush(etmq, tidq);
2464 break;
2465 case CS_ETM_EMPTY:
2466 /*
2467 * Should not receive empty packet,
2468 * report error.
2469 */
2470 pr_err("CS ETM Trace: empty packet\n");
2471 return -EINVAL;
2472 default:
2473 break;
2474 }
2475 }
2476
2477 return ret;
2478}
2479
2480static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2481{
2482 int idx;
2483 struct int_node *inode;
2484 struct cs_etm_traceid_queue *tidq;
2485 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2486
2487 intlist__for_each_entry(inode, traceid_queues_list) {
2488 idx = (int)(intptr_t)inode->priv;
2489 tidq = etmq->traceid_queues[idx];
2490
2491 /* Ignore return value */
2492 cs_etm__process_traceid_queue(etmq, tidq);
2493 }
2494}
2495
2496static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq)
2497{
2498 int err = 0;
2499 struct cs_etm_traceid_queue *tidq;
2500
2501 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2502 if (!tidq)
2503 return -EINVAL;
2504
2505 /* Go through each buffer in the queue and decode them one by one */
2506 while (1) {
2507 err = cs_etm__get_data_block(etmq);
2508 if (err <= 0)
2509 return err;
2510
2511 /* Run trace decoder until buffer consumed or end of trace */
2512 do {
2513 err = cs_etm__decode_data_block(etmq);
2514 if (err)
2515 return err;
2516
2517 /*
2518 * Process each packet in this chunk, nothing to do if
2519 * an error occurs other than hoping the next one will
2520 * be better.
2521 */
2522 err = cs_etm__process_traceid_queue(etmq, tidq);
2523
2524 } while (etmq->buf_len);
2525
2526 if (err == 0)
2527 /* Flush any remaining branch stack entries */
2528 err = cs_etm__end_block(etmq, tidq);
2529 }
2530
2531 return err;
2532}
2533
2534static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq)
2535{
2536 int idx, err = 0;
2537 struct cs_etm_traceid_queue *tidq;
2538 struct int_node *inode;
2539
2540 /* Go through each buffer in the queue and decode them one by one */
2541 while (1) {
2542 err = cs_etm__get_data_block(etmq);
2543 if (err <= 0)
2544 return err;
2545
2546 /* Run trace decoder until buffer consumed or end of trace */
2547 do {
2548 err = cs_etm__decode_data_block(etmq);
2549 if (err)
2550 return err;
2551
2552 /*
2553 * cs_etm__run_per_thread_timeless_decoder() runs on a
2554 * single traceID queue because each TID has a separate
2555 * buffer. But here in per-cpu mode we need to iterate
2556 * over each channel instead.
2557 */
2558 intlist__for_each_entry(inode,
2559 etmq->traceid_queues_list) {
2560 idx = (int)(intptr_t)inode->priv;
2561 tidq = etmq->traceid_queues[idx];
2562 cs_etm__process_traceid_queue(etmq, tidq);
2563 }
2564 } while (etmq->buf_len);
2565
2566 intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2567 idx = (int)(intptr_t)inode->priv;
2568 tidq = etmq->traceid_queues[idx];
2569 /* Flush any remaining branch stack entries */
2570 err = cs_etm__end_block(etmq, tidq);
2571 if (err)
2572 return err;
2573 }
2574 }
2575
2576 return err;
2577}
2578
2579static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2580 pid_t tid)
2581{
2582 unsigned int i;
2583 struct auxtrace_queues *queues = &etm->queues;
2584
2585 for (i = 0; i < queues->nr_queues; i++) {
2586 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2587 struct cs_etm_queue *etmq = queue->priv;
2588 struct cs_etm_traceid_queue *tidq;
2589
2590 if (!etmq)
2591 continue;
2592
2593 if (etm->per_thread_decoding) {
2594 tidq = cs_etm__etmq_get_traceid_queue(
2595 etmq, CS_ETM_PER_THREAD_TRACEID);
2596
2597 if (!tidq)
2598 continue;
2599
2600 if (tid == -1 || thread__tid(tidq->thread) == tid)
2601 cs_etm__run_per_thread_timeless_decoder(etmq);
2602 } else
2603 cs_etm__run_per_cpu_timeless_decoder(etmq);
2604 }
2605
2606 return 0;
2607}
2608
2609static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)
2610{
2611 int ret = 0;
2612 unsigned int cs_queue_nr, queue_nr, i;
2613 u8 trace_chan_id;
2614 u64 cs_timestamp;
2615 struct auxtrace_queue *queue;
2616 struct cs_etm_queue *etmq;
2617 struct cs_etm_traceid_queue *tidq;
2618
2619 /*
2620 * Pre-populate the heap with one entry from each queue so that we can
2621 * start processing in time order across all queues.
2622 */
2623 for (i = 0; i < etm->queues.nr_queues; i++) {
2624 etmq = etm->queues.queue_array[i].priv;
2625 if (!etmq)
2626 continue;
2627
2628 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2629 if (ret)
2630 return ret;
2631 }
2632
2633 while (1) {
2634 if (!etm->heap.heap_cnt)
2635 break;
2636
2637 /* Take the entry at the top of the min heap */
2638 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2639 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2640 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2641 queue = &etm->queues.queue_array[queue_nr];
2642 etmq = queue->priv;
2643
2644 /*
2645 * Remove the top entry from the heap since we are about
2646 * to process it.
2647 */
2648 auxtrace_heap__pop(&etm->heap);
2649
2650 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2651 if (!tidq) {
2652 /*
2653 * No traceID queue has been allocated for this traceID,
2654 * which means something somewhere went very wrong. No
2655 * other choice than simply exit.
2656 */
2657 ret = -EINVAL;
2658 goto out;
2659 }
2660
2661 /*
2662 * Packets associated with this timestamp are already in
2663 * the etmq's traceID queue, so process them.
2664 */
2665 ret = cs_etm__process_traceid_queue(etmq, tidq);
2666 if (ret < 0)
2667 goto out;
2668
2669 /*
2670 * Packets for this timestamp have been processed, time to
2671 * move on to the next timestamp, fetching a new auxtrace_buffer
2672 * if need be.
2673 */
2674refetch:
2675 ret = cs_etm__get_data_block(etmq);
2676 if (ret < 0)
2677 goto out;
2678
2679 /*
2680 * No more auxtrace_buffers to process in this etmq, simply
2681 * move on to another entry in the auxtrace_heap.
2682 */
2683 if (!ret)
2684 continue;
2685
2686 ret = cs_etm__decode_data_block(etmq);
2687 if (ret)
2688 goto out;
2689
2690 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2691
2692 if (!cs_timestamp) {
2693 /*
2694 * Function cs_etm__decode_data_block() returns when
2695 * there is no more traces to decode in the current
2696 * auxtrace_buffer OR when a timestamp has been
2697 * encountered on any of the traceID queues. Since we
2698 * did not get a timestamp, there is no more traces to
2699 * process in this auxtrace_buffer. As such empty and
2700 * flush all traceID queues.
2701 */
2702 cs_etm__clear_all_traceid_queues(etmq);
2703
2704 /* Fetch another auxtrace_buffer for this etmq */
2705 goto refetch;
2706 }
2707
2708 /*
2709 * Add to the min heap the timestamp for packets that have
2710 * just been decoded. They will be processed and synthesized
2711 * during the next call to cs_etm__process_traceid_queue() for
2712 * this queue/traceID.
2713 */
2714 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2715 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2716 }
2717
2718 for (i = 0; i < etm->queues.nr_queues; i++) {
2719 struct int_node *inode;
2720
2721 etmq = etm->queues.queue_array[i].priv;
2722 if (!etmq)
2723 continue;
2724
2725 intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2726 int idx = (int)(intptr_t)inode->priv;
2727
2728 /* Flush any remaining branch stack entries */
2729 tidq = etmq->traceid_queues[idx];
2730 ret = cs_etm__end_block(etmq, tidq);
2731 if (ret)
2732 return ret;
2733 }
2734 }
2735out:
2736 return ret;
2737}
2738
2739static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2740 union perf_event *event)
2741{
2742 struct thread *th;
2743
2744 if (etm->timeless_decoding)
2745 return 0;
2746
2747 /*
2748 * Add the tid/pid to the log so that we can get a match when we get a
2749 * contextID from the decoder. Only track for the host: only kernel
2750 * trace is supported for guests which wouldn't need pids so this should
2751 * be fine.
2752 */
2753 th = machine__findnew_thread(&etm->session->machines.host,
2754 event->itrace_start.pid,
2755 event->itrace_start.tid);
2756 if (!th)
2757 return -ENOMEM;
2758
2759 thread__put(th);
2760
2761 return 0;
2762}
2763
2764static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2765 union perf_event *event)
2766{
2767 struct thread *th;
2768 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2769
2770 /*
2771 * Context switch in per-thread mode are irrelevant since perf
2772 * will start/stop tracing as the process is scheduled.
2773 */
2774 if (etm->timeless_decoding)
2775 return 0;
2776
2777 /*
2778 * SWITCH_IN events carry the next process to be switched out while
2779 * SWITCH_OUT events carry the process to be switched in. As such
2780 * we don't care about IN events.
2781 */
2782 if (!out)
2783 return 0;
2784
2785 /*
2786 * Add the tid/pid to the log so that we can get a match when we get a
2787 * contextID from the decoder. Only track for the host: only kernel
2788 * trace is supported for guests which wouldn't need pids so this should
2789 * be fine.
2790 */
2791 th = machine__findnew_thread(&etm->session->machines.host,
2792 event->context_switch.next_prev_pid,
2793 event->context_switch.next_prev_tid);
2794 if (!th)
2795 return -ENOMEM;
2796
2797 thread__put(th);
2798
2799 return 0;
2800}
2801
2802static int cs_etm__process_event(struct perf_session *session,
2803 union perf_event *event,
2804 struct perf_sample *sample,
2805 const struct perf_tool *tool)
2806{
2807 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2808 struct cs_etm_auxtrace,
2809 auxtrace);
2810
2811 if (dump_trace)
2812 return 0;
2813
2814 if (!tool->ordered_events) {
2815 pr_err("CoreSight ETM Trace requires ordered events\n");
2816 return -EINVAL;
2817 }
2818
2819 switch (event->header.type) {
2820 case PERF_RECORD_EXIT:
2821 /*
2822 * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2823 * start the decode because we know there will be no more trace from
2824 * this thread. All this does is emit samples earlier than waiting for
2825 * the flush in other modes, but with timestamps it makes sense to wait
2826 * for flush so that events from different threads are interleaved
2827 * properly.
2828 */
2829 if (etm->per_thread_decoding && etm->timeless_decoding)
2830 return cs_etm__process_timeless_queues(etm,
2831 event->fork.tid);
2832 break;
2833
2834 case PERF_RECORD_ITRACE_START:
2835 return cs_etm__process_itrace_start(etm, event);
2836
2837 case PERF_RECORD_SWITCH_CPU_WIDE:
2838 return cs_etm__process_switch_cpu_wide(etm, event);
2839
2840 case PERF_RECORD_AUX:
2841 /*
2842 * Record the latest kernel timestamp available in the header
2843 * for samples so that synthesised samples occur from this point
2844 * onwards.
2845 */
2846 if (sample->time && (sample->time != (u64)-1))
2847 etm->latest_kernel_timestamp = sample->time;
2848 break;
2849
2850 default:
2851 break;
2852 }
2853
2854 return 0;
2855}
2856
2857static void dump_queued_data(struct cs_etm_auxtrace *etm,
2858 struct perf_record_auxtrace *event)
2859{
2860 struct auxtrace_buffer *buf;
2861 unsigned int i;
2862 /*
2863 * Find all buffers with same reference in the queues and dump them.
2864 * This is because the queues can contain multiple entries of the same
2865 * buffer that were split on aux records.
2866 */
2867 for (i = 0; i < etm->queues.nr_queues; ++i)
2868 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2869 if (buf->reference == event->reference)
2870 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2871}
2872
2873static int cs_etm__process_auxtrace_event(struct perf_session *session,
2874 union perf_event *event,
2875 const struct perf_tool *tool __maybe_unused)
2876{
2877 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2878 struct cs_etm_auxtrace,
2879 auxtrace);
2880 if (!etm->data_queued) {
2881 struct auxtrace_buffer *buffer;
2882 off_t data_offset;
2883 int fd = perf_data__fd(session->data);
2884 bool is_pipe = perf_data__is_pipe(session->data);
2885 int err;
2886 int idx = event->auxtrace.idx;
2887
2888 if (is_pipe)
2889 data_offset = 0;
2890 else {
2891 data_offset = lseek(fd, 0, SEEK_CUR);
2892 if (data_offset == -1)
2893 return -errno;
2894 }
2895
2896 err = auxtrace_queues__add_event(&etm->queues, session,
2897 event, data_offset, &buffer);
2898 if (err)
2899 return err;
2900
2901 if (dump_trace)
2902 if (auxtrace_buffer__get_data(buffer, fd)) {
2903 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2904 auxtrace_buffer__put_data(buffer);
2905 }
2906 } else if (dump_trace)
2907 dump_queued_data(etm, &event->auxtrace);
2908
2909 return 0;
2910}
2911
2912static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2913{
2914 struct evsel *evsel;
2915 struct evlist *evlist = etm->session->evlist;
2916
2917 /* Override timeless mode with user input from --itrace=Z */
2918 if (etm->synth_opts.timeless_decoding) {
2919 etm->timeless_decoding = true;
2920 return 0;
2921 }
2922
2923 /*
2924 * Find the cs_etm evsel and look at what its timestamp setting was
2925 */
2926 evlist__for_each_entry(evlist, evsel)
2927 if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2928 etm->timeless_decoding =
2929 !(evsel->core.attr.config & BIT(ETM_OPT_TS));
2930 return 0;
2931 }
2932
2933 pr_err("CS ETM: Couldn't find ETM evsel\n");
2934 return -EINVAL;
2935}
2936
2937/*
2938 * Read a single cpu parameter block from the auxtrace_info priv block.
2939 *
2940 * For version 1 there is a per cpu nr_params entry. If we are handling
2941 * version 1 file, then there may be less, the same, or more params
2942 * indicated by this value than the compile time number we understand.
2943 *
2944 * For a version 0 info block, there are a fixed number, and we need to
2945 * fill out the nr_param value in the metadata we create.
2946 */
2947static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2948 int out_blk_size, int nr_params_v0)
2949{
2950 u64 *metadata = NULL;
2951 int hdr_version;
2952 int nr_in_params, nr_out_params, nr_cmn_params;
2953 int i, k;
2954
2955 metadata = zalloc(sizeof(*metadata) * out_blk_size);
2956 if (!metadata)
2957 return NULL;
2958
2959 /* read block current index & version */
2960 i = *buff_in_offset;
2961 hdr_version = buff_in[CS_HEADER_VERSION];
2962
2963 if (!hdr_version) {
2964 /* read version 0 info block into a version 1 metadata block */
2965 nr_in_params = nr_params_v0;
2966 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2967 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2968 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2969 /* remaining block params at offset +1 from source */
2970 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2971 metadata[k + 1] = buff_in[i + k];
2972 /* version 0 has 2 common params */
2973 nr_cmn_params = 2;
2974 } else {
2975 /* read version 1 info block - input and output nr_params may differ */
2976 /* version 1 has 3 common params */
2977 nr_cmn_params = 3;
2978 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2979
2980 /* if input has more params than output - skip excess */
2981 nr_out_params = nr_in_params + nr_cmn_params;
2982 if (nr_out_params > out_blk_size)
2983 nr_out_params = out_blk_size;
2984
2985 for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2986 metadata[k] = buff_in[i + k];
2987
2988 /* record the actual nr params we copied */
2989 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2990 }
2991
2992 /* adjust in offset by number of in params used */
2993 i += nr_in_params + nr_cmn_params;
2994 *buff_in_offset = i;
2995 return metadata;
2996}
2997
2998/**
2999 * Puts a fragment of an auxtrace buffer into the auxtrace queues based
3000 * on the bounds of aux_event, if it matches with the buffer that's at
3001 * file_offset.
3002 *
3003 * Normally, whole auxtrace buffers would be added to the queue. But we
3004 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
3005 * is reset across each buffer, so splitting the buffers up in advance has
3006 * the same effect.
3007 */
3008static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
3009 struct perf_record_aux *aux_event, struct perf_sample *sample)
3010{
3011 int err;
3012 char buf[PERF_SAMPLE_MAX_SIZE];
3013 union perf_event *auxtrace_event_union;
3014 struct perf_record_auxtrace *auxtrace_event;
3015 union perf_event auxtrace_fragment;
3016 __u64 aux_offset, aux_size;
3017 enum cs_etm_format format;
3018
3019 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
3020 struct cs_etm_auxtrace,
3021 auxtrace);
3022
3023 /*
3024 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
3025 * from looping through the auxtrace index.
3026 */
3027 err = perf_session__peek_event(session, file_offset, buf,
3028 PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
3029 if (err)
3030 return err;
3031 auxtrace_event = &auxtrace_event_union->auxtrace;
3032 if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
3033 return -EINVAL;
3034
3035 if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
3036 auxtrace_event->header.size != sz) {
3037 return -EINVAL;
3038 }
3039
3040 /*
3041 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
3042 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
3043 * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
3044 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
3045 * Return 'not found' if mismatch.
3046 */
3047 if (auxtrace_event->cpu == (__u32) -1) {
3048 etm->per_thread_decoding = true;
3049 if (auxtrace_event->tid != sample->tid)
3050 return 1;
3051 } else if (auxtrace_event->cpu != sample->cpu) {
3052 if (etm->per_thread_decoding) {
3053 /*
3054 * Found a per-cpu buffer after a per-thread one was
3055 * already found
3056 */
3057 pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n");
3058 return -EINVAL;
3059 }
3060 return 1;
3061 }
3062
3063 if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
3064 /*
3065 * Clamp size in snapshot mode. The buffer size is clamped in
3066 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
3067 * the buffer size.
3068 */
3069 aux_size = min(aux_event->aux_size, auxtrace_event->size);
3070
3071 /*
3072 * In this mode, the head also points to the end of the buffer so aux_offset
3073 * needs to have the size subtracted so it points to the beginning as in normal mode
3074 */
3075 aux_offset = aux_event->aux_offset - aux_size;
3076 } else {
3077 aux_size = aux_event->aux_size;
3078 aux_offset = aux_event->aux_offset;
3079 }
3080
3081 if (aux_offset >= auxtrace_event->offset &&
3082 aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
3083 struct cs_etm_queue *etmq = etm->queues.queue_array[auxtrace_event->idx].priv;
3084
3085 /*
3086 * If this AUX event was inside this buffer somewhere, create a new auxtrace event
3087 * based on the sizes of the aux event, and queue that fragment.
3088 */
3089 auxtrace_fragment.auxtrace = *auxtrace_event;
3090 auxtrace_fragment.auxtrace.size = aux_size;
3091 auxtrace_fragment.auxtrace.offset = aux_offset;
3092 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
3093
3094 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
3095 " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
3096 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
3097 file_offset, NULL);
3098 if (err)
3099 return err;
3100
3101 format = (aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW) ?
3102 UNFORMATTED : FORMATTED;
3103 if (etmq->format != UNSET && format != etmq->format) {
3104 pr_err("CS_ETM: mixed formatted and unformatted trace not supported\n");
3105 return -EINVAL;
3106 }
3107 etmq->format = format;
3108 return 0;
3109 }
3110
3111 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
3112 return 1;
3113}
3114
3115static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
3116 u64 offset __maybe_unused, void *data __maybe_unused)
3117{
3118 /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
3119 if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
3120 (*(int *)data)++; /* increment found count */
3121 return cs_etm__process_aux_output_hw_id(session, event);
3122 }
3123 return 0;
3124}
3125
3126static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
3127 u64 offset __maybe_unused, void *data __maybe_unused)
3128{
3129 struct perf_sample sample;
3130 int ret;
3131 struct auxtrace_index_entry *ent;
3132 struct auxtrace_index *auxtrace_index;
3133 struct evsel *evsel;
3134 size_t i;
3135
3136 /* Don't care about any other events, we're only queuing buffers for AUX events */
3137 if (event->header.type != PERF_RECORD_AUX)
3138 return 0;
3139
3140 if (event->header.size < sizeof(struct perf_record_aux))
3141 return -EINVAL;
3142
3143 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
3144 if (!event->aux.aux_size)
3145 return 0;
3146
3147 /*
3148 * Parse the sample, we need the sample_id_all data that comes after the event so that the
3149 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
3150 */
3151 evsel = evlist__event2evsel(session->evlist, event);
3152 if (!evsel)
3153 return -EINVAL;
3154 ret = evsel__parse_sample(evsel, event, &sample);
3155 if (ret)
3156 return ret;
3157
3158 /*
3159 * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3160 */
3161 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
3162 for (i = 0; i < auxtrace_index->nr; i++) {
3163 ent = &auxtrace_index->entries[i];
3164 ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
3165 ent->sz, &event->aux, &sample);
3166 /*
3167 * Stop search on error or successful values. Continue search on
3168 * 1 ('not found')
3169 */
3170 if (ret != 1)
3171 return ret;
3172 }
3173 }
3174
3175 /*
3176 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3177 * don't exit with an error because it will still be possible to decode other aux records.
3178 */
3179 pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
3180 " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
3181 return 0;
3182}
3183
3184static int cs_etm__queue_aux_records(struct perf_session *session)
3185{
3186 struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
3187 struct auxtrace_index, list);
3188 if (index && index->nr > 0)
3189 return perf_session__peek_events(session, session->header.data_offset,
3190 session->header.data_size,
3191 cs_etm__queue_aux_records_cb, NULL);
3192
3193 /*
3194 * We would get here if there are no entries in the index (either no auxtrace
3195 * buffers or no index at all). Fail silently as there is the possibility of
3196 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3197 * false.
3198 *
3199 * In that scenario, buffers will not be split by AUX records.
3200 */
3201 return 0;
3202}
3203
3204#define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3205 (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3206
3207/*
3208 * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3209 * timestamps).
3210 */
3211static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
3212{
3213 int j;
3214
3215 for (j = 0; j < num_cpu; j++) {
3216 switch (metadata[j][CS_ETM_MAGIC]) {
3217 case __perf_cs_etmv4_magic:
3218 if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
3219 return false;
3220 break;
3221 case __perf_cs_ete_magic:
3222 if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
3223 return false;
3224 break;
3225 default:
3226 /* Unknown / unsupported magic number. */
3227 return false;
3228 }
3229 }
3230 return true;
3231}
3232
3233/* map trace ids to correct metadata block, from information in metadata */
3234static int cs_etm__map_trace_ids_metadata(struct cs_etm_auxtrace *etm, int num_cpu,
3235 u64 **metadata)
3236{
3237 u64 cs_etm_magic;
3238 u8 trace_chan_id;
3239 int i, err;
3240
3241 for (i = 0; i < num_cpu; i++) {
3242 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3243 switch (cs_etm_magic) {
3244 case __perf_cs_etmv3_magic:
3245 metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3246 trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3247 break;
3248 case __perf_cs_etmv4_magic:
3249 case __perf_cs_ete_magic:
3250 metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3251 trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3252 break;
3253 default:
3254 /* unknown magic number */
3255 return -EINVAL;
3256 }
3257 err = cs_etm__map_trace_id_v0(etm, trace_chan_id, metadata[i]);
3258 if (err)
3259 return err;
3260 }
3261 return 0;
3262}
3263
3264/*
3265 * Use the data gathered by the peeks for HW_ID (trace ID mappings) and AUX
3266 * (formatted or not) packets to create the decoders.
3267 */
3268static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq)
3269{
3270 struct cs_etm_decoder_params d_params;
3271 struct cs_etm_trace_params *t_params;
3272 int decoders = intlist__nr_entries(etmq->traceid_list);
3273
3274 if (decoders == 0)
3275 return 0;
3276
3277 /*
3278 * Each queue can only contain data from one CPU when unformatted, so only one decoder is
3279 * needed.
3280 */
3281 if (etmq->format == UNFORMATTED)
3282 assert(decoders == 1);
3283
3284 /* Use metadata to fill in trace parameters for trace decoder */
3285 t_params = zalloc(sizeof(*t_params) * decoders);
3286
3287 if (!t_params)
3288 goto out_free;
3289
3290 if (cs_etm__init_trace_params(t_params, etmq))
3291 goto out_free;
3292
3293 /* Set decoder parameters to decode trace packets */
3294 if (cs_etm__init_decoder_params(&d_params, etmq,
3295 dump_trace ? CS_ETM_OPERATION_PRINT :
3296 CS_ETM_OPERATION_DECODE))
3297 goto out_free;
3298
3299 etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
3300 t_params);
3301
3302 if (!etmq->decoder)
3303 goto out_free;
3304
3305 /*
3306 * Register a function to handle all memory accesses required by
3307 * the trace decoder library.
3308 */
3309 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
3310 0x0L, ((u64) -1L),
3311 cs_etm__mem_access))
3312 goto out_free_decoder;
3313
3314 zfree(&t_params);
3315 return 0;
3316
3317out_free_decoder:
3318 cs_etm_decoder__free(etmq->decoder);
3319out_free:
3320 zfree(&t_params);
3321 return -EINVAL;
3322}
3323
3324static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm)
3325{
3326 struct auxtrace_queues *queues = &etm->queues;
3327
3328 for (unsigned int i = 0; i < queues->nr_queues; i++) {
3329 bool empty = list_empty(&queues->queue_array[i].head);
3330 struct cs_etm_queue *etmq = queues->queue_array[i].priv;
3331 int ret;
3332
3333 /*
3334 * Don't create decoders for empty queues, mainly because
3335 * etmq->format is unknown for empty queues.
3336 */
3337 assert(empty || etmq->format != UNSET);
3338 if (empty)
3339 continue;
3340
3341 ret = cs_etm__create_queue_decoders(etmq);
3342 if (ret)
3343 return ret;
3344 }
3345 return 0;
3346}
3347
3348int cs_etm__process_auxtrace_info_full(union perf_event *event,
3349 struct perf_session *session)
3350{
3351 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3352 struct cs_etm_auxtrace *etm = NULL;
3353 struct perf_record_time_conv *tc = &session->time_conv;
3354 int event_header_size = sizeof(struct perf_event_header);
3355 int total_size = auxtrace_info->header.size;
3356 int priv_size = 0;
3357 int num_cpu, max_cpu = 0;
3358 int err = 0;
3359 int aux_hw_id_found;
3360 int i;
3361 u64 *ptr = NULL;
3362 u64 **metadata = NULL;
3363
3364 /* First the global part */
3365 ptr = (u64 *) auxtrace_info->priv;
3366 num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3367 metadata = zalloc(sizeof(*metadata) * num_cpu);
3368 if (!metadata)
3369 return -ENOMEM;
3370
3371 /* Start parsing after the common part of the header */
3372 i = CS_HEADER_VERSION_MAX;
3373
3374 /*
3375 * The metadata is stored in the auxtrace_info section and encodes
3376 * the configuration of the ARM embedded trace macrocell which is
3377 * required by the trace decoder to properly decode the trace due
3378 * to its highly compressed nature.
3379 */
3380 for (int j = 0; j < num_cpu; j++) {
3381 if (ptr[i] == __perf_cs_etmv3_magic) {
3382 metadata[j] =
3383 cs_etm__create_meta_blk(ptr, &i,
3384 CS_ETM_PRIV_MAX,
3385 CS_ETM_NR_TRC_PARAMS_V0);
3386 } else if (ptr[i] == __perf_cs_etmv4_magic) {
3387 metadata[j] =
3388 cs_etm__create_meta_blk(ptr, &i,
3389 CS_ETMV4_PRIV_MAX,
3390 CS_ETMV4_NR_TRC_PARAMS_V0);
3391 } else if (ptr[i] == __perf_cs_ete_magic) {
3392 metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3393 } else {
3394 ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3395 ptr[i]);
3396 err = -EINVAL;
3397 goto err_free_metadata;
3398 }
3399
3400 if (!metadata[j]) {
3401 err = -ENOMEM;
3402 goto err_free_metadata;
3403 }
3404
3405 if ((int) metadata[j][CS_ETM_CPU] > max_cpu)
3406 max_cpu = metadata[j][CS_ETM_CPU];
3407 }
3408
3409 /*
3410 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3411 * CS_ETMV4_PRIV_MAX mark how many double words are in the
3412 * global metadata, and each cpu's metadata respectively.
3413 * The following tests if the correct number of double words was
3414 * present in the auxtrace info section.
3415 */
3416 priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3417 if (i * 8 != priv_size) {
3418 err = -EINVAL;
3419 goto err_free_metadata;
3420 }
3421
3422 etm = zalloc(sizeof(*etm));
3423
3424 if (!etm) {
3425 err = -ENOMEM;
3426 goto err_free_metadata;
3427 }
3428
3429 /*
3430 * As all the ETMs run at the same exception level, the system should
3431 * have the same PID format crossing CPUs. So cache the PID format
3432 * and reuse it for sequential decoding.
3433 */
3434 etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]);
3435
3436 err = auxtrace_queues__init_nr(&etm->queues, max_cpu + 1);
3437 if (err)
3438 goto err_free_etm;
3439
3440 for (unsigned int j = 0; j < etm->queues.nr_queues; ++j) {
3441 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[j], j);
3442 if (err)
3443 goto err_free_queues;
3444 }
3445
3446 if (session->itrace_synth_opts->set) {
3447 etm->synth_opts = *session->itrace_synth_opts;
3448 } else {
3449 itrace_synth_opts__set_default(&etm->synth_opts,
3450 session->itrace_synth_opts->default_no_sample);
3451 etm->synth_opts.callchain = false;
3452 }
3453
3454 etm->session = session;
3455
3456 etm->num_cpu = num_cpu;
3457 etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3458 etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3459 etm->metadata = metadata;
3460 etm->auxtrace_type = auxtrace_info->type;
3461
3462 if (etm->synth_opts.use_timestamp)
3463 /*
3464 * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature,
3465 * therefore the decoder cannot know if the timestamp trace is
3466 * same with the kernel time.
3467 *
3468 * If a user has knowledge for the working platform and can
3469 * specify itrace option 'T' to tell decoder to forcely use the
3470 * traced timestamp as the kernel time.
3471 */
3472 etm->has_virtual_ts = true;
3473 else
3474 /* Use virtual timestamps if all ETMs report ts_source = 1 */
3475 etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3476
3477 if (!etm->has_virtual_ts)
3478 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3479 "The time field of the samples will not be set accurately.\n"
3480 "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n"
3481 "you can specify the itrace option 'T' for timestamp decoding\n"
3482 "if the Coresight timestamp on the platform is same with the kernel time.\n\n");
3483
3484 etm->auxtrace.process_event = cs_etm__process_event;
3485 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3486 etm->auxtrace.flush_events = cs_etm__flush_events;
3487 etm->auxtrace.free_events = cs_etm__free_events;
3488 etm->auxtrace.free = cs_etm__free;
3489 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3490 session->auxtrace = &etm->auxtrace;
3491
3492 err = cs_etm__setup_timeless_decoding(etm);
3493 if (err)
3494 return err;
3495
3496 etm->tc.time_shift = tc->time_shift;
3497 etm->tc.time_mult = tc->time_mult;
3498 etm->tc.time_zero = tc->time_zero;
3499 if (event_contains(*tc, time_cycles)) {
3500 etm->tc.time_cycles = tc->time_cycles;
3501 etm->tc.time_mask = tc->time_mask;
3502 etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3503 etm->tc.cap_user_time_short = tc->cap_user_time_short;
3504 }
3505 err = cs_etm__synth_events(etm, session);
3506 if (err)
3507 goto err_free_queues;
3508
3509 err = cs_etm__queue_aux_records(session);
3510 if (err)
3511 goto err_free_queues;
3512
3513 /*
3514 * Map Trace ID values to CPU metadata.
3515 *
3516 * Trace metadata will always contain Trace ID values from the legacy algorithm
3517 * in case it's read by a version of Perf that doesn't know about HW_ID packets
3518 * or the kernel doesn't emit them.
3519 *
3520 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3521 * the same IDs as the old algorithm as far as is possible, unless there are clashes
3522 * in which case a different value will be used. This means an older perf may still
3523 * be able to record and read files generate on a newer system.
3524 *
3525 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3526 * those packets. If they are there then the values will be mapped and plugged into
3527 * the metadata and decoders are only created for each mapping received.
3528 *
3529 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3530 * then we map Trace ID values to CPU directly from the metadata and create decoders
3531 * for all mappings.
3532 */
3533
3534 /* Scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3535 aux_hw_id_found = 0;
3536 err = perf_session__peek_events(session, session->header.data_offset,
3537 session->header.data_size,
3538 cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3539 if (err)
3540 goto err_free_queues;
3541
3542 /* if no HW ID found this is a file with metadata values only, map from metadata */
3543 if (!aux_hw_id_found) {
3544 err = cs_etm__map_trace_ids_metadata(etm, num_cpu, metadata);
3545 if (err)
3546 goto err_free_queues;
3547 }
3548
3549 err = cs_etm__create_decoders(etm);
3550 if (err)
3551 goto err_free_queues;
3552
3553 etm->data_queued = etm->queues.populated;
3554 return 0;
3555
3556err_free_queues:
3557 auxtrace_queues__free(&etm->queues);
3558 session->auxtrace = NULL;
3559err_free_etm:
3560 zfree(&etm);
3561err_free_metadata:
3562 /* No need to check @metadata[j], free(NULL) is supported */
3563 for (int j = 0; j < num_cpu; j++)
3564 zfree(&metadata[j]);
3565 zfree(&metadata);
3566 return err;
3567}