Loading...
1/*
2 * builtin-record.c
3 *
4 * Builtin record command: Record the profile of a workload
5 * (or a CPU, or a PID) into the perf.data output file - for
6 * later analysis via perf report.
7 */
8#include "builtin.h"
9
10#include "perf.h"
11
12#include "util/build-id.h"
13#include "util/util.h"
14#include "util/parse-options.h"
15#include "util/parse-events.h"
16
17#include "util/header.h"
18#include "util/event.h"
19#include "util/evlist.h"
20#include "util/evsel.h"
21#include "util/debug.h"
22#include "util/session.h"
23#include "util/tool.h"
24#include "util/symbol.h"
25#include "util/cpumap.h"
26#include "util/thread_map.h"
27#include "util/data.h"
28
29#include <unistd.h>
30#include <sched.h>
31#include <sys/mman.h>
32
33#ifndef HAVE_ON_EXIT_SUPPORT
34#ifndef ATEXIT_MAX
35#define ATEXIT_MAX 32
36#endif
37static int __on_exit_count = 0;
38typedef void (*on_exit_func_t) (int, void *);
39static on_exit_func_t __on_exit_funcs[ATEXIT_MAX];
40static void *__on_exit_args[ATEXIT_MAX];
41static int __exitcode = 0;
42static void __handle_on_exit_funcs(void);
43static int on_exit(on_exit_func_t function, void *arg);
44#define exit(x) (exit)(__exitcode = (x))
45
46static int on_exit(on_exit_func_t function, void *arg)
47{
48 if (__on_exit_count == ATEXIT_MAX)
49 return -ENOMEM;
50 else if (__on_exit_count == 0)
51 atexit(__handle_on_exit_funcs);
52 __on_exit_funcs[__on_exit_count] = function;
53 __on_exit_args[__on_exit_count++] = arg;
54 return 0;
55}
56
57static void __handle_on_exit_funcs(void)
58{
59 int i;
60 for (i = 0; i < __on_exit_count; i++)
61 __on_exit_funcs[i] (__exitcode, __on_exit_args[i]);
62}
63#endif
64
65struct record {
66 struct perf_tool tool;
67 struct record_opts opts;
68 u64 bytes_written;
69 struct perf_data_file file;
70 struct perf_evlist *evlist;
71 struct perf_session *session;
72 const char *progname;
73 int realtime_prio;
74 bool no_buildid;
75 bool no_buildid_cache;
76 long samples;
77};
78
79static int record__write(struct record *rec, void *bf, size_t size)
80{
81 if (perf_data_file__write(rec->session->file, bf, size) < 0) {
82 pr_err("failed to write perf data, error: %m\n");
83 return -1;
84 }
85
86 rec->bytes_written += size;
87 return 0;
88}
89
90static int process_synthesized_event(struct perf_tool *tool,
91 union perf_event *event,
92 struct perf_sample *sample __maybe_unused,
93 struct machine *machine __maybe_unused)
94{
95 struct record *rec = container_of(tool, struct record, tool);
96 return record__write(rec, event, event->header.size);
97}
98
99static int record__mmap_read(struct record *rec, struct perf_mmap *md)
100{
101 unsigned int head = perf_mmap__read_head(md);
102 unsigned int old = md->prev;
103 unsigned char *data = md->base + page_size;
104 unsigned long size;
105 void *buf;
106 int rc = 0;
107
108 if (old == head)
109 return 0;
110
111 rec->samples++;
112
113 size = head - old;
114
115 if ((old & md->mask) + size != (head & md->mask)) {
116 buf = &data[old & md->mask];
117 size = md->mask + 1 - (old & md->mask);
118 old += size;
119
120 if (record__write(rec, buf, size) < 0) {
121 rc = -1;
122 goto out;
123 }
124 }
125
126 buf = &data[old & md->mask];
127 size = head - old;
128 old += size;
129
130 if (record__write(rec, buf, size) < 0) {
131 rc = -1;
132 goto out;
133 }
134
135 md->prev = old;
136 perf_mmap__write_tail(md, old);
137
138out:
139 return rc;
140}
141
142static volatile int done = 0;
143static volatile int signr = -1;
144static volatile int child_finished = 0;
145
146static void sig_handler(int sig)
147{
148 if (sig == SIGCHLD)
149 child_finished = 1;
150
151 done = 1;
152 signr = sig;
153}
154
155static void record__sig_exit(int exit_status __maybe_unused, void *arg)
156{
157 struct record *rec = arg;
158 int status;
159
160 if (rec->evlist->workload.pid > 0) {
161 if (!child_finished)
162 kill(rec->evlist->workload.pid, SIGTERM);
163
164 wait(&status);
165 if (WIFSIGNALED(status))
166 psignal(WTERMSIG(status), rec->progname);
167 }
168
169 if (signr == -1 || signr == SIGUSR1)
170 return;
171
172 signal(signr, SIG_DFL);
173}
174
175static int record__open(struct record *rec)
176{
177 char msg[512];
178 struct perf_evsel *pos;
179 struct perf_evlist *evlist = rec->evlist;
180 struct perf_session *session = rec->session;
181 struct record_opts *opts = &rec->opts;
182 int rc = 0;
183
184 perf_evlist__config(evlist, opts);
185
186 evlist__for_each(evlist, pos) {
187try_again:
188 if (perf_evsel__open(pos, evlist->cpus, evlist->threads) < 0) {
189 if (perf_evsel__fallback(pos, errno, msg, sizeof(msg))) {
190 if (verbose)
191 ui__warning("%s\n", msg);
192 goto try_again;
193 }
194
195 rc = -errno;
196 perf_evsel__open_strerror(pos, &opts->target,
197 errno, msg, sizeof(msg));
198 ui__error("%s\n", msg);
199 goto out;
200 }
201 }
202
203 if (perf_evlist__apply_filters(evlist)) {
204 error("failed to set filter with %d (%s)\n", errno,
205 strerror(errno));
206 rc = -1;
207 goto out;
208 }
209
210 if (perf_evlist__mmap(evlist, opts->mmap_pages, false) < 0) {
211 if (errno == EPERM) {
212 pr_err("Permission error mapping pages.\n"
213 "Consider increasing "
214 "/proc/sys/kernel/perf_event_mlock_kb,\n"
215 "or try again with a smaller value of -m/--mmap_pages.\n"
216 "(current value: %u)\n", opts->mmap_pages);
217 rc = -errno;
218 } else {
219 pr_err("failed to mmap with %d (%s)\n", errno, strerror(errno));
220 rc = -errno;
221 }
222 goto out;
223 }
224
225 session->evlist = evlist;
226 perf_session__set_id_hdr_size(session);
227out:
228 return rc;
229}
230
231static int process_buildids(struct record *rec)
232{
233 struct perf_data_file *file = &rec->file;
234 struct perf_session *session = rec->session;
235 u64 start = session->header.data_offset;
236
237 u64 size = lseek(file->fd, 0, SEEK_CUR);
238 if (size == 0)
239 return 0;
240
241 return __perf_session__process_events(session, start,
242 size - start,
243 size, &build_id__mark_dso_hit_ops);
244}
245
246static void record__exit(int status, void *arg)
247{
248 struct record *rec = arg;
249 struct perf_data_file *file = &rec->file;
250
251 if (status != 0)
252 return;
253
254 if (!file->is_pipe) {
255 rec->session->header.data_size += rec->bytes_written;
256
257 if (!rec->no_buildid)
258 process_buildids(rec);
259 perf_session__write_header(rec->session, rec->evlist,
260 file->fd, true);
261 perf_session__delete(rec->session);
262 perf_evlist__delete(rec->evlist);
263 symbol__exit();
264 }
265}
266
267static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
268{
269 int err;
270 struct perf_tool *tool = data;
271 /*
272 *As for guest kernel when processing subcommand record&report,
273 *we arrange module mmap prior to guest kernel mmap and trigger
274 *a preload dso because default guest module symbols are loaded
275 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
276 *method is used to avoid symbol missing when the first addr is
277 *in module instead of in guest kernel.
278 */
279 err = perf_event__synthesize_modules(tool, process_synthesized_event,
280 machine);
281 if (err < 0)
282 pr_err("Couldn't record guest kernel [%d]'s reference"
283 " relocation symbol.\n", machine->pid);
284
285 /*
286 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
287 * have no _text sometimes.
288 */
289 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
290 machine);
291 if (err < 0)
292 pr_err("Couldn't record guest kernel [%d]'s reference"
293 " relocation symbol.\n", machine->pid);
294}
295
296static struct perf_event_header finished_round_event = {
297 .size = sizeof(struct perf_event_header),
298 .type = PERF_RECORD_FINISHED_ROUND,
299};
300
301static int record__mmap_read_all(struct record *rec)
302{
303 int i;
304 int rc = 0;
305
306 for (i = 0; i < rec->evlist->nr_mmaps; i++) {
307 if (rec->evlist->mmap[i].base) {
308 if (record__mmap_read(rec, &rec->evlist->mmap[i]) != 0) {
309 rc = -1;
310 goto out;
311 }
312 }
313 }
314
315 if (perf_header__has_feat(&rec->session->header, HEADER_TRACING_DATA))
316 rc = record__write(rec, &finished_round_event, sizeof(finished_round_event));
317
318out:
319 return rc;
320}
321
322static void record__init_features(struct record *rec)
323{
324 struct perf_session *session = rec->session;
325 int feat;
326
327 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
328 perf_header__set_feat(&session->header, feat);
329
330 if (rec->no_buildid)
331 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
332
333 if (!have_tracepoints(&rec->evlist->entries))
334 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
335
336 if (!rec->opts.branch_stack)
337 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
338}
339
340static volatile int workload_exec_errno;
341
342/*
343 * perf_evlist__prepare_workload will send a SIGUSR1
344 * if the fork fails, since we asked by setting its
345 * want_signal to true.
346 */
347static void workload_exec_failed_signal(int signo, siginfo_t *info,
348 void *ucontext __maybe_unused)
349{
350 workload_exec_errno = info->si_value.sival_int;
351 done = 1;
352 signr = signo;
353 child_finished = 1;
354}
355
356static int __cmd_record(struct record *rec, int argc, const char **argv)
357{
358 int err;
359 unsigned long waking = 0;
360 const bool forks = argc > 0;
361 struct machine *machine;
362 struct perf_tool *tool = &rec->tool;
363 struct record_opts *opts = &rec->opts;
364 struct perf_data_file *file = &rec->file;
365 struct perf_session *session;
366 bool disabled = false;
367
368 rec->progname = argv[0];
369
370 on_exit(record__sig_exit, rec);
371 signal(SIGCHLD, sig_handler);
372 signal(SIGINT, sig_handler);
373 signal(SIGTERM, sig_handler);
374
375 session = perf_session__new(file, false, NULL);
376 if (session == NULL) {
377 pr_err("Perf session creation failed.\n");
378 return -1;
379 }
380
381 rec->session = session;
382
383 record__init_features(rec);
384
385 if (forks) {
386 err = perf_evlist__prepare_workload(rec->evlist, &opts->target,
387 argv, file->is_pipe,
388 workload_exec_failed_signal);
389 if (err < 0) {
390 pr_err("Couldn't run the workload!\n");
391 goto out_delete_session;
392 }
393 }
394
395 if (record__open(rec) != 0) {
396 err = -1;
397 goto out_delete_session;
398 }
399
400 if (!rec->evlist->nr_groups)
401 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
402
403 /*
404 * perf_session__delete(session) will be called at record__exit()
405 */
406 on_exit(record__exit, rec);
407
408 if (file->is_pipe) {
409 err = perf_header__write_pipe(file->fd);
410 if (err < 0)
411 goto out_delete_session;
412 } else {
413 err = perf_session__write_header(session, rec->evlist,
414 file->fd, false);
415 if (err < 0)
416 goto out_delete_session;
417 }
418
419 if (!rec->no_buildid
420 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
421 pr_err("Couldn't generate buildids. "
422 "Use --no-buildid to profile anyway.\n");
423 err = -1;
424 goto out_delete_session;
425 }
426
427 machine = &session->machines.host;
428
429 if (file->is_pipe) {
430 err = perf_event__synthesize_attrs(tool, session,
431 process_synthesized_event);
432 if (err < 0) {
433 pr_err("Couldn't synthesize attrs.\n");
434 goto out_delete_session;
435 }
436
437 if (have_tracepoints(&rec->evlist->entries)) {
438 /*
439 * FIXME err <= 0 here actually means that
440 * there were no tracepoints so its not really
441 * an error, just that we don't need to
442 * synthesize anything. We really have to
443 * return this more properly and also
444 * propagate errors that now are calling die()
445 */
446 err = perf_event__synthesize_tracing_data(tool, file->fd, rec->evlist,
447 process_synthesized_event);
448 if (err <= 0) {
449 pr_err("Couldn't record tracing data.\n");
450 goto out_delete_session;
451 }
452 rec->bytes_written += err;
453 }
454 }
455
456 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
457 machine);
458 if (err < 0)
459 pr_err("Couldn't record kernel reference relocation symbol\n"
460 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
461 "Check /proc/kallsyms permission or run as root.\n");
462
463 err = perf_event__synthesize_modules(tool, process_synthesized_event,
464 machine);
465 if (err < 0)
466 pr_err("Couldn't record kernel module information.\n"
467 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
468 "Check /proc/modules permission or run as root.\n");
469
470 if (perf_guest) {
471 machines__process_guests(&session->machines,
472 perf_event__synthesize_guest_os, tool);
473 }
474
475 err = __machine__synthesize_threads(machine, tool, &opts->target, rec->evlist->threads,
476 process_synthesized_event, opts->sample_address);
477 if (err != 0)
478 goto out_delete_session;
479
480 if (rec->realtime_prio) {
481 struct sched_param param;
482
483 param.sched_priority = rec->realtime_prio;
484 if (sched_setscheduler(0, SCHED_FIFO, ¶m)) {
485 pr_err("Could not set realtime priority.\n");
486 err = -1;
487 goto out_delete_session;
488 }
489 }
490
491 /*
492 * When perf is starting the traced process, all the events
493 * (apart from group members) have enable_on_exec=1 set,
494 * so don't spoil it by prematurely enabling them.
495 */
496 if (!target__none(&opts->target) && !opts->initial_delay)
497 perf_evlist__enable(rec->evlist);
498
499 /*
500 * Let the child rip
501 */
502 if (forks)
503 perf_evlist__start_workload(rec->evlist);
504
505 if (opts->initial_delay) {
506 usleep(opts->initial_delay * 1000);
507 perf_evlist__enable(rec->evlist);
508 }
509
510 for (;;) {
511 int hits = rec->samples;
512
513 if (record__mmap_read_all(rec) < 0) {
514 err = -1;
515 goto out_delete_session;
516 }
517
518 if (hits == rec->samples) {
519 if (done)
520 break;
521 err = poll(rec->evlist->pollfd, rec->evlist->nr_fds, -1);
522 waking++;
523 }
524
525 /*
526 * When perf is starting the traced process, at the end events
527 * die with the process and we wait for that. Thus no need to
528 * disable events in this case.
529 */
530 if (done && !disabled && !target__none(&opts->target)) {
531 perf_evlist__disable(rec->evlist);
532 disabled = true;
533 }
534 }
535
536 if (forks && workload_exec_errno) {
537 char msg[512];
538 const char *emsg = strerror_r(workload_exec_errno, msg, sizeof(msg));
539 pr_err("Workload failed: %s\n", emsg);
540 err = -1;
541 goto out_delete_session;
542 }
543
544 if (quiet || signr == SIGUSR1)
545 return 0;
546
547 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n", waking);
548
549 /*
550 * Approximate RIP event size: 24 bytes.
551 */
552 fprintf(stderr,
553 "[ perf record: Captured and wrote %.3f MB %s (~%" PRIu64 " samples) ]\n",
554 (double)rec->bytes_written / 1024.0 / 1024.0,
555 file->path,
556 rec->bytes_written / 24);
557
558 return 0;
559
560out_delete_session:
561 perf_session__delete(session);
562 return err;
563}
564
565#define BRANCH_OPT(n, m) \
566 { .name = n, .mode = (m) }
567
568#define BRANCH_END { .name = NULL }
569
570struct branch_mode {
571 const char *name;
572 int mode;
573};
574
575static const struct branch_mode branch_modes[] = {
576 BRANCH_OPT("u", PERF_SAMPLE_BRANCH_USER),
577 BRANCH_OPT("k", PERF_SAMPLE_BRANCH_KERNEL),
578 BRANCH_OPT("hv", PERF_SAMPLE_BRANCH_HV),
579 BRANCH_OPT("any", PERF_SAMPLE_BRANCH_ANY),
580 BRANCH_OPT("any_call", PERF_SAMPLE_BRANCH_ANY_CALL),
581 BRANCH_OPT("any_ret", PERF_SAMPLE_BRANCH_ANY_RETURN),
582 BRANCH_OPT("ind_call", PERF_SAMPLE_BRANCH_IND_CALL),
583 BRANCH_OPT("abort_tx", PERF_SAMPLE_BRANCH_ABORT_TX),
584 BRANCH_OPT("in_tx", PERF_SAMPLE_BRANCH_IN_TX),
585 BRANCH_OPT("no_tx", PERF_SAMPLE_BRANCH_NO_TX),
586 BRANCH_END
587};
588
589static int
590parse_branch_stack(const struct option *opt, const char *str, int unset)
591{
592#define ONLY_PLM \
593 (PERF_SAMPLE_BRANCH_USER |\
594 PERF_SAMPLE_BRANCH_KERNEL |\
595 PERF_SAMPLE_BRANCH_HV)
596
597 uint64_t *mode = (uint64_t *)opt->value;
598 const struct branch_mode *br;
599 char *s, *os = NULL, *p;
600 int ret = -1;
601
602 if (unset)
603 return 0;
604
605 /*
606 * cannot set it twice, -b + --branch-filter for instance
607 */
608 if (*mode)
609 return -1;
610
611 /* str may be NULL in case no arg is passed to -b */
612 if (str) {
613 /* because str is read-only */
614 s = os = strdup(str);
615 if (!s)
616 return -1;
617
618 for (;;) {
619 p = strchr(s, ',');
620 if (p)
621 *p = '\0';
622
623 for (br = branch_modes; br->name; br++) {
624 if (!strcasecmp(s, br->name))
625 break;
626 }
627 if (!br->name) {
628 ui__warning("unknown branch filter %s,"
629 " check man page\n", s);
630 goto error;
631 }
632
633 *mode |= br->mode;
634
635 if (!p)
636 break;
637
638 s = p + 1;
639 }
640 }
641 ret = 0;
642
643 /* default to any branch */
644 if ((*mode & ~ONLY_PLM) == 0) {
645 *mode = PERF_SAMPLE_BRANCH_ANY;
646 }
647error:
648 free(os);
649 return ret;
650}
651
652#ifdef HAVE_DWARF_UNWIND_SUPPORT
653static int get_stack_size(char *str, unsigned long *_size)
654{
655 char *endptr;
656 unsigned long size;
657 unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
658
659 size = strtoul(str, &endptr, 0);
660
661 do {
662 if (*endptr)
663 break;
664
665 size = round_up(size, sizeof(u64));
666 if (!size || size > max_size)
667 break;
668
669 *_size = size;
670 return 0;
671
672 } while (0);
673
674 pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
675 max_size, str);
676 return -1;
677}
678#endif /* HAVE_DWARF_UNWIND_SUPPORT */
679
680int record_parse_callchain(const char *arg, struct record_opts *opts)
681{
682 char *tok, *name, *saveptr = NULL;
683 char *buf;
684 int ret = -1;
685
686 /* We need buffer that we know we can write to. */
687 buf = malloc(strlen(arg) + 1);
688 if (!buf)
689 return -ENOMEM;
690
691 strcpy(buf, arg);
692
693 tok = strtok_r((char *)buf, ",", &saveptr);
694 name = tok ? : (char *)buf;
695
696 do {
697 /* Framepointer style */
698 if (!strncmp(name, "fp", sizeof("fp"))) {
699 if (!strtok_r(NULL, ",", &saveptr)) {
700 opts->call_graph = CALLCHAIN_FP;
701 ret = 0;
702 } else
703 pr_err("callchain: No more arguments "
704 "needed for -g fp\n");
705 break;
706
707#ifdef HAVE_DWARF_UNWIND_SUPPORT
708 /* Dwarf style */
709 } else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
710 const unsigned long default_stack_dump_size = 8192;
711
712 ret = 0;
713 opts->call_graph = CALLCHAIN_DWARF;
714 opts->stack_dump_size = default_stack_dump_size;
715
716 tok = strtok_r(NULL, ",", &saveptr);
717 if (tok) {
718 unsigned long size = 0;
719
720 ret = get_stack_size(tok, &size);
721 opts->stack_dump_size = size;
722 }
723#endif /* HAVE_DWARF_UNWIND_SUPPORT */
724 } else {
725 pr_err("callchain: Unknown --call-graph option "
726 "value: %s\n", arg);
727 break;
728 }
729
730 } while (0);
731
732 free(buf);
733 return ret;
734}
735
736static void callchain_debug(struct record_opts *opts)
737{
738 static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF" };
739
740 pr_debug("callchain: type %s\n", str[opts->call_graph]);
741
742 if (opts->call_graph == CALLCHAIN_DWARF)
743 pr_debug("callchain: stack dump size %d\n",
744 opts->stack_dump_size);
745}
746
747int record_parse_callchain_opt(const struct option *opt,
748 const char *arg,
749 int unset)
750{
751 struct record_opts *opts = opt->value;
752 int ret;
753
754 opts->call_graph_enabled = !unset;
755
756 /* --no-call-graph */
757 if (unset) {
758 opts->call_graph = CALLCHAIN_NONE;
759 pr_debug("callchain: disabled\n");
760 return 0;
761 }
762
763 ret = record_parse_callchain(arg, opts);
764 if (!ret)
765 callchain_debug(opts);
766
767 return ret;
768}
769
770int record_callchain_opt(const struct option *opt,
771 const char *arg __maybe_unused,
772 int unset __maybe_unused)
773{
774 struct record_opts *opts = opt->value;
775
776 opts->call_graph_enabled = !unset;
777
778 if (opts->call_graph == CALLCHAIN_NONE)
779 opts->call_graph = CALLCHAIN_FP;
780
781 callchain_debug(opts);
782 return 0;
783}
784
785static int perf_record_config(const char *var, const char *value, void *cb)
786{
787 struct record *rec = cb;
788
789 if (!strcmp(var, "record.call-graph"))
790 return record_parse_callchain(value, &rec->opts);
791
792 return perf_default_config(var, value, cb);
793}
794
795static const char * const record_usage[] = {
796 "perf record [<options>] [<command>]",
797 "perf record [<options>] -- <command> [<options>]",
798 NULL
799};
800
801/*
802 * XXX Ideally would be local to cmd_record() and passed to a record__new
803 * because we need to have access to it in record__exit, that is called
804 * after cmd_record() exits, but since record_options need to be accessible to
805 * builtin-script, leave it here.
806 *
807 * At least we don't ouch it in all the other functions here directly.
808 *
809 * Just say no to tons of global variables, sigh.
810 */
811static struct record record = {
812 .opts = {
813 .mmap_pages = UINT_MAX,
814 .user_freq = UINT_MAX,
815 .user_interval = ULLONG_MAX,
816 .freq = 4000,
817 .target = {
818 .uses_mmap = true,
819 .default_per_cpu = true,
820 },
821 },
822};
823
824#define CALLCHAIN_HELP "setup and enables call-graph (stack chain/backtrace) recording: "
825
826#ifdef HAVE_DWARF_UNWIND_SUPPORT
827const char record_callchain_help[] = CALLCHAIN_HELP "fp dwarf";
828#else
829const char record_callchain_help[] = CALLCHAIN_HELP "fp";
830#endif
831
832/*
833 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
834 * with it and switch to use the library functions in perf_evlist that came
835 * from builtin-record.c, i.e. use record_opts,
836 * perf_evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
837 * using pipes, etc.
838 */
839const struct option record_options[] = {
840 OPT_CALLBACK('e', "event", &record.evlist, "event",
841 "event selector. use 'perf list' to list available events",
842 parse_events_option),
843 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
844 "event filter", parse_filter),
845 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
846 "record events on existing process id"),
847 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
848 "record events on existing thread id"),
849 OPT_INTEGER('r', "realtime", &record.realtime_prio,
850 "collect data with this RT SCHED_FIFO priority"),
851 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
852 "collect data without buffering"),
853 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
854 "collect raw sample records from all opened counters"),
855 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
856 "system-wide collection from all CPUs"),
857 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
858 "list of cpus to monitor"),
859 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
860 OPT_STRING('o', "output", &record.file.path, "file",
861 "output file name"),
862 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
863 &record.opts.no_inherit_set,
864 "child tasks do not inherit counters"),
865 OPT_UINTEGER('F', "freq", &record.opts.user_freq, "profile at this frequency"),
866 OPT_CALLBACK('m', "mmap-pages", &record.opts.mmap_pages, "pages",
867 "number of mmap data pages",
868 perf_evlist__parse_mmap_pages),
869 OPT_BOOLEAN(0, "group", &record.opts.group,
870 "put the counters into a counter group"),
871 OPT_CALLBACK_NOOPT('g', NULL, &record.opts,
872 NULL, "enables call-graph recording" ,
873 &record_callchain_opt),
874 OPT_CALLBACK(0, "call-graph", &record.opts,
875 "mode[,dump_size]", record_callchain_help,
876 &record_parse_callchain_opt),
877 OPT_INCR('v', "verbose", &verbose,
878 "be more verbose (show counter open errors, etc)"),
879 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any message"),
880 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
881 "per thread counts"),
882 OPT_BOOLEAN('d', "data", &record.opts.sample_address,
883 "Sample addresses"),
884 OPT_BOOLEAN('T', "timestamp", &record.opts.sample_time, "Sample timestamps"),
885 OPT_BOOLEAN('P', "period", &record.opts.period, "Sample period"),
886 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
887 "don't sample"),
888 OPT_BOOLEAN('N', "no-buildid-cache", &record.no_buildid_cache,
889 "do not update the buildid cache"),
890 OPT_BOOLEAN('B', "no-buildid", &record.no_buildid,
891 "do not collect buildids in perf.data"),
892 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
893 "monitor event in cgroup name only",
894 parse_cgroups),
895 OPT_UINTEGER('D', "delay", &record.opts.initial_delay,
896 "ms to wait before starting measurement after program start"),
897 OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
898 "user to profile"),
899
900 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
901 "branch any", "sample any taken branches",
902 parse_branch_stack),
903
904 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
905 "branch filter mask", "branch stack filter modes",
906 parse_branch_stack),
907 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
908 "sample by weight (on special events only)"),
909 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
910 "sample transaction flags (special events only)"),
911 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
912 "use per-thread mmaps"),
913 OPT_END()
914};
915
916int cmd_record(int argc, const char **argv, const char *prefix __maybe_unused)
917{
918 int err = -ENOMEM;
919 struct record *rec = &record;
920 char errbuf[BUFSIZ];
921
922 rec->evlist = perf_evlist__new();
923 if (rec->evlist == NULL)
924 return -ENOMEM;
925
926 perf_config(perf_record_config, rec);
927
928 argc = parse_options(argc, argv, record_options, record_usage,
929 PARSE_OPT_STOP_AT_NON_OPTION);
930 if (!argc && target__none(&rec->opts.target))
931 usage_with_options(record_usage, record_options);
932
933 if (nr_cgroups && !rec->opts.target.system_wide) {
934 ui__error("cgroup monitoring only available in"
935 " system-wide mode\n");
936 usage_with_options(record_usage, record_options);
937 }
938
939 symbol__init();
940
941 if (symbol_conf.kptr_restrict)
942 pr_warning(
943"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
944"check /proc/sys/kernel/kptr_restrict.\n\n"
945"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
946"file is not found in the buildid cache or in the vmlinux path.\n\n"
947"Samples in kernel modules won't be resolved at all.\n\n"
948"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
949"even with a suitable vmlinux or kallsyms file.\n\n");
950
951 if (rec->no_buildid_cache || rec->no_buildid)
952 disable_buildid_cache();
953
954 if (rec->evlist->nr_entries == 0 &&
955 perf_evlist__add_default(rec->evlist) < 0) {
956 pr_err("Not enough memory for event selector list\n");
957 goto out_symbol_exit;
958 }
959
960 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
961 rec->opts.no_inherit = true;
962
963 err = target__validate(&rec->opts.target);
964 if (err) {
965 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
966 ui__warning("%s", errbuf);
967 }
968
969 err = target__parse_uid(&rec->opts.target);
970 if (err) {
971 int saved_errno = errno;
972
973 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
974 ui__error("%s", errbuf);
975
976 err = -saved_errno;
977 goto out_symbol_exit;
978 }
979
980 err = -ENOMEM;
981 if (perf_evlist__create_maps(rec->evlist, &rec->opts.target) < 0)
982 usage_with_options(record_usage, record_options);
983
984 if (record_opts__config(&rec->opts)) {
985 err = -EINVAL;
986 goto out_symbol_exit;
987 }
988
989 err = __cmd_record(&record, argc, argv);
990out_symbol_exit:
991 symbol__exit();
992 return err;
993}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * builtin-record.c
4 *
5 * Builtin record command: Record the profile of a workload
6 * (or a CPU, or a PID) into the perf.data output file - for
7 * later analysis via perf report.
8 */
9#include "builtin.h"
10
11#include "util/build-id.h"
12#include <subcmd/parse-options.h>
13#include <internal/xyarray.h>
14#include "util/parse-events.h"
15#include "util/config.h"
16
17#include "util/callchain.h"
18#include "util/cgroup.h"
19#include "util/header.h"
20#include "util/event.h"
21#include "util/evlist.h"
22#include "util/evsel.h"
23#include "util/debug.h"
24#include "util/mmap.h"
25#include "util/mutex.h"
26#include "util/target.h"
27#include "util/session.h"
28#include "util/tool.h"
29#include "util/symbol.h"
30#include "util/record.h"
31#include "util/cpumap.h"
32#include "util/thread_map.h"
33#include "util/data.h"
34#include "util/perf_regs.h"
35#include "util/auxtrace.h"
36#include "util/tsc.h"
37#include "util/parse-branch-options.h"
38#include "util/parse-regs-options.h"
39#include "util/perf_api_probe.h"
40#include "util/llvm-utils.h"
41#include "util/bpf-loader.h"
42#include "util/trigger.h"
43#include "util/perf-hooks.h"
44#include "util/cpu-set-sched.h"
45#include "util/synthetic-events.h"
46#include "util/time-utils.h"
47#include "util/units.h"
48#include "util/bpf-event.h"
49#include "util/util.h"
50#include "util/pfm.h"
51#include "util/clockid.h"
52#include "util/pmu-hybrid.h"
53#include "util/evlist-hybrid.h"
54#include "util/off_cpu.h"
55#include "asm/bug.h"
56#include "perf.h"
57#include "cputopo.h"
58
59#include <errno.h>
60#include <inttypes.h>
61#include <locale.h>
62#include <poll.h>
63#include <pthread.h>
64#include <unistd.h>
65#ifndef HAVE_GETTID
66#include <syscall.h>
67#endif
68#include <sched.h>
69#include <signal.h>
70#ifdef HAVE_EVENTFD_SUPPORT
71#include <sys/eventfd.h>
72#endif
73#include <sys/mman.h>
74#include <sys/wait.h>
75#include <sys/types.h>
76#include <sys/stat.h>
77#include <fcntl.h>
78#include <linux/err.h>
79#include <linux/string.h>
80#include <linux/time64.h>
81#include <linux/zalloc.h>
82#include <linux/bitmap.h>
83#include <sys/time.h>
84
85struct switch_output {
86 bool enabled;
87 bool signal;
88 unsigned long size;
89 unsigned long time;
90 const char *str;
91 bool set;
92 char **filenames;
93 int num_files;
94 int cur_file;
95};
96
97struct thread_mask {
98 struct mmap_cpu_mask maps;
99 struct mmap_cpu_mask affinity;
100};
101
102struct record_thread {
103 pid_t tid;
104 struct thread_mask *mask;
105 struct {
106 int msg[2];
107 int ack[2];
108 } pipes;
109 struct fdarray pollfd;
110 int ctlfd_pos;
111 int nr_mmaps;
112 struct mmap **maps;
113 struct mmap **overwrite_maps;
114 struct record *rec;
115 unsigned long long samples;
116 unsigned long waking;
117 u64 bytes_written;
118 u64 bytes_transferred;
119 u64 bytes_compressed;
120};
121
122static __thread struct record_thread *thread;
123
124enum thread_msg {
125 THREAD_MSG__UNDEFINED = 0,
126 THREAD_MSG__READY,
127 THREAD_MSG__MAX,
128};
129
130static const char *thread_msg_tags[THREAD_MSG__MAX] = {
131 "UNDEFINED", "READY"
132};
133
134enum thread_spec {
135 THREAD_SPEC__UNDEFINED = 0,
136 THREAD_SPEC__CPU,
137 THREAD_SPEC__CORE,
138 THREAD_SPEC__PACKAGE,
139 THREAD_SPEC__NUMA,
140 THREAD_SPEC__USER,
141 THREAD_SPEC__MAX,
142};
143
144static const char *thread_spec_tags[THREAD_SPEC__MAX] = {
145 "undefined", "cpu", "core", "package", "numa", "user"
146};
147
148struct pollfd_index_map {
149 int evlist_pollfd_index;
150 int thread_pollfd_index;
151};
152
153struct record {
154 struct perf_tool tool;
155 struct record_opts opts;
156 u64 bytes_written;
157 struct perf_data data;
158 struct auxtrace_record *itr;
159 struct evlist *evlist;
160 struct perf_session *session;
161 struct evlist *sb_evlist;
162 pthread_t thread_id;
163 int realtime_prio;
164 bool switch_output_event_set;
165 bool no_buildid;
166 bool no_buildid_set;
167 bool no_buildid_cache;
168 bool no_buildid_cache_set;
169 bool buildid_all;
170 bool buildid_mmap;
171 bool timestamp_filename;
172 bool timestamp_boundary;
173 bool off_cpu;
174 struct switch_output switch_output;
175 unsigned long long samples;
176 unsigned long output_max_size; /* = 0: unlimited */
177 struct perf_debuginfod debuginfod;
178 int nr_threads;
179 struct thread_mask *thread_masks;
180 struct record_thread *thread_data;
181 struct pollfd_index_map *index_map;
182 size_t index_map_sz;
183 size_t index_map_cnt;
184};
185
186static volatile int done;
187
188static volatile int auxtrace_record__snapshot_started;
189static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
190static DEFINE_TRIGGER(switch_output_trigger);
191
192static const char *affinity_tags[PERF_AFFINITY_MAX] = {
193 "SYS", "NODE", "CPU"
194};
195
196#ifndef HAVE_GETTID
197static inline pid_t gettid(void)
198{
199 return (pid_t)syscall(__NR_gettid);
200}
201#endif
202
203static int record__threads_enabled(struct record *rec)
204{
205 return rec->opts.threads_spec;
206}
207
208static bool switch_output_signal(struct record *rec)
209{
210 return rec->switch_output.signal &&
211 trigger_is_ready(&switch_output_trigger);
212}
213
214static bool switch_output_size(struct record *rec)
215{
216 return rec->switch_output.size &&
217 trigger_is_ready(&switch_output_trigger) &&
218 (rec->bytes_written >= rec->switch_output.size);
219}
220
221static bool switch_output_time(struct record *rec)
222{
223 return rec->switch_output.time &&
224 trigger_is_ready(&switch_output_trigger);
225}
226
227static u64 record__bytes_written(struct record *rec)
228{
229 int t;
230 u64 bytes_written = rec->bytes_written;
231 struct record_thread *thread_data = rec->thread_data;
232
233 for (t = 0; t < rec->nr_threads; t++)
234 bytes_written += thread_data[t].bytes_written;
235
236 return bytes_written;
237}
238
239static bool record__output_max_size_exceeded(struct record *rec)
240{
241 return rec->output_max_size &&
242 (record__bytes_written(rec) >= rec->output_max_size);
243}
244
245static int record__write(struct record *rec, struct mmap *map __maybe_unused,
246 void *bf, size_t size)
247{
248 struct perf_data_file *file = &rec->session->data->file;
249
250 if (map && map->file)
251 file = map->file;
252
253 if (perf_data_file__write(file, bf, size) < 0) {
254 pr_err("failed to write perf data, error: %m\n");
255 return -1;
256 }
257
258 if (map && map->file)
259 thread->bytes_written += size;
260 else
261 rec->bytes_written += size;
262
263 if (record__output_max_size_exceeded(rec) && !done) {
264 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
265 " stopping session ]\n",
266 record__bytes_written(rec) >> 10);
267 done = 1;
268 }
269
270 if (switch_output_size(rec))
271 trigger_hit(&switch_output_trigger);
272
273 return 0;
274}
275
276static int record__aio_enabled(struct record *rec);
277static int record__comp_enabled(struct record *rec);
278static size_t zstd_compress(struct perf_session *session, struct mmap *map,
279 void *dst, size_t dst_size, void *src, size_t src_size);
280
281#ifdef HAVE_AIO_SUPPORT
282static int record__aio_write(struct aiocb *cblock, int trace_fd,
283 void *buf, size_t size, off_t off)
284{
285 int rc;
286
287 cblock->aio_fildes = trace_fd;
288 cblock->aio_buf = buf;
289 cblock->aio_nbytes = size;
290 cblock->aio_offset = off;
291 cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
292
293 do {
294 rc = aio_write(cblock);
295 if (rc == 0) {
296 break;
297 } else if (errno != EAGAIN) {
298 cblock->aio_fildes = -1;
299 pr_err("failed to queue perf data, error: %m\n");
300 break;
301 }
302 } while (1);
303
304 return rc;
305}
306
307static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
308{
309 void *rem_buf;
310 off_t rem_off;
311 size_t rem_size;
312 int rc, aio_errno;
313 ssize_t aio_ret, written;
314
315 aio_errno = aio_error(cblock);
316 if (aio_errno == EINPROGRESS)
317 return 0;
318
319 written = aio_ret = aio_return(cblock);
320 if (aio_ret < 0) {
321 if (aio_errno != EINTR)
322 pr_err("failed to write perf data, error: %m\n");
323 written = 0;
324 }
325
326 rem_size = cblock->aio_nbytes - written;
327
328 if (rem_size == 0) {
329 cblock->aio_fildes = -1;
330 /*
331 * md->refcount is incremented in record__aio_pushfn() for
332 * every aio write request started in record__aio_push() so
333 * decrement it because the request is now complete.
334 */
335 perf_mmap__put(&md->core);
336 rc = 1;
337 } else {
338 /*
339 * aio write request may require restart with the
340 * reminder if the kernel didn't write whole
341 * chunk at once.
342 */
343 rem_off = cblock->aio_offset + written;
344 rem_buf = (void *)(cblock->aio_buf + written);
345 record__aio_write(cblock, cblock->aio_fildes,
346 rem_buf, rem_size, rem_off);
347 rc = 0;
348 }
349
350 return rc;
351}
352
353static int record__aio_sync(struct mmap *md, bool sync_all)
354{
355 struct aiocb **aiocb = md->aio.aiocb;
356 struct aiocb *cblocks = md->aio.cblocks;
357 struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
358 int i, do_suspend;
359
360 do {
361 do_suspend = 0;
362 for (i = 0; i < md->aio.nr_cblocks; ++i) {
363 if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
364 if (sync_all)
365 aiocb[i] = NULL;
366 else
367 return i;
368 } else {
369 /*
370 * Started aio write is not complete yet
371 * so it has to be waited before the
372 * next allocation.
373 */
374 aiocb[i] = &cblocks[i];
375 do_suspend = 1;
376 }
377 }
378 if (!do_suspend)
379 return -1;
380
381 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
382 if (!(errno == EAGAIN || errno == EINTR))
383 pr_err("failed to sync perf data, error: %m\n");
384 }
385 } while (1);
386}
387
388struct record_aio {
389 struct record *rec;
390 void *data;
391 size_t size;
392};
393
394static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
395{
396 struct record_aio *aio = to;
397
398 /*
399 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
400 * to release space in the kernel buffer as fast as possible, calling
401 * perf_mmap__consume() from perf_mmap__push() function.
402 *
403 * That lets the kernel to proceed with storing more profiling data into
404 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
405 *
406 * Coping can be done in two steps in case the chunk of profiling data
407 * crosses the upper bound of the kernel buffer. In this case we first move
408 * part of data from map->start till the upper bound and then the reminder
409 * from the beginning of the kernel buffer till the end of the data chunk.
410 */
411
412 if (record__comp_enabled(aio->rec)) {
413 size = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
414 mmap__mmap_len(map) - aio->size,
415 buf, size);
416 } else {
417 memcpy(aio->data + aio->size, buf, size);
418 }
419
420 if (!aio->size) {
421 /*
422 * Increment map->refcount to guard map->aio.data[] buffer
423 * from premature deallocation because map object can be
424 * released earlier than aio write request started on
425 * map->aio.data[] buffer is complete.
426 *
427 * perf_mmap__put() is done at record__aio_complete()
428 * after started aio request completion or at record__aio_push()
429 * if the request failed to start.
430 */
431 perf_mmap__get(&map->core);
432 }
433
434 aio->size += size;
435
436 return size;
437}
438
439static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
440{
441 int ret, idx;
442 int trace_fd = rec->session->data->file.fd;
443 struct record_aio aio = { .rec = rec, .size = 0 };
444
445 /*
446 * Call record__aio_sync() to wait till map->aio.data[] buffer
447 * becomes available after previous aio write operation.
448 */
449
450 idx = record__aio_sync(map, false);
451 aio.data = map->aio.data[idx];
452 ret = perf_mmap__push(map, &aio, record__aio_pushfn);
453 if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
454 return ret;
455
456 rec->samples++;
457 ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
458 if (!ret) {
459 *off += aio.size;
460 rec->bytes_written += aio.size;
461 if (switch_output_size(rec))
462 trigger_hit(&switch_output_trigger);
463 } else {
464 /*
465 * Decrement map->refcount incremented in record__aio_pushfn()
466 * back if record__aio_write() operation failed to start, otherwise
467 * map->refcount is decremented in record__aio_complete() after
468 * aio write operation finishes successfully.
469 */
470 perf_mmap__put(&map->core);
471 }
472
473 return ret;
474}
475
476static off_t record__aio_get_pos(int trace_fd)
477{
478 return lseek(trace_fd, 0, SEEK_CUR);
479}
480
481static void record__aio_set_pos(int trace_fd, off_t pos)
482{
483 lseek(trace_fd, pos, SEEK_SET);
484}
485
486static void record__aio_mmap_read_sync(struct record *rec)
487{
488 int i;
489 struct evlist *evlist = rec->evlist;
490 struct mmap *maps = evlist->mmap;
491
492 if (!record__aio_enabled(rec))
493 return;
494
495 for (i = 0; i < evlist->core.nr_mmaps; i++) {
496 struct mmap *map = &maps[i];
497
498 if (map->core.base)
499 record__aio_sync(map, true);
500 }
501}
502
503static int nr_cblocks_default = 1;
504static int nr_cblocks_max = 4;
505
506static int record__aio_parse(const struct option *opt,
507 const char *str,
508 int unset)
509{
510 struct record_opts *opts = (struct record_opts *)opt->value;
511
512 if (unset) {
513 opts->nr_cblocks = 0;
514 } else {
515 if (str)
516 opts->nr_cblocks = strtol(str, NULL, 0);
517 if (!opts->nr_cblocks)
518 opts->nr_cblocks = nr_cblocks_default;
519 }
520
521 return 0;
522}
523#else /* HAVE_AIO_SUPPORT */
524static int nr_cblocks_max = 0;
525
526static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
527 off_t *off __maybe_unused)
528{
529 return -1;
530}
531
532static off_t record__aio_get_pos(int trace_fd __maybe_unused)
533{
534 return -1;
535}
536
537static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
538{
539}
540
541static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
542{
543}
544#endif
545
546static int record__aio_enabled(struct record *rec)
547{
548 return rec->opts.nr_cblocks > 0;
549}
550
551#define MMAP_FLUSH_DEFAULT 1
552static int record__mmap_flush_parse(const struct option *opt,
553 const char *str,
554 int unset)
555{
556 int flush_max;
557 struct record_opts *opts = (struct record_opts *)opt->value;
558 static struct parse_tag tags[] = {
559 { .tag = 'B', .mult = 1 },
560 { .tag = 'K', .mult = 1 << 10 },
561 { .tag = 'M', .mult = 1 << 20 },
562 { .tag = 'G', .mult = 1 << 30 },
563 { .tag = 0 },
564 };
565
566 if (unset)
567 return 0;
568
569 if (str) {
570 opts->mmap_flush = parse_tag_value(str, tags);
571 if (opts->mmap_flush == (int)-1)
572 opts->mmap_flush = strtol(str, NULL, 0);
573 }
574
575 if (!opts->mmap_flush)
576 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
577
578 flush_max = evlist__mmap_size(opts->mmap_pages);
579 flush_max /= 4;
580 if (opts->mmap_flush > flush_max)
581 opts->mmap_flush = flush_max;
582
583 return 0;
584}
585
586#ifdef HAVE_ZSTD_SUPPORT
587static unsigned int comp_level_default = 1;
588
589static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
590{
591 struct record_opts *opts = opt->value;
592
593 if (unset) {
594 opts->comp_level = 0;
595 } else {
596 if (str)
597 opts->comp_level = strtol(str, NULL, 0);
598 if (!opts->comp_level)
599 opts->comp_level = comp_level_default;
600 }
601
602 return 0;
603}
604#endif
605static unsigned int comp_level_max = 22;
606
607static int record__comp_enabled(struct record *rec)
608{
609 return rec->opts.comp_level > 0;
610}
611
612static int process_synthesized_event(struct perf_tool *tool,
613 union perf_event *event,
614 struct perf_sample *sample __maybe_unused,
615 struct machine *machine __maybe_unused)
616{
617 struct record *rec = container_of(tool, struct record, tool);
618 return record__write(rec, NULL, event, event->header.size);
619}
620
621static struct mutex synth_lock;
622
623static int process_locked_synthesized_event(struct perf_tool *tool,
624 union perf_event *event,
625 struct perf_sample *sample __maybe_unused,
626 struct machine *machine __maybe_unused)
627{
628 int ret;
629
630 mutex_lock(&synth_lock);
631 ret = process_synthesized_event(tool, event, sample, machine);
632 mutex_unlock(&synth_lock);
633 return ret;
634}
635
636static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
637{
638 struct record *rec = to;
639
640 if (record__comp_enabled(rec)) {
641 size = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size);
642 bf = map->data;
643 }
644
645 thread->samples++;
646 return record__write(rec, map, bf, size);
647}
648
649static volatile sig_atomic_t signr = -1;
650static volatile sig_atomic_t child_finished;
651#ifdef HAVE_EVENTFD_SUPPORT
652static volatile sig_atomic_t done_fd = -1;
653#endif
654
655static void sig_handler(int sig)
656{
657 if (sig == SIGCHLD)
658 child_finished = 1;
659 else
660 signr = sig;
661
662 done = 1;
663#ifdef HAVE_EVENTFD_SUPPORT
664 if (done_fd >= 0) {
665 u64 tmp = 1;
666 int orig_errno = errno;
667
668 /*
669 * It is possible for this signal handler to run after done is
670 * checked in the main loop, but before the perf counter fds are
671 * polled. If this happens, the poll() will continue to wait
672 * even though done is set, and will only break out if either
673 * another signal is received, or the counters are ready for
674 * read. To ensure the poll() doesn't sleep when done is set,
675 * use an eventfd (done_fd) to wake up the poll().
676 */
677 if (write(done_fd, &tmp, sizeof(tmp)) < 0)
678 pr_err("failed to signal wakeup fd, error: %m\n");
679
680 errno = orig_errno;
681 }
682#endif // HAVE_EVENTFD_SUPPORT
683}
684
685static void sigsegv_handler(int sig)
686{
687 perf_hooks__recover();
688 sighandler_dump_stack(sig);
689}
690
691static void record__sig_exit(void)
692{
693 if (signr == -1)
694 return;
695
696 signal(signr, SIG_DFL);
697 raise(signr);
698}
699
700#ifdef HAVE_AUXTRACE_SUPPORT
701
702static int record__process_auxtrace(struct perf_tool *tool,
703 struct mmap *map,
704 union perf_event *event, void *data1,
705 size_t len1, void *data2, size_t len2)
706{
707 struct record *rec = container_of(tool, struct record, tool);
708 struct perf_data *data = &rec->data;
709 size_t padding;
710 u8 pad[8] = {0};
711
712 if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
713 off_t file_offset;
714 int fd = perf_data__fd(data);
715 int err;
716
717 file_offset = lseek(fd, 0, SEEK_CUR);
718 if (file_offset == -1)
719 return -1;
720 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
721 event, file_offset);
722 if (err)
723 return err;
724 }
725
726 /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
727 padding = (len1 + len2) & 7;
728 if (padding)
729 padding = 8 - padding;
730
731 record__write(rec, map, event, event->header.size);
732 record__write(rec, map, data1, len1);
733 if (len2)
734 record__write(rec, map, data2, len2);
735 record__write(rec, map, &pad, padding);
736
737 return 0;
738}
739
740static int record__auxtrace_mmap_read(struct record *rec,
741 struct mmap *map)
742{
743 int ret;
744
745 ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
746 record__process_auxtrace);
747 if (ret < 0)
748 return ret;
749
750 if (ret)
751 rec->samples++;
752
753 return 0;
754}
755
756static int record__auxtrace_mmap_read_snapshot(struct record *rec,
757 struct mmap *map)
758{
759 int ret;
760
761 ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
762 record__process_auxtrace,
763 rec->opts.auxtrace_snapshot_size);
764 if (ret < 0)
765 return ret;
766
767 if (ret)
768 rec->samples++;
769
770 return 0;
771}
772
773static int record__auxtrace_read_snapshot_all(struct record *rec)
774{
775 int i;
776 int rc = 0;
777
778 for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
779 struct mmap *map = &rec->evlist->mmap[i];
780
781 if (!map->auxtrace_mmap.base)
782 continue;
783
784 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
785 rc = -1;
786 goto out;
787 }
788 }
789out:
790 return rc;
791}
792
793static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
794{
795 pr_debug("Recording AUX area tracing snapshot\n");
796 if (record__auxtrace_read_snapshot_all(rec) < 0) {
797 trigger_error(&auxtrace_snapshot_trigger);
798 } else {
799 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
800 trigger_error(&auxtrace_snapshot_trigger);
801 else
802 trigger_ready(&auxtrace_snapshot_trigger);
803 }
804}
805
806static int record__auxtrace_snapshot_exit(struct record *rec)
807{
808 if (trigger_is_error(&auxtrace_snapshot_trigger))
809 return 0;
810
811 if (!auxtrace_record__snapshot_started &&
812 auxtrace_record__snapshot_start(rec->itr))
813 return -1;
814
815 record__read_auxtrace_snapshot(rec, true);
816 if (trigger_is_error(&auxtrace_snapshot_trigger))
817 return -1;
818
819 return 0;
820}
821
822static int record__auxtrace_init(struct record *rec)
823{
824 int err;
825
826 if ((rec->opts.auxtrace_snapshot_opts || rec->opts.auxtrace_sample_opts)
827 && record__threads_enabled(rec)) {
828 pr_err("AUX area tracing options are not available in parallel streaming mode.\n");
829 return -EINVAL;
830 }
831
832 if (!rec->itr) {
833 rec->itr = auxtrace_record__init(rec->evlist, &err);
834 if (err)
835 return err;
836 }
837
838 err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
839 rec->opts.auxtrace_snapshot_opts);
840 if (err)
841 return err;
842
843 err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
844 rec->opts.auxtrace_sample_opts);
845 if (err)
846 return err;
847
848 auxtrace_regroup_aux_output(rec->evlist);
849
850 return auxtrace_parse_filters(rec->evlist);
851}
852
853#else
854
855static inline
856int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
857 struct mmap *map __maybe_unused)
858{
859 return 0;
860}
861
862static inline
863void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
864 bool on_exit __maybe_unused)
865{
866}
867
868static inline
869int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
870{
871 return 0;
872}
873
874static inline
875int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
876{
877 return 0;
878}
879
880static int record__auxtrace_init(struct record *rec __maybe_unused)
881{
882 return 0;
883}
884
885#endif
886
887static int record__config_text_poke(struct evlist *evlist)
888{
889 struct evsel *evsel;
890
891 /* Nothing to do if text poke is already configured */
892 evlist__for_each_entry(evlist, evsel) {
893 if (evsel->core.attr.text_poke)
894 return 0;
895 }
896
897 evsel = evlist__add_dummy_on_all_cpus(evlist);
898 if (!evsel)
899 return -ENOMEM;
900
901 evsel->core.attr.text_poke = 1;
902 evsel->core.attr.ksymbol = 1;
903 evsel->immediate = true;
904 evsel__set_sample_bit(evsel, TIME);
905
906 return 0;
907}
908
909static int record__config_off_cpu(struct record *rec)
910{
911 return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts);
912}
913
914static bool record__kcore_readable(struct machine *machine)
915{
916 char kcore[PATH_MAX];
917 int fd;
918
919 scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
920
921 fd = open(kcore, O_RDONLY);
922 if (fd < 0)
923 return false;
924
925 close(fd);
926
927 return true;
928}
929
930static int record__kcore_copy(struct machine *machine, struct perf_data *data)
931{
932 char from_dir[PATH_MAX];
933 char kcore_dir[PATH_MAX];
934 int ret;
935
936 snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
937
938 ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
939 if (ret)
940 return ret;
941
942 return kcore_copy(from_dir, kcore_dir);
943}
944
945static void record__thread_data_init_pipes(struct record_thread *thread_data)
946{
947 thread_data->pipes.msg[0] = -1;
948 thread_data->pipes.msg[1] = -1;
949 thread_data->pipes.ack[0] = -1;
950 thread_data->pipes.ack[1] = -1;
951}
952
953static int record__thread_data_open_pipes(struct record_thread *thread_data)
954{
955 if (pipe(thread_data->pipes.msg))
956 return -EINVAL;
957
958 if (pipe(thread_data->pipes.ack)) {
959 close(thread_data->pipes.msg[0]);
960 thread_data->pipes.msg[0] = -1;
961 close(thread_data->pipes.msg[1]);
962 thread_data->pipes.msg[1] = -1;
963 return -EINVAL;
964 }
965
966 pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
967 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
968 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
969
970 return 0;
971}
972
973static void record__thread_data_close_pipes(struct record_thread *thread_data)
974{
975 if (thread_data->pipes.msg[0] != -1) {
976 close(thread_data->pipes.msg[0]);
977 thread_data->pipes.msg[0] = -1;
978 }
979 if (thread_data->pipes.msg[1] != -1) {
980 close(thread_data->pipes.msg[1]);
981 thread_data->pipes.msg[1] = -1;
982 }
983 if (thread_data->pipes.ack[0] != -1) {
984 close(thread_data->pipes.ack[0]);
985 thread_data->pipes.ack[0] = -1;
986 }
987 if (thread_data->pipes.ack[1] != -1) {
988 close(thread_data->pipes.ack[1]);
989 thread_data->pipes.ack[1] = -1;
990 }
991}
992
993static bool evlist__per_thread(struct evlist *evlist)
994{
995 return cpu_map__is_dummy(evlist->core.user_requested_cpus);
996}
997
998static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
999{
1000 int m, tm, nr_mmaps = evlist->core.nr_mmaps;
1001 struct mmap *mmap = evlist->mmap;
1002 struct mmap *overwrite_mmap = evlist->overwrite_mmap;
1003 struct perf_cpu_map *cpus = evlist->core.all_cpus;
1004 bool per_thread = evlist__per_thread(evlist);
1005
1006 if (per_thread)
1007 thread_data->nr_mmaps = nr_mmaps;
1008 else
1009 thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
1010 thread_data->mask->maps.nbits);
1011 if (mmap) {
1012 thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1013 if (!thread_data->maps)
1014 return -ENOMEM;
1015 }
1016 if (overwrite_mmap) {
1017 thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1018 if (!thread_data->overwrite_maps) {
1019 zfree(&thread_data->maps);
1020 return -ENOMEM;
1021 }
1022 }
1023 pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
1024 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
1025
1026 for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
1027 if (per_thread ||
1028 test_bit(perf_cpu_map__cpu(cpus, m).cpu, thread_data->mask->maps.bits)) {
1029 if (thread_data->maps) {
1030 thread_data->maps[tm] = &mmap[m];
1031 pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
1032 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1033 }
1034 if (thread_data->overwrite_maps) {
1035 thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
1036 pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
1037 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1038 }
1039 tm++;
1040 }
1041 }
1042
1043 return 0;
1044}
1045
1046static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1047{
1048 int f, tm, pos;
1049 struct mmap *map, *overwrite_map;
1050
1051 fdarray__init(&thread_data->pollfd, 64);
1052
1053 for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1054 map = thread_data->maps ? thread_data->maps[tm] : NULL;
1055 overwrite_map = thread_data->overwrite_maps ?
1056 thread_data->overwrite_maps[tm] : NULL;
1057
1058 for (f = 0; f < evlist->core.pollfd.nr; f++) {
1059 void *ptr = evlist->core.pollfd.priv[f].ptr;
1060
1061 if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1062 pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1063 &evlist->core.pollfd);
1064 if (pos < 0)
1065 return pos;
1066 pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1067 thread_data, pos, evlist->core.pollfd.entries[f].fd);
1068 }
1069 }
1070 }
1071
1072 return 0;
1073}
1074
1075static void record__free_thread_data(struct record *rec)
1076{
1077 int t;
1078 struct record_thread *thread_data = rec->thread_data;
1079
1080 if (thread_data == NULL)
1081 return;
1082
1083 for (t = 0; t < rec->nr_threads; t++) {
1084 record__thread_data_close_pipes(&thread_data[t]);
1085 zfree(&thread_data[t].maps);
1086 zfree(&thread_data[t].overwrite_maps);
1087 fdarray__exit(&thread_data[t].pollfd);
1088 }
1089
1090 zfree(&rec->thread_data);
1091}
1092
1093static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
1094 int evlist_pollfd_index,
1095 int thread_pollfd_index)
1096{
1097 size_t x = rec->index_map_cnt;
1098
1099 if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
1100 return -ENOMEM;
1101 rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
1102 rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
1103 rec->index_map_cnt += 1;
1104 return 0;
1105}
1106
1107static int record__update_evlist_pollfd_from_thread(struct record *rec,
1108 struct evlist *evlist,
1109 struct record_thread *thread_data)
1110{
1111 struct pollfd *e_entries = evlist->core.pollfd.entries;
1112 struct pollfd *t_entries = thread_data->pollfd.entries;
1113 int err = 0;
1114 size_t i;
1115
1116 for (i = 0; i < rec->index_map_cnt; i++) {
1117 int e_pos = rec->index_map[i].evlist_pollfd_index;
1118 int t_pos = rec->index_map[i].thread_pollfd_index;
1119
1120 if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
1121 e_entries[e_pos].events != t_entries[t_pos].events) {
1122 pr_err("Thread and evlist pollfd index mismatch\n");
1123 err = -EINVAL;
1124 continue;
1125 }
1126 e_entries[e_pos].revents = t_entries[t_pos].revents;
1127 }
1128 return err;
1129}
1130
1131static int record__dup_non_perf_events(struct record *rec,
1132 struct evlist *evlist,
1133 struct record_thread *thread_data)
1134{
1135 struct fdarray *fda = &evlist->core.pollfd;
1136 int i, ret;
1137
1138 for (i = 0; i < fda->nr; i++) {
1139 if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
1140 continue;
1141 ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
1142 if (ret < 0) {
1143 pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1144 return ret;
1145 }
1146 pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
1147 thread_data, ret, fda->entries[i].fd);
1148 ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret);
1149 if (ret < 0) {
1150 pr_err("Failed to map thread and evlist pollfd indexes\n");
1151 return ret;
1152 }
1153 }
1154 return 0;
1155}
1156
1157static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1158{
1159 int t, ret;
1160 struct record_thread *thread_data;
1161
1162 rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
1163 if (!rec->thread_data) {
1164 pr_err("Failed to allocate thread data\n");
1165 return -ENOMEM;
1166 }
1167 thread_data = rec->thread_data;
1168
1169 for (t = 0; t < rec->nr_threads; t++)
1170 record__thread_data_init_pipes(&thread_data[t]);
1171
1172 for (t = 0; t < rec->nr_threads; t++) {
1173 thread_data[t].rec = rec;
1174 thread_data[t].mask = &rec->thread_masks[t];
1175 ret = record__thread_data_init_maps(&thread_data[t], evlist);
1176 if (ret) {
1177 pr_err("Failed to initialize thread[%d] maps\n", t);
1178 goto out_free;
1179 }
1180 ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
1181 if (ret) {
1182 pr_err("Failed to initialize thread[%d] pollfd\n", t);
1183 goto out_free;
1184 }
1185 if (t) {
1186 thread_data[t].tid = -1;
1187 ret = record__thread_data_open_pipes(&thread_data[t]);
1188 if (ret) {
1189 pr_err("Failed to open thread[%d] communication pipes\n", t);
1190 goto out_free;
1191 }
1192 ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1193 POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1194 if (ret < 0) {
1195 pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1196 goto out_free;
1197 }
1198 thread_data[t].ctlfd_pos = ret;
1199 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1200 thread_data, thread_data[t].ctlfd_pos,
1201 thread_data[t].pipes.msg[0]);
1202 } else {
1203 thread_data[t].tid = gettid();
1204
1205 ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]);
1206 if (ret < 0)
1207 goto out_free;
1208
1209 thread_data[t].ctlfd_pos = -1; /* Not used */
1210 }
1211 }
1212
1213 return 0;
1214
1215out_free:
1216 record__free_thread_data(rec);
1217
1218 return ret;
1219}
1220
1221static int record__mmap_evlist(struct record *rec,
1222 struct evlist *evlist)
1223{
1224 int i, ret;
1225 struct record_opts *opts = &rec->opts;
1226 bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1227 opts->auxtrace_sample_mode;
1228 char msg[512];
1229
1230 if (opts->affinity != PERF_AFFINITY_SYS)
1231 cpu__setup_cpunode_map();
1232
1233 if (evlist__mmap_ex(evlist, opts->mmap_pages,
1234 opts->auxtrace_mmap_pages,
1235 auxtrace_overwrite,
1236 opts->nr_cblocks, opts->affinity,
1237 opts->mmap_flush, opts->comp_level) < 0) {
1238 if (errno == EPERM) {
1239 pr_err("Permission error mapping pages.\n"
1240 "Consider increasing "
1241 "/proc/sys/kernel/perf_event_mlock_kb,\n"
1242 "or try again with a smaller value of -m/--mmap_pages.\n"
1243 "(current value: %u,%u)\n",
1244 opts->mmap_pages, opts->auxtrace_mmap_pages);
1245 return -errno;
1246 } else {
1247 pr_err("failed to mmap with %d (%s)\n", errno,
1248 str_error_r(errno, msg, sizeof(msg)));
1249 if (errno)
1250 return -errno;
1251 else
1252 return -EINVAL;
1253 }
1254 }
1255
1256 if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
1257 return -1;
1258
1259 ret = record__alloc_thread_data(rec, evlist);
1260 if (ret)
1261 return ret;
1262
1263 if (record__threads_enabled(rec)) {
1264 ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
1265 if (ret) {
1266 pr_err("Failed to create data directory: %s\n", strerror(-ret));
1267 return ret;
1268 }
1269 for (i = 0; i < evlist->core.nr_mmaps; i++) {
1270 if (evlist->mmap)
1271 evlist->mmap[i].file = &rec->data.dir.files[i];
1272 if (evlist->overwrite_mmap)
1273 evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
1274 }
1275 }
1276
1277 return 0;
1278}
1279
1280static int record__mmap(struct record *rec)
1281{
1282 return record__mmap_evlist(rec, rec->evlist);
1283}
1284
1285static int record__open(struct record *rec)
1286{
1287 char msg[BUFSIZ];
1288 struct evsel *pos;
1289 struct evlist *evlist = rec->evlist;
1290 struct perf_session *session = rec->session;
1291 struct record_opts *opts = &rec->opts;
1292 int rc = 0;
1293
1294 /*
1295 * For initial_delay, system wide or a hybrid system, we need to add a
1296 * dummy event so that we can track PERF_RECORD_MMAP to cover the delay
1297 * of waiting or event synthesis.
1298 */
1299 if (opts->initial_delay || target__has_cpu(&opts->target) ||
1300 perf_pmu__has_hybrid()) {
1301 pos = evlist__get_tracking_event(evlist);
1302 if (!evsel__is_dummy_event(pos)) {
1303 /* Set up dummy event. */
1304 if (evlist__add_dummy(evlist))
1305 return -ENOMEM;
1306 pos = evlist__last(evlist);
1307 evlist__set_tracking_event(evlist, pos);
1308 }
1309
1310 /*
1311 * Enable the dummy event when the process is forked for
1312 * initial_delay, immediately for system wide.
1313 */
1314 if (opts->initial_delay && !pos->immediate &&
1315 !target__has_cpu(&opts->target))
1316 pos->core.attr.enable_on_exec = 1;
1317 else
1318 pos->immediate = 1;
1319 }
1320
1321 evlist__config(evlist, opts, &callchain_param);
1322
1323 evlist__for_each_entry(evlist, pos) {
1324try_again:
1325 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
1326 if (evsel__fallback(pos, errno, msg, sizeof(msg))) {
1327 if (verbose > 0)
1328 ui__warning("%s\n", msg);
1329 goto try_again;
1330 }
1331 if ((errno == EINVAL || errno == EBADF) &&
1332 pos->core.leader != &pos->core &&
1333 pos->weak_group) {
1334 pos = evlist__reset_weak_group(evlist, pos, true);
1335 goto try_again;
1336 }
1337 rc = -errno;
1338 evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
1339 ui__error("%s\n", msg);
1340 goto out;
1341 }
1342
1343 pos->supported = true;
1344 }
1345
1346 if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
1347 pr_warning(
1348"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1349"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1350"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1351"file is not found in the buildid cache or in the vmlinux path.\n\n"
1352"Samples in kernel modules won't be resolved at all.\n\n"
1353"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1354"even with a suitable vmlinux or kallsyms file.\n\n");
1355 }
1356
1357 if (evlist__apply_filters(evlist, &pos)) {
1358 pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
1359 pos->filter, evsel__name(pos), errno,
1360 str_error_r(errno, msg, sizeof(msg)));
1361 rc = -1;
1362 goto out;
1363 }
1364
1365 rc = record__mmap(rec);
1366 if (rc)
1367 goto out;
1368
1369 session->evlist = evlist;
1370 perf_session__set_id_hdr_size(session);
1371out:
1372 return rc;
1373}
1374
1375static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1376{
1377 if (rec->evlist->first_sample_time == 0)
1378 rec->evlist->first_sample_time = sample_time;
1379
1380 if (sample_time)
1381 rec->evlist->last_sample_time = sample_time;
1382}
1383
1384static int process_sample_event(struct perf_tool *tool,
1385 union perf_event *event,
1386 struct perf_sample *sample,
1387 struct evsel *evsel,
1388 struct machine *machine)
1389{
1390 struct record *rec = container_of(tool, struct record, tool);
1391
1392 set_timestamp_boundary(rec, sample->time);
1393
1394 if (rec->buildid_all)
1395 return 0;
1396
1397 rec->samples++;
1398 return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
1399}
1400
1401static int process_buildids(struct record *rec)
1402{
1403 struct perf_session *session = rec->session;
1404
1405 if (perf_data__size(&rec->data) == 0)
1406 return 0;
1407
1408 /*
1409 * During this process, it'll load kernel map and replace the
1410 * dso->long_name to a real pathname it found. In this case
1411 * we prefer the vmlinux path like
1412 * /lib/modules/3.16.4/build/vmlinux
1413 *
1414 * rather than build-id path (in debug directory).
1415 * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1416 */
1417 symbol_conf.ignore_vmlinux_buildid = true;
1418
1419 /*
1420 * If --buildid-all is given, it marks all DSO regardless of hits,
1421 * so no need to process samples. But if timestamp_boundary is enabled,
1422 * it still needs to walk on all samples to get the timestamps of
1423 * first/last samples.
1424 */
1425 if (rec->buildid_all && !rec->timestamp_boundary)
1426 rec->tool.sample = NULL;
1427
1428 return perf_session__process_events(session);
1429}
1430
1431static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
1432{
1433 int err;
1434 struct perf_tool *tool = data;
1435 /*
1436 *As for guest kernel when processing subcommand record&report,
1437 *we arrange module mmap prior to guest kernel mmap and trigger
1438 *a preload dso because default guest module symbols are loaded
1439 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1440 *method is used to avoid symbol missing when the first addr is
1441 *in module instead of in guest kernel.
1442 */
1443 err = perf_event__synthesize_modules(tool, process_synthesized_event,
1444 machine);
1445 if (err < 0)
1446 pr_err("Couldn't record guest kernel [%d]'s reference"
1447 " relocation symbol.\n", machine->pid);
1448
1449 /*
1450 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1451 * have no _text sometimes.
1452 */
1453 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1454 machine);
1455 if (err < 0)
1456 pr_err("Couldn't record guest kernel [%d]'s reference"
1457 " relocation symbol.\n", machine->pid);
1458}
1459
1460static struct perf_event_header finished_round_event = {
1461 .size = sizeof(struct perf_event_header),
1462 .type = PERF_RECORD_FINISHED_ROUND,
1463};
1464
1465static struct perf_event_header finished_init_event = {
1466 .size = sizeof(struct perf_event_header),
1467 .type = PERF_RECORD_FINISHED_INIT,
1468};
1469
1470static void record__adjust_affinity(struct record *rec, struct mmap *map)
1471{
1472 if (rec->opts.affinity != PERF_AFFINITY_SYS &&
1473 !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
1474 thread->mask->affinity.nbits)) {
1475 bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
1476 bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
1477 map->affinity_mask.bits, thread->mask->affinity.nbits);
1478 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1479 (cpu_set_t *)thread->mask->affinity.bits);
1480 if (verbose == 2) {
1481 pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1482 mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
1483 }
1484 }
1485}
1486
1487static size_t process_comp_header(void *record, size_t increment)
1488{
1489 struct perf_record_compressed *event = record;
1490 size_t size = sizeof(*event);
1491
1492 if (increment) {
1493 event->header.size += increment;
1494 return increment;
1495 }
1496
1497 event->header.type = PERF_RECORD_COMPRESSED;
1498 event->header.size = size;
1499
1500 return size;
1501}
1502
1503static size_t zstd_compress(struct perf_session *session, struct mmap *map,
1504 void *dst, size_t dst_size, void *src, size_t src_size)
1505{
1506 size_t compressed;
1507 size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
1508 struct zstd_data *zstd_data = &session->zstd_data;
1509
1510 if (map && map->file)
1511 zstd_data = &map->zstd_data;
1512
1513 compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
1514 max_record_size, process_comp_header);
1515
1516 if (map && map->file) {
1517 thread->bytes_transferred += src_size;
1518 thread->bytes_compressed += compressed;
1519 } else {
1520 session->bytes_transferred += src_size;
1521 session->bytes_compressed += compressed;
1522 }
1523
1524 return compressed;
1525}
1526
1527static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
1528 bool overwrite, bool synch)
1529{
1530 u64 bytes_written = rec->bytes_written;
1531 int i;
1532 int rc = 0;
1533 int nr_mmaps;
1534 struct mmap **maps;
1535 int trace_fd = rec->data.file.fd;
1536 off_t off = 0;
1537
1538 if (!evlist)
1539 return 0;
1540
1541 nr_mmaps = thread->nr_mmaps;
1542 maps = overwrite ? thread->overwrite_maps : thread->maps;
1543
1544 if (!maps)
1545 return 0;
1546
1547 if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
1548 return 0;
1549
1550 if (record__aio_enabled(rec))
1551 off = record__aio_get_pos(trace_fd);
1552
1553 for (i = 0; i < nr_mmaps; i++) {
1554 u64 flush = 0;
1555 struct mmap *map = maps[i];
1556
1557 if (map->core.base) {
1558 record__adjust_affinity(rec, map);
1559 if (synch) {
1560 flush = map->core.flush;
1561 map->core.flush = 1;
1562 }
1563 if (!record__aio_enabled(rec)) {
1564 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
1565 if (synch)
1566 map->core.flush = flush;
1567 rc = -1;
1568 goto out;
1569 }
1570 } else {
1571 if (record__aio_push(rec, map, &off) < 0) {
1572 record__aio_set_pos(trace_fd, off);
1573 if (synch)
1574 map->core.flush = flush;
1575 rc = -1;
1576 goto out;
1577 }
1578 }
1579 if (synch)
1580 map->core.flush = flush;
1581 }
1582
1583 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
1584 !rec->opts.auxtrace_sample_mode &&
1585 record__auxtrace_mmap_read(rec, map) != 0) {
1586 rc = -1;
1587 goto out;
1588 }
1589 }
1590
1591 if (record__aio_enabled(rec))
1592 record__aio_set_pos(trace_fd, off);
1593
1594 /*
1595 * Mark the round finished in case we wrote
1596 * at least one event.
1597 *
1598 * No need for round events in directory mode,
1599 * because per-cpu maps and files have data
1600 * sorted by kernel.
1601 */
1602 if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
1603 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
1604
1605 if (overwrite)
1606 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
1607out:
1608 return rc;
1609}
1610
1611static int record__mmap_read_all(struct record *rec, bool synch)
1612{
1613 int err;
1614
1615 err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
1616 if (err)
1617 return err;
1618
1619 return record__mmap_read_evlist(rec, rec->evlist, true, synch);
1620}
1621
1622static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1623 void *arg __maybe_unused)
1624{
1625 struct perf_mmap *map = fda->priv[fd].ptr;
1626
1627 if (map)
1628 perf_mmap__put(map);
1629}
1630
1631static void *record__thread(void *arg)
1632{
1633 enum thread_msg msg = THREAD_MSG__READY;
1634 bool terminate = false;
1635 struct fdarray *pollfd;
1636 int err, ctlfd_pos;
1637
1638 thread = arg;
1639 thread->tid = gettid();
1640
1641 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1642 if (err == -1)
1643 pr_warning("threads[%d]: failed to notify on start: %s\n",
1644 thread->tid, strerror(errno));
1645
1646 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1647
1648 pollfd = &thread->pollfd;
1649 ctlfd_pos = thread->ctlfd_pos;
1650
1651 for (;;) {
1652 unsigned long long hits = thread->samples;
1653
1654 if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
1655 break;
1656
1657 if (hits == thread->samples) {
1658
1659 err = fdarray__poll(pollfd, -1);
1660 /*
1661 * Propagate error, only if there's any. Ignore positive
1662 * number of returned events and interrupt error.
1663 */
1664 if (err > 0 || (err < 0 && errno == EINTR))
1665 err = 0;
1666 thread->waking++;
1667
1668 if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1669 record__thread_munmap_filtered, NULL) == 0)
1670 break;
1671 }
1672
1673 if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1674 terminate = true;
1675 close(thread->pipes.msg[0]);
1676 thread->pipes.msg[0] = -1;
1677 pollfd->entries[ctlfd_pos].fd = -1;
1678 pollfd->entries[ctlfd_pos].events = 0;
1679 }
1680
1681 pollfd->entries[ctlfd_pos].revents = 0;
1682 }
1683 record__mmap_read_all(thread->rec, true);
1684
1685 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1686 if (err == -1)
1687 pr_warning("threads[%d]: failed to notify on termination: %s\n",
1688 thread->tid, strerror(errno));
1689
1690 return NULL;
1691}
1692
1693static void record__init_features(struct record *rec)
1694{
1695 struct perf_session *session = rec->session;
1696 int feat;
1697
1698 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1699 perf_header__set_feat(&session->header, feat);
1700
1701 if (rec->no_buildid)
1702 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1703
1704#ifdef HAVE_LIBTRACEEVENT
1705 if (!have_tracepoints(&rec->evlist->core.entries))
1706 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1707#endif
1708
1709 if (!rec->opts.branch_stack)
1710 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
1711
1712 if (!rec->opts.full_auxtrace)
1713 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
1714
1715 if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1716 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1717
1718 if (!rec->opts.use_clockid)
1719 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1720
1721 if (!record__threads_enabled(rec))
1722 perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1723
1724 if (!record__comp_enabled(rec))
1725 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
1726
1727 perf_header__clear_feat(&session->header, HEADER_STAT);
1728}
1729
1730static void
1731record__finish_output(struct record *rec)
1732{
1733 int i;
1734 struct perf_data *data = &rec->data;
1735 int fd = perf_data__fd(data);
1736
1737 if (data->is_pipe)
1738 return;
1739
1740 rec->session->header.data_size += rec->bytes_written;
1741 data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
1742 if (record__threads_enabled(rec)) {
1743 for (i = 0; i < data->dir.nr; i++)
1744 data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
1745 }
1746
1747 if (!rec->no_buildid) {
1748 process_buildids(rec);
1749
1750 if (rec->buildid_all)
1751 dsos__hit_all(rec->session);
1752 }
1753 perf_session__write_header(rec->session, rec->evlist, fd, true);
1754
1755 return;
1756}
1757
1758static int record__synthesize_workload(struct record *rec, bool tail)
1759{
1760 int err;
1761 struct perf_thread_map *thread_map;
1762 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
1763
1764 if (rec->opts.tail_synthesize != tail)
1765 return 0;
1766
1767 thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
1768 if (thread_map == NULL)
1769 return -1;
1770
1771 err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
1772 process_synthesized_event,
1773 &rec->session->machines.host,
1774 needs_mmap,
1775 rec->opts.sample_address);
1776 perf_thread_map__put(thread_map);
1777 return err;
1778}
1779
1780static int write_finished_init(struct record *rec, bool tail)
1781{
1782 if (rec->opts.tail_synthesize != tail)
1783 return 0;
1784
1785 return record__write(rec, NULL, &finished_init_event, sizeof(finished_init_event));
1786}
1787
1788static int record__synthesize(struct record *rec, bool tail);
1789
1790static int
1791record__switch_output(struct record *rec, bool at_exit)
1792{
1793 struct perf_data *data = &rec->data;
1794 int fd, err;
1795 char *new_filename;
1796
1797 /* Same Size: "2015122520103046"*/
1798 char timestamp[] = "InvalidTimestamp";
1799
1800 record__aio_mmap_read_sync(rec);
1801
1802 write_finished_init(rec, true);
1803
1804 record__synthesize(rec, true);
1805 if (target__none(&rec->opts.target))
1806 record__synthesize_workload(rec, true);
1807
1808 rec->samples = 0;
1809 record__finish_output(rec);
1810 err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1811 if (err) {
1812 pr_err("Failed to get current timestamp\n");
1813 return -EINVAL;
1814 }
1815
1816 fd = perf_data__switch(data, timestamp,
1817 rec->session->header.data_offset,
1818 at_exit, &new_filename);
1819 if (fd >= 0 && !at_exit) {
1820 rec->bytes_written = 0;
1821 rec->session->header.data_size = 0;
1822 }
1823
1824 if (!quiet)
1825 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
1826 data->path, timestamp);
1827
1828 if (rec->switch_output.num_files) {
1829 int n = rec->switch_output.cur_file + 1;
1830
1831 if (n >= rec->switch_output.num_files)
1832 n = 0;
1833 rec->switch_output.cur_file = n;
1834 if (rec->switch_output.filenames[n]) {
1835 remove(rec->switch_output.filenames[n]);
1836 zfree(&rec->switch_output.filenames[n]);
1837 }
1838 rec->switch_output.filenames[n] = new_filename;
1839 } else {
1840 free(new_filename);
1841 }
1842
1843 /* Output tracking events */
1844 if (!at_exit) {
1845 record__synthesize(rec, false);
1846
1847 /*
1848 * In 'perf record --switch-output' without -a,
1849 * record__synthesize() in record__switch_output() won't
1850 * generate tracking events because there's no thread_map
1851 * in evlist. Which causes newly created perf.data doesn't
1852 * contain map and comm information.
1853 * Create a fake thread_map and directly call
1854 * perf_event__synthesize_thread_map() for those events.
1855 */
1856 if (target__none(&rec->opts.target))
1857 record__synthesize_workload(rec, false);
1858 write_finished_init(rec, false);
1859 }
1860 return fd;
1861}
1862
1863static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
1864 struct perf_record_lost_samples *lost,
1865 int cpu_idx, int thread_idx)
1866{
1867 struct perf_counts_values count;
1868 struct perf_sample_id *sid;
1869 struct perf_sample sample = {};
1870 int id_hdr_size;
1871
1872 if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
1873 pr_err("read LOST count failed\n");
1874 return;
1875 }
1876
1877 if (count.lost == 0)
1878 return;
1879
1880 lost->lost = count.lost;
1881 if (evsel->core.ids) {
1882 sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
1883 sample.id = sid->id;
1884 }
1885
1886 id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
1887 evsel->core.attr.sample_type, &sample);
1888 lost->header.size = sizeof(*lost) + id_hdr_size;
1889 record__write(rec, NULL, lost, lost->header.size);
1890}
1891
1892static void record__read_lost_samples(struct record *rec)
1893{
1894 struct perf_session *session = rec->session;
1895 struct perf_record_lost_samples *lost;
1896 struct evsel *evsel;
1897
1898 /* there was an error during record__open */
1899 if (session->evlist == NULL)
1900 return;
1901
1902 lost = zalloc(PERF_SAMPLE_MAX_SIZE);
1903 if (lost == NULL) {
1904 pr_debug("Memory allocation failed\n");
1905 return;
1906 }
1907
1908 lost->header.type = PERF_RECORD_LOST_SAMPLES;
1909
1910 evlist__for_each_entry(session->evlist, evsel) {
1911 struct xyarray *xy = evsel->core.sample_id;
1912
1913 if (xy == NULL || evsel->core.fd == NULL)
1914 continue;
1915 if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
1916 xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
1917 pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
1918 continue;
1919 }
1920
1921 for (int x = 0; x < xyarray__max_x(xy); x++) {
1922 for (int y = 0; y < xyarray__max_y(xy); y++) {
1923 __record__read_lost_samples(rec, evsel, lost, x, y);
1924 }
1925 }
1926 }
1927 free(lost);
1928
1929}
1930
1931static volatile sig_atomic_t workload_exec_errno;
1932
1933/*
1934 * evlist__prepare_workload will send a SIGUSR1
1935 * if the fork fails, since we asked by setting its
1936 * want_signal to true.
1937 */
1938static void workload_exec_failed_signal(int signo __maybe_unused,
1939 siginfo_t *info,
1940 void *ucontext __maybe_unused)
1941{
1942 workload_exec_errno = info->si_value.sival_int;
1943 done = 1;
1944 child_finished = 1;
1945}
1946
1947static void snapshot_sig_handler(int sig);
1948static void alarm_sig_handler(int sig);
1949
1950static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
1951{
1952 if (evlist) {
1953 if (evlist->mmap && evlist->mmap[0].core.base)
1954 return evlist->mmap[0].core.base;
1955 if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
1956 return evlist->overwrite_mmap[0].core.base;
1957 }
1958 return NULL;
1959}
1960
1961static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
1962{
1963 const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
1964 if (pc)
1965 return pc;
1966 return NULL;
1967}
1968
1969static int record__synthesize(struct record *rec, bool tail)
1970{
1971 struct perf_session *session = rec->session;
1972 struct machine *machine = &session->machines.host;
1973 struct perf_data *data = &rec->data;
1974 struct record_opts *opts = &rec->opts;
1975 struct perf_tool *tool = &rec->tool;
1976 int err = 0;
1977 event_op f = process_synthesized_event;
1978
1979 if (rec->opts.tail_synthesize != tail)
1980 return 0;
1981
1982 if (data->is_pipe) {
1983 err = perf_event__synthesize_for_pipe(tool, session, data,
1984 process_synthesized_event);
1985 if (err < 0)
1986 goto out;
1987
1988 rec->bytes_written += err;
1989 }
1990
1991 err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
1992 process_synthesized_event, machine);
1993 if (err)
1994 goto out;
1995
1996 /* Synthesize id_index before auxtrace_info */
1997 err = perf_event__synthesize_id_index(tool,
1998 process_synthesized_event,
1999 session->evlist, machine);
2000 if (err)
2001 goto out;
2002
2003 if (rec->opts.full_auxtrace) {
2004 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
2005 session, process_synthesized_event);
2006 if (err)
2007 goto out;
2008 }
2009
2010 if (!evlist__exclude_kernel(rec->evlist)) {
2011 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
2012 machine);
2013 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
2014 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2015 "Check /proc/kallsyms permission or run as root.\n");
2016
2017 err = perf_event__synthesize_modules(tool, process_synthesized_event,
2018 machine);
2019 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
2020 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2021 "Check /proc/modules permission or run as root.\n");
2022 }
2023
2024 if (perf_guest) {
2025 machines__process_guests(&session->machines,
2026 perf_event__synthesize_guest_os, tool);
2027 }
2028
2029 err = perf_event__synthesize_extra_attr(&rec->tool,
2030 rec->evlist,
2031 process_synthesized_event,
2032 data->is_pipe);
2033 if (err)
2034 goto out;
2035
2036 err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
2037 process_synthesized_event,
2038 NULL);
2039 if (err < 0) {
2040 pr_err("Couldn't synthesize thread map.\n");
2041 return err;
2042 }
2043
2044 err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus,
2045 process_synthesized_event, NULL);
2046 if (err < 0) {
2047 pr_err("Couldn't synthesize cpu map.\n");
2048 return err;
2049 }
2050
2051 err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
2052 machine, opts);
2053 if (err < 0) {
2054 pr_warning("Couldn't synthesize bpf events.\n");
2055 err = 0;
2056 }
2057
2058 if (rec->opts.synth & PERF_SYNTH_CGROUP) {
2059 err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
2060 machine);
2061 if (err < 0) {
2062 pr_warning("Couldn't synthesize cgroup events.\n");
2063 err = 0;
2064 }
2065 }
2066
2067 if (rec->opts.nr_threads_synthesize > 1) {
2068 mutex_init(&synth_lock);
2069 perf_set_multithreaded();
2070 f = process_locked_synthesized_event;
2071 }
2072
2073 if (rec->opts.synth & PERF_SYNTH_TASK) {
2074 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
2075
2076 err = __machine__synthesize_threads(machine, tool, &opts->target,
2077 rec->evlist->core.threads,
2078 f, needs_mmap, opts->sample_address,
2079 rec->opts.nr_threads_synthesize);
2080 }
2081
2082 if (rec->opts.nr_threads_synthesize > 1) {
2083 perf_set_singlethreaded();
2084 mutex_destroy(&synth_lock);
2085 }
2086
2087out:
2088 return err;
2089}
2090
2091static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
2092{
2093 struct record *rec = data;
2094 pthread_kill(rec->thread_id, SIGUSR2);
2095 return 0;
2096}
2097
2098static int record__setup_sb_evlist(struct record *rec)
2099{
2100 struct record_opts *opts = &rec->opts;
2101
2102 if (rec->sb_evlist != NULL) {
2103 /*
2104 * We get here if --switch-output-event populated the
2105 * sb_evlist, so associate a callback that will send a SIGUSR2
2106 * to the main thread.
2107 */
2108 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
2109 rec->thread_id = pthread_self();
2110 }
2111#ifdef HAVE_LIBBPF_SUPPORT
2112 if (!opts->no_bpf_event) {
2113 if (rec->sb_evlist == NULL) {
2114 rec->sb_evlist = evlist__new();
2115
2116 if (rec->sb_evlist == NULL) {
2117 pr_err("Couldn't create side band evlist.\n.");
2118 return -1;
2119 }
2120 }
2121
2122 if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
2123 pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
2124 return -1;
2125 }
2126 }
2127#endif
2128 if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
2129 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
2130 opts->no_bpf_event = true;
2131 }
2132
2133 return 0;
2134}
2135
2136static int record__init_clock(struct record *rec)
2137{
2138 struct perf_session *session = rec->session;
2139 struct timespec ref_clockid;
2140 struct timeval ref_tod;
2141 u64 ref;
2142
2143 if (!rec->opts.use_clockid)
2144 return 0;
2145
2146 if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
2147 session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
2148
2149 session->header.env.clock.clockid = rec->opts.clockid;
2150
2151 if (gettimeofday(&ref_tod, NULL) != 0) {
2152 pr_err("gettimeofday failed, cannot set reference time.\n");
2153 return -1;
2154 }
2155
2156 if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
2157 pr_err("clock_gettime failed, cannot set reference time.\n");
2158 return -1;
2159 }
2160
2161 ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
2162 (u64) ref_tod.tv_usec * NSEC_PER_USEC;
2163
2164 session->header.env.clock.tod_ns = ref;
2165
2166 ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
2167 (u64) ref_clockid.tv_nsec;
2168
2169 session->header.env.clock.clockid_ns = ref;
2170 return 0;
2171}
2172
2173static void hit_auxtrace_snapshot_trigger(struct record *rec)
2174{
2175 if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
2176 trigger_hit(&auxtrace_snapshot_trigger);
2177 auxtrace_record__snapshot_started = 1;
2178 if (auxtrace_record__snapshot_start(rec->itr))
2179 trigger_error(&auxtrace_snapshot_trigger);
2180 }
2181}
2182
2183static void record__uniquify_name(struct record *rec)
2184{
2185 struct evsel *pos;
2186 struct evlist *evlist = rec->evlist;
2187 char *new_name;
2188 int ret;
2189
2190 if (!perf_pmu__has_hybrid())
2191 return;
2192
2193 evlist__for_each_entry(evlist, pos) {
2194 if (!evsel__is_hybrid(pos))
2195 continue;
2196
2197 if (strchr(pos->name, '/'))
2198 continue;
2199
2200 ret = asprintf(&new_name, "%s/%s/",
2201 pos->pmu_name, pos->name);
2202 if (ret) {
2203 free(pos->name);
2204 pos->name = new_name;
2205 }
2206 }
2207}
2208
2209static int record__terminate_thread(struct record_thread *thread_data)
2210{
2211 int err;
2212 enum thread_msg ack = THREAD_MSG__UNDEFINED;
2213 pid_t tid = thread_data->tid;
2214
2215 close(thread_data->pipes.msg[1]);
2216 thread_data->pipes.msg[1] = -1;
2217 err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2218 if (err > 0)
2219 pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2220 else
2221 pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2222 thread->tid, tid);
2223
2224 return 0;
2225}
2226
2227static int record__start_threads(struct record *rec)
2228{
2229 int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
2230 struct record_thread *thread_data = rec->thread_data;
2231 sigset_t full, mask;
2232 pthread_t handle;
2233 pthread_attr_t attrs;
2234
2235 thread = &thread_data[0];
2236
2237 if (!record__threads_enabled(rec))
2238 return 0;
2239
2240 sigfillset(&full);
2241 if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2242 pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
2243 return -1;
2244 }
2245
2246 pthread_attr_init(&attrs);
2247 pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2248
2249 for (t = 1; t < nr_threads; t++) {
2250 enum thread_msg msg = THREAD_MSG__UNDEFINED;
2251
2252#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2253 pthread_attr_setaffinity_np(&attrs,
2254 MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2255 (cpu_set_t *)(thread_data[t].mask->affinity.bits));
2256#endif
2257 if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2258 for (tt = 1; tt < t; tt++)
2259 record__terminate_thread(&thread_data[t]);
2260 pr_err("Failed to start threads: %s\n", strerror(errno));
2261 ret = -1;
2262 goto out_err;
2263 }
2264
2265 err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2266 if (err > 0)
2267 pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2268 thread_msg_tags[msg]);
2269 else
2270 pr_warning("threads[%d]: failed to receive start notification from %d\n",
2271 thread->tid, rec->thread_data[t].tid);
2272 }
2273
2274 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2275 (cpu_set_t *)thread->mask->affinity.bits);
2276
2277 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2278
2279out_err:
2280 pthread_attr_destroy(&attrs);
2281
2282 if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2283 pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
2284 ret = -1;
2285 }
2286
2287 return ret;
2288}
2289
2290static int record__stop_threads(struct record *rec)
2291{
2292 int t;
2293 struct record_thread *thread_data = rec->thread_data;
2294
2295 for (t = 1; t < rec->nr_threads; t++)
2296 record__terminate_thread(&thread_data[t]);
2297
2298 for (t = 0; t < rec->nr_threads; t++) {
2299 rec->samples += thread_data[t].samples;
2300 if (!record__threads_enabled(rec))
2301 continue;
2302 rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2303 rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2304 pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2305 thread_data[t].samples, thread_data[t].waking);
2306 if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2307 pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2308 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2309 else
2310 pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2311 }
2312
2313 return 0;
2314}
2315
2316static unsigned long record__waking(struct record *rec)
2317{
2318 int t;
2319 unsigned long waking = 0;
2320 struct record_thread *thread_data = rec->thread_data;
2321
2322 for (t = 0; t < rec->nr_threads; t++)
2323 waking += thread_data[t].waking;
2324
2325 return waking;
2326}
2327
2328static int __cmd_record(struct record *rec, int argc, const char **argv)
2329{
2330 int err;
2331 int status = 0;
2332 const bool forks = argc > 0;
2333 struct perf_tool *tool = &rec->tool;
2334 struct record_opts *opts = &rec->opts;
2335 struct perf_data *data = &rec->data;
2336 struct perf_session *session;
2337 bool disabled = false, draining = false;
2338 int fd;
2339 float ratio = 0;
2340 enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
2341
2342 atexit(record__sig_exit);
2343 signal(SIGCHLD, sig_handler);
2344 signal(SIGINT, sig_handler);
2345 signal(SIGTERM, sig_handler);
2346 signal(SIGSEGV, sigsegv_handler);
2347
2348 if (rec->opts.record_namespaces)
2349 tool->namespace_events = true;
2350
2351 if (rec->opts.record_cgroup) {
2352#ifdef HAVE_FILE_HANDLE
2353 tool->cgroup_events = true;
2354#else
2355 pr_err("cgroup tracking is not supported\n");
2356 return -1;
2357#endif
2358 }
2359
2360 if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2361 signal(SIGUSR2, snapshot_sig_handler);
2362 if (rec->opts.auxtrace_snapshot_mode)
2363 trigger_on(&auxtrace_snapshot_trigger);
2364 if (rec->switch_output.enabled)
2365 trigger_on(&switch_output_trigger);
2366 } else {
2367 signal(SIGUSR2, SIG_IGN);
2368 }
2369
2370 session = perf_session__new(data, tool);
2371 if (IS_ERR(session)) {
2372 pr_err("Perf session creation failed.\n");
2373 return PTR_ERR(session);
2374 }
2375
2376 if (record__threads_enabled(rec)) {
2377 if (perf_data__is_pipe(&rec->data)) {
2378 pr_err("Parallel trace streaming is not available in pipe mode.\n");
2379 return -1;
2380 }
2381 if (rec->opts.full_auxtrace) {
2382 pr_err("Parallel trace streaming is not available in AUX area tracing mode.\n");
2383 return -1;
2384 }
2385 }
2386
2387 fd = perf_data__fd(data);
2388 rec->session = session;
2389
2390 if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2391 pr_err("Compression initialization failed.\n");
2392 return -1;
2393 }
2394#ifdef HAVE_EVENTFD_SUPPORT
2395 done_fd = eventfd(0, EFD_NONBLOCK);
2396 if (done_fd < 0) {
2397 pr_err("Failed to create wakeup eventfd, error: %m\n");
2398 status = -1;
2399 goto out_delete_session;
2400 }
2401 err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
2402 if (err < 0) {
2403 pr_err("Failed to add wakeup eventfd to poll list\n");
2404 status = err;
2405 goto out_delete_session;
2406 }
2407#endif // HAVE_EVENTFD_SUPPORT
2408
2409 session->header.env.comp_type = PERF_COMP_ZSTD;
2410 session->header.env.comp_level = rec->opts.comp_level;
2411
2412 if (rec->opts.kcore &&
2413 !record__kcore_readable(&session->machines.host)) {
2414 pr_err("ERROR: kcore is not readable.\n");
2415 return -1;
2416 }
2417
2418 if (record__init_clock(rec))
2419 return -1;
2420
2421 record__init_features(rec);
2422
2423 if (forks) {
2424 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
2425 workload_exec_failed_signal);
2426 if (err < 0) {
2427 pr_err("Couldn't run the workload!\n");
2428 status = err;
2429 goto out_delete_session;
2430 }
2431 }
2432
2433 /*
2434 * If we have just single event and are sending data
2435 * through pipe, we need to force the ids allocation,
2436 * because we synthesize event name through the pipe
2437 * and need the id for that.
2438 */
2439 if (data->is_pipe && rec->evlist->core.nr_entries == 1)
2440 rec->opts.sample_id = true;
2441
2442 record__uniquify_name(rec);
2443
2444 /* Debug message used by test scripts */
2445 pr_debug3("perf record opening and mmapping events\n");
2446 if (record__open(rec) != 0) {
2447 err = -1;
2448 goto out_free_threads;
2449 }
2450 /* Debug message used by test scripts */
2451 pr_debug3("perf record done opening and mmapping events\n");
2452 session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
2453
2454 if (rec->opts.kcore) {
2455 err = record__kcore_copy(&session->machines.host, data);
2456 if (err) {
2457 pr_err("ERROR: Failed to copy kcore\n");
2458 goto out_free_threads;
2459 }
2460 }
2461
2462 err = bpf__apply_obj_config();
2463 if (err) {
2464 char errbuf[BUFSIZ];
2465
2466 bpf__strerror_apply_obj_config(err, errbuf, sizeof(errbuf));
2467 pr_err("ERROR: Apply config to BPF failed: %s\n",
2468 errbuf);
2469 goto out_free_threads;
2470 }
2471
2472 /*
2473 * Normally perf_session__new would do this, but it doesn't have the
2474 * evlist.
2475 */
2476 if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
2477 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2478 rec->tool.ordered_events = false;
2479 }
2480
2481 if (!rec->evlist->core.nr_groups)
2482 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
2483
2484 if (data->is_pipe) {
2485 err = perf_header__write_pipe(fd);
2486 if (err < 0)
2487 goto out_free_threads;
2488 } else {
2489 err = perf_session__write_header(session, rec->evlist, fd, false);
2490 if (err < 0)
2491 goto out_free_threads;
2492 }
2493
2494 err = -1;
2495 if (!rec->no_buildid
2496 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
2497 pr_err("Couldn't generate buildids. "
2498 "Use --no-buildid to profile anyway.\n");
2499 goto out_free_threads;
2500 }
2501
2502 err = record__setup_sb_evlist(rec);
2503 if (err)
2504 goto out_free_threads;
2505
2506 err = record__synthesize(rec, false);
2507 if (err < 0)
2508 goto out_free_threads;
2509
2510 if (rec->realtime_prio) {
2511 struct sched_param param;
2512
2513 param.sched_priority = rec->realtime_prio;
2514 if (sched_setscheduler(0, SCHED_FIFO, ¶m)) {
2515 pr_err("Could not set realtime priority.\n");
2516 err = -1;
2517 goto out_free_threads;
2518 }
2519 }
2520
2521 if (record__start_threads(rec))
2522 goto out_free_threads;
2523
2524 /*
2525 * When perf is starting the traced process, all the events
2526 * (apart from group members) have enable_on_exec=1 set,
2527 * so don't spoil it by prematurely enabling them.
2528 */
2529 if (!target__none(&opts->target) && !opts->initial_delay)
2530 evlist__enable(rec->evlist);
2531
2532 /*
2533 * Let the child rip
2534 */
2535 if (forks) {
2536 struct machine *machine = &session->machines.host;
2537 union perf_event *event;
2538 pid_t tgid;
2539
2540 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2541 if (event == NULL) {
2542 err = -ENOMEM;
2543 goto out_child;
2544 }
2545
2546 /*
2547 * Some H/W events are generated before COMM event
2548 * which is emitted during exec(), so perf script
2549 * cannot see a correct process name for those events.
2550 * Synthesize COMM event to prevent it.
2551 */
2552 tgid = perf_event__synthesize_comm(tool, event,
2553 rec->evlist->workload.pid,
2554 process_synthesized_event,
2555 machine);
2556 free(event);
2557
2558 if (tgid == -1)
2559 goto out_child;
2560
2561 event = malloc(sizeof(event->namespaces) +
2562 (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2563 machine->id_hdr_size);
2564 if (event == NULL) {
2565 err = -ENOMEM;
2566 goto out_child;
2567 }
2568
2569 /*
2570 * Synthesize NAMESPACES event for the command specified.
2571 */
2572 perf_event__synthesize_namespaces(tool, event,
2573 rec->evlist->workload.pid,
2574 tgid, process_synthesized_event,
2575 machine);
2576 free(event);
2577
2578 evlist__start_workload(rec->evlist);
2579 }
2580
2581 if (opts->initial_delay) {
2582 pr_info(EVLIST_DISABLED_MSG);
2583 if (opts->initial_delay > 0) {
2584 usleep(opts->initial_delay * USEC_PER_MSEC);
2585 evlist__enable(rec->evlist);
2586 pr_info(EVLIST_ENABLED_MSG);
2587 }
2588 }
2589
2590 err = event_enable_timer__start(rec->evlist->eet);
2591 if (err)
2592 goto out_child;
2593
2594 /* Debug message used by test scripts */
2595 pr_debug3("perf record has started\n");
2596 fflush(stderr);
2597
2598 trigger_ready(&auxtrace_snapshot_trigger);
2599 trigger_ready(&switch_output_trigger);
2600 perf_hooks__invoke_record_start();
2601
2602 /*
2603 * Must write FINISHED_INIT so it will be seen after all other
2604 * synthesized user events, but before any regular events.
2605 */
2606 err = write_finished_init(rec, false);
2607 if (err < 0)
2608 goto out_child;
2609
2610 for (;;) {
2611 unsigned long long hits = thread->samples;
2612
2613 /*
2614 * rec->evlist->bkw_mmap_state is possible to be
2615 * BKW_MMAP_EMPTY here: when done == true and
2616 * hits != rec->samples in previous round.
2617 *
2618 * evlist__toggle_bkw_mmap ensure we never
2619 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2620 */
2621 if (trigger_is_hit(&switch_output_trigger) || done || draining)
2622 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
2623
2624 if (record__mmap_read_all(rec, false) < 0) {
2625 trigger_error(&auxtrace_snapshot_trigger);
2626 trigger_error(&switch_output_trigger);
2627 err = -1;
2628 goto out_child;
2629 }
2630
2631 if (auxtrace_record__snapshot_started) {
2632 auxtrace_record__snapshot_started = 0;
2633 if (!trigger_is_error(&auxtrace_snapshot_trigger))
2634 record__read_auxtrace_snapshot(rec, false);
2635 if (trigger_is_error(&auxtrace_snapshot_trigger)) {
2636 pr_err("AUX area tracing snapshot failed\n");
2637 err = -1;
2638 goto out_child;
2639 }
2640 }
2641
2642 if (trigger_is_hit(&switch_output_trigger)) {
2643 /*
2644 * If switch_output_trigger is hit, the data in
2645 * overwritable ring buffer should have been collected,
2646 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2647 *
2648 * If SIGUSR2 raise after or during record__mmap_read_all(),
2649 * record__mmap_read_all() didn't collect data from
2650 * overwritable ring buffer. Read again.
2651 */
2652 if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
2653 continue;
2654 trigger_ready(&switch_output_trigger);
2655
2656 /*
2657 * Reenable events in overwrite ring buffer after
2658 * record__mmap_read_all(): we should have collected
2659 * data from it.
2660 */
2661 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
2662
2663 if (!quiet)
2664 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
2665 record__waking(rec));
2666 thread->waking = 0;
2667 fd = record__switch_output(rec, false);
2668 if (fd < 0) {
2669 pr_err("Failed to switch to new file\n");
2670 trigger_error(&switch_output_trigger);
2671 err = fd;
2672 goto out_child;
2673 }
2674
2675 /* re-arm the alarm */
2676 if (rec->switch_output.time)
2677 alarm(rec->switch_output.time);
2678 }
2679
2680 if (hits == thread->samples) {
2681 if (done || draining)
2682 break;
2683 err = fdarray__poll(&thread->pollfd, -1);
2684 /*
2685 * Propagate error, only if there's any. Ignore positive
2686 * number of returned events and interrupt error.
2687 */
2688 if (err > 0 || (err < 0 && errno == EINTR))
2689 err = 0;
2690 thread->waking++;
2691
2692 if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2693 record__thread_munmap_filtered, NULL) == 0)
2694 draining = true;
2695
2696 err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread);
2697 if (err)
2698 goto out_child;
2699 }
2700
2701 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
2702 switch (cmd) {
2703 case EVLIST_CTL_CMD_SNAPSHOT:
2704 hit_auxtrace_snapshot_trigger(rec);
2705 evlist__ctlfd_ack(rec->evlist);
2706 break;
2707 case EVLIST_CTL_CMD_STOP:
2708 done = 1;
2709 break;
2710 case EVLIST_CTL_CMD_ACK:
2711 case EVLIST_CTL_CMD_UNSUPPORTED:
2712 case EVLIST_CTL_CMD_ENABLE:
2713 case EVLIST_CTL_CMD_DISABLE:
2714 case EVLIST_CTL_CMD_EVLIST:
2715 case EVLIST_CTL_CMD_PING:
2716 default:
2717 break;
2718 }
2719 }
2720
2721 err = event_enable_timer__process(rec->evlist->eet);
2722 if (err < 0)
2723 goto out_child;
2724 if (err) {
2725 err = 0;
2726 done = 1;
2727 }
2728
2729 /*
2730 * When perf is starting the traced process, at the end events
2731 * die with the process and we wait for that. Thus no need to
2732 * disable events in this case.
2733 */
2734 if (done && !disabled && !target__none(&opts->target)) {
2735 trigger_off(&auxtrace_snapshot_trigger);
2736 evlist__disable(rec->evlist);
2737 disabled = true;
2738 }
2739 }
2740
2741 trigger_off(&auxtrace_snapshot_trigger);
2742 trigger_off(&switch_output_trigger);
2743
2744 if (opts->auxtrace_snapshot_on_exit)
2745 record__auxtrace_snapshot_exit(rec);
2746
2747 if (forks && workload_exec_errno) {
2748 char msg[STRERR_BUFSIZE], strevsels[2048];
2749 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
2750
2751 evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);
2752
2753 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2754 strevsels, argv[0], emsg);
2755 err = -1;
2756 goto out_child;
2757 }
2758
2759 if (!quiet)
2760 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2761 record__waking(rec));
2762
2763 write_finished_init(rec, true);
2764
2765 if (target__none(&rec->opts.target))
2766 record__synthesize_workload(rec, true);
2767
2768out_child:
2769 record__stop_threads(rec);
2770 record__mmap_read_all(rec, true);
2771out_free_threads:
2772 record__free_thread_data(rec);
2773 evlist__finalize_ctlfd(rec->evlist);
2774 record__aio_mmap_read_sync(rec);
2775
2776 if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2777 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2778 session->header.env.comp_ratio = ratio + 0.5;
2779 }
2780
2781 if (forks) {
2782 int exit_status;
2783
2784 if (!child_finished)
2785 kill(rec->evlist->workload.pid, SIGTERM);
2786
2787 wait(&exit_status);
2788
2789 if (err < 0)
2790 status = err;
2791 else if (WIFEXITED(exit_status))
2792 status = WEXITSTATUS(exit_status);
2793 else if (WIFSIGNALED(exit_status))
2794 signr = WTERMSIG(exit_status);
2795 } else
2796 status = err;
2797
2798 if (rec->off_cpu)
2799 rec->bytes_written += off_cpu_write(rec->session);
2800
2801 record__read_lost_samples(rec);
2802 record__synthesize(rec, true);
2803 /* this will be recalculated during process_buildids() */
2804 rec->samples = 0;
2805
2806 if (!err) {
2807 if (!rec->timestamp_filename) {
2808 record__finish_output(rec);
2809 } else {
2810 fd = record__switch_output(rec, true);
2811 if (fd < 0) {
2812 status = fd;
2813 goto out_delete_session;
2814 }
2815 }
2816 }
2817
2818 perf_hooks__invoke_record_end();
2819
2820 if (!err && !quiet) {
2821 char samples[128];
2822 const char *postfix = rec->timestamp_filename ?
2823 ".<timestamp>" : "";
2824
2825 if (rec->samples && !rec->opts.full_auxtrace)
2826 scnprintf(samples, sizeof(samples),
2827 " (%" PRIu64 " samples)", rec->samples);
2828 else
2829 samples[0] = '\0';
2830
2831 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
2832 perf_data__size(data) / 1024.0 / 1024.0,
2833 data->path, postfix, samples);
2834 if (ratio) {
2835 fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2836 rec->session->bytes_transferred / 1024.0 / 1024.0,
2837 ratio);
2838 }
2839 fprintf(stderr, " ]\n");
2840 }
2841
2842out_delete_session:
2843#ifdef HAVE_EVENTFD_SUPPORT
2844 if (done_fd >= 0) {
2845 fd = done_fd;
2846 done_fd = -1;
2847
2848 close(fd);
2849 }
2850#endif
2851 zstd_fini(&session->zstd_data);
2852 perf_session__delete(session);
2853
2854 if (!opts->no_bpf_event)
2855 evlist__stop_sb_thread(rec->sb_evlist);
2856 return status;
2857}
2858
2859static void callchain_debug(struct callchain_param *callchain)
2860{
2861 static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
2862
2863 pr_debug("callchain: type %s\n", str[callchain->record_mode]);
2864
2865 if (callchain->record_mode == CALLCHAIN_DWARF)
2866 pr_debug("callchain: stack dump size %d\n",
2867 callchain->dump_size);
2868}
2869
2870int record_opts__parse_callchain(struct record_opts *record,
2871 struct callchain_param *callchain,
2872 const char *arg, bool unset)
2873{
2874 int ret;
2875 callchain->enabled = !unset;
2876
2877 /* --no-call-graph */
2878 if (unset) {
2879 callchain->record_mode = CALLCHAIN_NONE;
2880 pr_debug("callchain: disabled\n");
2881 return 0;
2882 }
2883
2884 ret = parse_callchain_record_opt(arg, callchain);
2885 if (!ret) {
2886 /* Enable data address sampling for DWARF unwind. */
2887 if (callchain->record_mode == CALLCHAIN_DWARF)
2888 record->sample_address = true;
2889 callchain_debug(callchain);
2890 }
2891
2892 return ret;
2893}
2894
2895int record_parse_callchain_opt(const struct option *opt,
2896 const char *arg,
2897 int unset)
2898{
2899 return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
2900}
2901
2902int record_callchain_opt(const struct option *opt,
2903 const char *arg __maybe_unused,
2904 int unset __maybe_unused)
2905{
2906 struct callchain_param *callchain = opt->value;
2907
2908 callchain->enabled = true;
2909
2910 if (callchain->record_mode == CALLCHAIN_NONE)
2911 callchain->record_mode = CALLCHAIN_FP;
2912
2913 callchain_debug(callchain);
2914 return 0;
2915}
2916
2917static int perf_record_config(const char *var, const char *value, void *cb)
2918{
2919 struct record *rec = cb;
2920
2921 if (!strcmp(var, "record.build-id")) {
2922 if (!strcmp(value, "cache"))
2923 rec->no_buildid_cache = false;
2924 else if (!strcmp(value, "no-cache"))
2925 rec->no_buildid_cache = true;
2926 else if (!strcmp(value, "skip"))
2927 rec->no_buildid = true;
2928 else if (!strcmp(value, "mmap"))
2929 rec->buildid_mmap = true;
2930 else
2931 return -1;
2932 return 0;
2933 }
2934 if (!strcmp(var, "record.call-graph")) {
2935 var = "call-graph.record-mode";
2936 return perf_default_config(var, value, cb);
2937 }
2938#ifdef HAVE_AIO_SUPPORT
2939 if (!strcmp(var, "record.aio")) {
2940 rec->opts.nr_cblocks = strtol(value, NULL, 0);
2941 if (!rec->opts.nr_cblocks)
2942 rec->opts.nr_cblocks = nr_cblocks_default;
2943 }
2944#endif
2945 if (!strcmp(var, "record.debuginfod")) {
2946 rec->debuginfod.urls = strdup(value);
2947 if (!rec->debuginfod.urls)
2948 return -ENOMEM;
2949 rec->debuginfod.set = true;
2950 }
2951
2952 return 0;
2953}
2954
2955static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
2956{
2957 struct record *rec = (struct record *)opt->value;
2958
2959 return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset);
2960}
2961
2962static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2963{
2964 struct record_opts *opts = (struct record_opts *)opt->value;
2965
2966 if (unset || !str)
2967 return 0;
2968
2969 if (!strcasecmp(str, "node"))
2970 opts->affinity = PERF_AFFINITY_NODE;
2971 else if (!strcasecmp(str, "cpu"))
2972 opts->affinity = PERF_AFFINITY_CPU;
2973
2974 return 0;
2975}
2976
2977static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
2978{
2979 mask->nbits = nr_bits;
2980 mask->bits = bitmap_zalloc(mask->nbits);
2981 if (!mask->bits)
2982 return -ENOMEM;
2983
2984 return 0;
2985}
2986
2987static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
2988{
2989 bitmap_free(mask->bits);
2990 mask->nbits = 0;
2991}
2992
2993static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
2994{
2995 int ret;
2996
2997 ret = record__mmap_cpu_mask_alloc(&mask->maps, nr_bits);
2998 if (ret) {
2999 mask->affinity.bits = NULL;
3000 return ret;
3001 }
3002
3003 ret = record__mmap_cpu_mask_alloc(&mask->affinity, nr_bits);
3004 if (ret) {
3005 record__mmap_cpu_mask_free(&mask->maps);
3006 mask->maps.bits = NULL;
3007 }
3008
3009 return ret;
3010}
3011
3012static void record__thread_mask_free(struct thread_mask *mask)
3013{
3014 record__mmap_cpu_mask_free(&mask->maps);
3015 record__mmap_cpu_mask_free(&mask->affinity);
3016}
3017
3018static int record__parse_threads(const struct option *opt, const char *str, int unset)
3019{
3020 int s;
3021 struct record_opts *opts = opt->value;
3022
3023 if (unset || !str || !strlen(str)) {
3024 opts->threads_spec = THREAD_SPEC__CPU;
3025 } else {
3026 for (s = 1; s < THREAD_SPEC__MAX; s++) {
3027 if (s == THREAD_SPEC__USER) {
3028 opts->threads_user_spec = strdup(str);
3029 if (!opts->threads_user_spec)
3030 return -ENOMEM;
3031 opts->threads_spec = THREAD_SPEC__USER;
3032 break;
3033 }
3034 if (!strncasecmp(str, thread_spec_tags[s], strlen(thread_spec_tags[s]))) {
3035 opts->threads_spec = s;
3036 break;
3037 }
3038 }
3039 }
3040
3041 if (opts->threads_spec == THREAD_SPEC__USER)
3042 pr_debug("threads_spec: %s\n", opts->threads_user_spec);
3043 else
3044 pr_debug("threads_spec: %s\n", thread_spec_tags[opts->threads_spec]);
3045
3046 return 0;
3047}
3048
3049static int parse_output_max_size(const struct option *opt,
3050 const char *str, int unset)
3051{
3052 unsigned long *s = (unsigned long *)opt->value;
3053 static struct parse_tag tags_size[] = {
3054 { .tag = 'B', .mult = 1 },
3055 { .tag = 'K', .mult = 1 << 10 },
3056 { .tag = 'M', .mult = 1 << 20 },
3057 { .tag = 'G', .mult = 1 << 30 },
3058 { .tag = 0 },
3059 };
3060 unsigned long val;
3061
3062 if (unset) {
3063 *s = 0;
3064 return 0;
3065 }
3066
3067 val = parse_tag_value(str, tags_size);
3068 if (val != (unsigned long) -1) {
3069 *s = val;
3070 return 0;
3071 }
3072
3073 return -1;
3074}
3075
3076static int record__parse_mmap_pages(const struct option *opt,
3077 const char *str,
3078 int unset __maybe_unused)
3079{
3080 struct record_opts *opts = opt->value;
3081 char *s, *p;
3082 unsigned int mmap_pages;
3083 int ret;
3084
3085 if (!str)
3086 return -EINVAL;
3087
3088 s = strdup(str);
3089 if (!s)
3090 return -ENOMEM;
3091
3092 p = strchr(s, ',');
3093 if (p)
3094 *p = '\0';
3095
3096 if (*s) {
3097 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
3098 if (ret)
3099 goto out_free;
3100 opts->mmap_pages = mmap_pages;
3101 }
3102
3103 if (!p) {
3104 ret = 0;
3105 goto out_free;
3106 }
3107
3108 ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
3109 if (ret)
3110 goto out_free;
3111
3112 opts->auxtrace_mmap_pages = mmap_pages;
3113
3114out_free:
3115 free(s);
3116 return ret;
3117}
3118
3119void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused)
3120{
3121}
3122
3123static int parse_control_option(const struct option *opt,
3124 const char *str,
3125 int unset __maybe_unused)
3126{
3127 struct record_opts *opts = opt->value;
3128
3129 return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
3130}
3131
3132static void switch_output_size_warn(struct record *rec)
3133{
3134 u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
3135 struct switch_output *s = &rec->switch_output;
3136
3137 wakeup_size /= 2;
3138
3139 if (s->size < wakeup_size) {
3140 char buf[100];
3141
3142 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
3143 pr_warning("WARNING: switch-output data size lower than "
3144 "wakeup kernel buffer size (%s) "
3145 "expect bigger perf.data sizes\n", buf);
3146 }
3147}
3148
3149static int switch_output_setup(struct record *rec)
3150{
3151 struct switch_output *s = &rec->switch_output;
3152 static struct parse_tag tags_size[] = {
3153 { .tag = 'B', .mult = 1 },
3154 { .tag = 'K', .mult = 1 << 10 },
3155 { .tag = 'M', .mult = 1 << 20 },
3156 { .tag = 'G', .mult = 1 << 30 },
3157 { .tag = 0 },
3158 };
3159 static struct parse_tag tags_time[] = {
3160 { .tag = 's', .mult = 1 },
3161 { .tag = 'm', .mult = 60 },
3162 { .tag = 'h', .mult = 60*60 },
3163 { .tag = 'd', .mult = 60*60*24 },
3164 { .tag = 0 },
3165 };
3166 unsigned long val;
3167
3168 /*
3169 * If we're using --switch-output-events, then we imply its
3170 * --switch-output=signal, as we'll send a SIGUSR2 from the side band
3171 * thread to its parent.
3172 */
3173 if (rec->switch_output_event_set) {
3174 if (record__threads_enabled(rec)) {
3175 pr_warning("WARNING: --switch-output-event option is not available in parallel streaming mode.\n");
3176 return 0;
3177 }
3178 goto do_signal;
3179 }
3180
3181 if (!s->set)
3182 return 0;
3183
3184 if (record__threads_enabled(rec)) {
3185 pr_warning("WARNING: --switch-output option is not available in parallel streaming mode.\n");
3186 return 0;
3187 }
3188
3189 if (!strcmp(s->str, "signal")) {
3190do_signal:
3191 s->signal = true;
3192 pr_debug("switch-output with SIGUSR2 signal\n");
3193 goto enabled;
3194 }
3195
3196 val = parse_tag_value(s->str, tags_size);
3197 if (val != (unsigned long) -1) {
3198 s->size = val;
3199 pr_debug("switch-output with %s size threshold\n", s->str);
3200 goto enabled;
3201 }
3202
3203 val = parse_tag_value(s->str, tags_time);
3204 if (val != (unsigned long) -1) {
3205 s->time = val;
3206 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
3207 s->str, s->time);
3208 goto enabled;
3209 }
3210
3211 return -1;
3212
3213enabled:
3214 rec->timestamp_filename = true;
3215 s->enabled = true;
3216
3217 if (s->size && !rec->opts.no_buffering)
3218 switch_output_size_warn(rec);
3219
3220 return 0;
3221}
3222
3223static const char * const __record_usage[] = {
3224 "perf record [<options>] [<command>]",
3225 "perf record [<options>] -- <command> [<options>]",
3226 NULL
3227};
3228const char * const *record_usage = __record_usage;
3229
3230static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
3231 struct perf_sample *sample, struct machine *machine)
3232{
3233 /*
3234 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3235 * no need to add them twice.
3236 */
3237 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3238 return 0;
3239 return perf_event__process_mmap(tool, event, sample, machine);
3240}
3241
3242static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
3243 struct perf_sample *sample, struct machine *machine)
3244{
3245 /*
3246 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3247 * no need to add them twice.
3248 */
3249 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3250 return 0;
3251
3252 return perf_event__process_mmap2(tool, event, sample, machine);
3253}
3254
3255static int process_timestamp_boundary(struct perf_tool *tool,
3256 union perf_event *event __maybe_unused,
3257 struct perf_sample *sample,
3258 struct machine *machine __maybe_unused)
3259{
3260 struct record *rec = container_of(tool, struct record, tool);
3261
3262 set_timestamp_boundary(rec, sample->time);
3263 return 0;
3264}
3265
3266static int parse_record_synth_option(const struct option *opt,
3267 const char *str,
3268 int unset __maybe_unused)
3269{
3270 struct record_opts *opts = opt->value;
3271 char *p = strdup(str);
3272
3273 if (p == NULL)
3274 return -1;
3275
3276 opts->synth = parse_synth_opt(p);
3277 free(p);
3278
3279 if (opts->synth < 0) {
3280 pr_err("Invalid synth option: %s\n", str);
3281 return -1;
3282 }
3283 return 0;
3284}
3285
3286/*
3287 * XXX Ideally would be local to cmd_record() and passed to a record__new
3288 * because we need to have access to it in record__exit, that is called
3289 * after cmd_record() exits, but since record_options need to be accessible to
3290 * builtin-script, leave it here.
3291 *
3292 * At least we don't ouch it in all the other functions here directly.
3293 *
3294 * Just say no to tons of global variables, sigh.
3295 */
3296static struct record record = {
3297 .opts = {
3298 .sample_time = true,
3299 .mmap_pages = UINT_MAX,
3300 .user_freq = UINT_MAX,
3301 .user_interval = ULLONG_MAX,
3302 .freq = 4000,
3303 .target = {
3304 .uses_mmap = true,
3305 .default_per_cpu = true,
3306 },
3307 .mmap_flush = MMAP_FLUSH_DEFAULT,
3308 .nr_threads_synthesize = 1,
3309 .ctl_fd = -1,
3310 .ctl_fd_ack = -1,
3311 .synth = PERF_SYNTH_ALL,
3312 },
3313 .tool = {
3314 .sample = process_sample_event,
3315 .fork = perf_event__process_fork,
3316 .exit = perf_event__process_exit,
3317 .comm = perf_event__process_comm,
3318 .namespaces = perf_event__process_namespaces,
3319 .mmap = build_id__process_mmap,
3320 .mmap2 = build_id__process_mmap2,
3321 .itrace_start = process_timestamp_boundary,
3322 .aux = process_timestamp_boundary,
3323 .ordered_events = true,
3324 },
3325};
3326
3327const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3328 "\n\t\t\t\tDefault: fp";
3329
3330static bool dry_run;
3331
3332/*
3333 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
3334 * with it and switch to use the library functions in perf_evlist that came
3335 * from builtin-record.c, i.e. use record_opts,
3336 * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
3337 * using pipes, etc.
3338 */
3339static struct option __record_options[] = {
3340 OPT_CALLBACK('e', "event", &record.evlist, "event",
3341 "event selector. use 'perf list' to list available events",
3342 parse_events_option),
3343 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
3344 "event filter", parse_filter),
3345 OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3346 NULL, "don't record events from perf itself",
3347 exclude_perf),
3348 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
3349 "record events on existing process id"),
3350 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
3351 "record events on existing thread id"),
3352 OPT_INTEGER('r', "realtime", &record.realtime_prio,
3353 "collect data with this RT SCHED_FIFO priority"),
3354 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
3355 "collect data without buffering"),
3356 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
3357 "collect raw sample records from all opened counters"),
3358 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
3359 "system-wide collection from all CPUs"),
3360 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
3361 "list of cpus to monitor"),
3362 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
3363 OPT_STRING('o', "output", &record.data.path, "file",
3364 "output file name"),
3365 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3366 &record.opts.no_inherit_set,
3367 "child tasks do not inherit counters"),
3368 OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3369 "synthesize non-sample events at the end of output"),
3370 OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
3371 OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
3372 OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3373 "Fail if the specified frequency can't be used"),
3374 OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3375 "profile at this frequency",
3376 record__parse_freq),
3377 OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3378 "number of mmap data pages and AUX area tracing mmap pages",
3379 record__parse_mmap_pages),
3380 OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3381 "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3382 record__mmap_flush_parse),
3383 OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
3384 NULL, "enables call-graph recording" ,
3385 &record_callchain_opt),
3386 OPT_CALLBACK(0, "call-graph", &record.opts,
3387 "record_mode[,record_size]", record_callchain_help,
3388 &record_parse_callchain_opt),
3389 OPT_INCR('v', "verbose", &verbose,
3390 "be more verbose (show counter open errors, etc)"),
3391 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any warnings or messages"),
3392 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
3393 "per thread counts"),
3394 OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3395 OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3396 "Record the sample physical addresses"),
3397 OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3398 "Record the sampled data address data page size"),
3399 OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3400 "Record the sampled code address (ip) page size"),
3401 OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
3402 OPT_BOOLEAN(0, "sample-identifier", &record.opts.sample_identifier,
3403 "Record the sample identifier"),
3404 OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3405 &record.opts.sample_time_set,
3406 "Record the sample timestamps"),
3407 OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3408 "Record the sample period"),
3409 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
3410 "don't sample"),
3411 OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3412 &record.no_buildid_cache_set,
3413 "do not update the buildid cache"),
3414 OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3415 &record.no_buildid_set,
3416 "do not collect buildids in perf.data"),
3417 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
3418 "monitor event in cgroup name only",
3419 parse_cgroups),
3420 OPT_CALLBACK('D', "delay", &record, "ms",
3421 "ms to wait before starting measurement after program start (-1: start with events disabled), "
3422 "or ranges of time to enable events e.g. '-D 10-20,30-40'",
3423 record__parse_event_enable_time),
3424 OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
3425 OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
3426 "user to profile"),
3427
3428 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3429 "branch any", "sample any taken branches",
3430 parse_branch_stack),
3431
3432 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3433 "branch filter mask", "branch stack filter modes",
3434 parse_branch_stack),
3435 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3436 "sample by weight (on special events only)"),
3437 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3438 "sample transaction flags (special events only)"),
3439 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3440 "use per-thread mmaps"),
3441 OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3442 "sample selected machine registers on interrupt,"
3443 " use '-I?' to list register names", parse_intr_regs),
3444 OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3445 "sample selected machine registers on interrupt,"
3446 " use '--user-regs=?' to list register names", parse_user_regs),
3447 OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3448 "Record running/enabled time of read (:S) events"),
3449 OPT_CALLBACK('k', "clockid", &record.opts,
3450 "clockid", "clockid to use for events, see clock_gettime()",
3451 parse_clockid),
3452 OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3453 "opts", "AUX area tracing Snapshot Mode", ""),
3454 OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3455 "opts", "sample AUX area", ""),
3456 OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
3457 "per thread proc mmap processing timeout in ms"),
3458 OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3459 "Record namespaces events"),
3460 OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3461 "Record cgroup events"),
3462 OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3463 &record.opts.record_switch_events_set,
3464 "Record context switch events"),
3465 OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3466 "Configure all used events to run in kernel space.",
3467 PARSE_OPT_EXCLUSIVE),
3468 OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3469 "Configure all used events to run in user space.",
3470 PARSE_OPT_EXCLUSIVE),
3471 OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3472 "collect kernel callchains"),
3473 OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3474 "collect user callchains"),
3475 OPT_STRING(0, "clang-path", &llvm_param.clang_path, "clang path",
3476 "clang binary to use for compiling BPF scriptlets"),
3477 OPT_STRING(0, "clang-opt", &llvm_param.clang_opt, "clang options",
3478 "options passed to clang when compiling BPF scriptlets"),
3479 OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3480 "file", "vmlinux pathname"),
3481 OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3482 "Record build-id of all DSOs regardless of hits"),
3483 OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
3484 "Record build-id in map events"),
3485 OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3486 "append timestamp to output filename"),
3487 OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3488 "Record timestamp boundary (time of first/last samples)"),
3489 OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
3490 &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3491 "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
3492 "signal"),
3493 OPT_CALLBACK_SET(0, "switch-output-event", &record.sb_evlist, &record.switch_output_event_set, "switch output event",
3494 "switch output event selector. use 'perf list' to list available events",
3495 parse_events_option_new_evlist),
3496 OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3497 "Limit number of switch output generated files"),
3498 OPT_BOOLEAN(0, "dry-run", &dry_run,
3499 "Parse options then exit"),
3500#ifdef HAVE_AIO_SUPPORT
3501 OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3502 &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
3503 record__aio_parse),
3504#endif
3505 OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3506 "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3507 record__parse_affinity),
3508#ifdef HAVE_ZSTD_SUPPORT
3509 OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default, "n",
3510 "Compress records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
3511 record__parse_comp_level),
3512#endif
3513 OPT_CALLBACK(0, "max-size", &record.output_max_size,
3514 "size", "Limit the maximum size of the output file", parse_output_max_size),
3515 OPT_UINTEGER(0, "num-thread-synthesize",
3516 &record.opts.nr_threads_synthesize,
3517 "number of threads to run for event synthesis"),
3518#ifdef HAVE_LIBPFM
3519 OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3520 "libpfm4 event selector. use 'perf list' to list available events",
3521 parse_libpfm_events_option),
3522#endif
3523 OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
3524 "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3525 "\t\t\t 'snapshot': AUX area tracing snapshot).\n"
3526 "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3527 "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
3528 parse_control_option),
3529 OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3530 "Fine-tune event synthesis: default=all", parse_record_synth_option),
3531 OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3532 &record.debuginfod.set, "debuginfod urls",
3533 "Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3534 "system"),
3535 OPT_CALLBACK_OPTARG(0, "threads", &record.opts, NULL, "spec",
3536 "write collected trace data into several data files using parallel threads",
3537 record__parse_threads),
3538 OPT_BOOLEAN(0, "off-cpu", &record.off_cpu, "Enable off-cpu analysis"),
3539 OPT_END()
3540};
3541
3542struct option *record_options = __record_options;
3543
3544static int record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
3545{
3546 struct perf_cpu cpu;
3547 int idx;
3548
3549 if (cpu_map__is_dummy(cpus))
3550 return 0;
3551
3552 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
3553 if (cpu.cpu == -1)
3554 continue;
3555 /* Return ENODEV is input cpu is greater than max cpu */
3556 if ((unsigned long)cpu.cpu > mask->nbits)
3557 return -ENODEV;
3558 __set_bit(cpu.cpu, mask->bits);
3559 }
3560
3561 return 0;
3562}
3563
3564static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const char *mask_spec)
3565{
3566 struct perf_cpu_map *cpus;
3567
3568 cpus = perf_cpu_map__new(mask_spec);
3569 if (!cpus)
3570 return -ENOMEM;
3571
3572 bitmap_zero(mask->bits, mask->nbits);
3573 if (record__mmap_cpu_mask_init(mask, cpus))
3574 return -ENODEV;
3575
3576 perf_cpu_map__put(cpus);
3577
3578 return 0;
3579}
3580
3581static void record__free_thread_masks(struct record *rec, int nr_threads)
3582{
3583 int t;
3584
3585 if (rec->thread_masks)
3586 for (t = 0; t < nr_threads; t++)
3587 record__thread_mask_free(&rec->thread_masks[t]);
3588
3589 zfree(&rec->thread_masks);
3590}
3591
3592static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3593{
3594 int t, ret;
3595
3596 rec->thread_masks = zalloc(nr_threads * sizeof(*(rec->thread_masks)));
3597 if (!rec->thread_masks) {
3598 pr_err("Failed to allocate thread masks\n");
3599 return -ENOMEM;
3600 }
3601
3602 for (t = 0; t < nr_threads; t++) {
3603 ret = record__thread_mask_alloc(&rec->thread_masks[t], nr_bits);
3604 if (ret) {
3605 pr_err("Failed to allocate thread masks[%d]\n", t);
3606 goto out_free;
3607 }
3608 }
3609
3610 return 0;
3611
3612out_free:
3613 record__free_thread_masks(rec, nr_threads);
3614
3615 return ret;
3616}
3617
3618static int record__init_thread_cpu_masks(struct record *rec, struct perf_cpu_map *cpus)
3619{
3620 int t, ret, nr_cpus = perf_cpu_map__nr(cpus);
3621
3622 ret = record__alloc_thread_masks(rec, nr_cpus, cpu__max_cpu().cpu);
3623 if (ret)
3624 return ret;
3625
3626 rec->nr_threads = nr_cpus;
3627 pr_debug("nr_threads: %d\n", rec->nr_threads);
3628
3629 for (t = 0; t < rec->nr_threads; t++) {
3630 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].maps.bits);
3631 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].affinity.bits);
3632 if (verbose > 0) {
3633 pr_debug("thread_masks[%d]: ", t);
3634 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3635 pr_debug("thread_masks[%d]: ", t);
3636 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3637 }
3638 }
3639
3640 return 0;
3641}
3642
3643static int record__init_thread_masks_spec(struct record *rec, struct perf_cpu_map *cpus,
3644 const char **maps_spec, const char **affinity_spec,
3645 u32 nr_spec)
3646{
3647 u32 s;
3648 int ret = 0, t = 0;
3649 struct mmap_cpu_mask cpus_mask;
3650 struct thread_mask thread_mask, full_mask, *thread_masks;
3651
3652 ret = record__mmap_cpu_mask_alloc(&cpus_mask, cpu__max_cpu().cpu);
3653 if (ret) {
3654 pr_err("Failed to allocate CPUs mask\n");
3655 return ret;
3656 }
3657
3658 ret = record__mmap_cpu_mask_init(&cpus_mask, cpus);
3659 if (ret) {
3660 pr_err("Failed to init cpu mask\n");
3661 goto out_free_cpu_mask;
3662 }
3663
3664 ret = record__thread_mask_alloc(&full_mask, cpu__max_cpu().cpu);
3665 if (ret) {
3666 pr_err("Failed to allocate full mask\n");
3667 goto out_free_cpu_mask;
3668 }
3669
3670 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3671 if (ret) {
3672 pr_err("Failed to allocate thread mask\n");
3673 goto out_free_full_and_cpu_masks;
3674 }
3675
3676 for (s = 0; s < nr_spec; s++) {
3677 ret = record__mmap_cpu_mask_init_spec(&thread_mask.maps, maps_spec[s]);
3678 if (ret) {
3679 pr_err("Failed to initialize maps thread mask\n");
3680 goto out_free;
3681 }
3682 ret = record__mmap_cpu_mask_init_spec(&thread_mask.affinity, affinity_spec[s]);
3683 if (ret) {
3684 pr_err("Failed to initialize affinity thread mask\n");
3685 goto out_free;
3686 }
3687
3688 /* ignore invalid CPUs but do not allow empty masks */
3689 if (!bitmap_and(thread_mask.maps.bits, thread_mask.maps.bits,
3690 cpus_mask.bits, thread_mask.maps.nbits)) {
3691 pr_err("Empty maps mask: %s\n", maps_spec[s]);
3692 ret = -EINVAL;
3693 goto out_free;
3694 }
3695 if (!bitmap_and(thread_mask.affinity.bits, thread_mask.affinity.bits,
3696 cpus_mask.bits, thread_mask.affinity.nbits)) {
3697 pr_err("Empty affinity mask: %s\n", affinity_spec[s]);
3698 ret = -EINVAL;
3699 goto out_free;
3700 }
3701
3702 /* do not allow intersection with other masks (full_mask) */
3703 if (bitmap_intersects(thread_mask.maps.bits, full_mask.maps.bits,
3704 thread_mask.maps.nbits)) {
3705 pr_err("Intersecting maps mask: %s\n", maps_spec[s]);
3706 ret = -EINVAL;
3707 goto out_free;
3708 }
3709 if (bitmap_intersects(thread_mask.affinity.bits, full_mask.affinity.bits,
3710 thread_mask.affinity.nbits)) {
3711 pr_err("Intersecting affinity mask: %s\n", affinity_spec[s]);
3712 ret = -EINVAL;
3713 goto out_free;
3714 }
3715
3716 bitmap_or(full_mask.maps.bits, full_mask.maps.bits,
3717 thread_mask.maps.bits, full_mask.maps.nbits);
3718 bitmap_or(full_mask.affinity.bits, full_mask.affinity.bits,
3719 thread_mask.affinity.bits, full_mask.maps.nbits);
3720
3721 thread_masks = realloc(rec->thread_masks, (t + 1) * sizeof(struct thread_mask));
3722 if (!thread_masks) {
3723 pr_err("Failed to reallocate thread masks\n");
3724 ret = -ENOMEM;
3725 goto out_free;
3726 }
3727 rec->thread_masks = thread_masks;
3728 rec->thread_masks[t] = thread_mask;
3729 if (verbose > 0) {
3730 pr_debug("thread_masks[%d]: ", t);
3731 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3732 pr_debug("thread_masks[%d]: ", t);
3733 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3734 }
3735 t++;
3736 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3737 if (ret) {
3738 pr_err("Failed to allocate thread mask\n");
3739 goto out_free_full_and_cpu_masks;
3740 }
3741 }
3742 rec->nr_threads = t;
3743 pr_debug("nr_threads: %d\n", rec->nr_threads);
3744 if (!rec->nr_threads)
3745 ret = -EINVAL;
3746
3747out_free:
3748 record__thread_mask_free(&thread_mask);
3749out_free_full_and_cpu_masks:
3750 record__thread_mask_free(&full_mask);
3751out_free_cpu_mask:
3752 record__mmap_cpu_mask_free(&cpus_mask);
3753
3754 return ret;
3755}
3756
3757static int record__init_thread_core_masks(struct record *rec, struct perf_cpu_map *cpus)
3758{
3759 int ret;
3760 struct cpu_topology *topo;
3761
3762 topo = cpu_topology__new();
3763 if (!topo) {
3764 pr_err("Failed to allocate CPU topology\n");
3765 return -ENOMEM;
3766 }
3767
3768 ret = record__init_thread_masks_spec(rec, cpus, topo->core_cpus_list,
3769 topo->core_cpus_list, topo->core_cpus_lists);
3770 cpu_topology__delete(topo);
3771
3772 return ret;
3773}
3774
3775static int record__init_thread_package_masks(struct record *rec, struct perf_cpu_map *cpus)
3776{
3777 int ret;
3778 struct cpu_topology *topo;
3779
3780 topo = cpu_topology__new();
3781 if (!topo) {
3782 pr_err("Failed to allocate CPU topology\n");
3783 return -ENOMEM;
3784 }
3785
3786 ret = record__init_thread_masks_spec(rec, cpus, topo->package_cpus_list,
3787 topo->package_cpus_list, topo->package_cpus_lists);
3788 cpu_topology__delete(topo);
3789
3790 return ret;
3791}
3792
3793static int record__init_thread_numa_masks(struct record *rec, struct perf_cpu_map *cpus)
3794{
3795 u32 s;
3796 int ret;
3797 const char **spec;
3798 struct numa_topology *topo;
3799
3800 topo = numa_topology__new();
3801 if (!topo) {
3802 pr_err("Failed to allocate NUMA topology\n");
3803 return -ENOMEM;
3804 }
3805
3806 spec = zalloc(topo->nr * sizeof(char *));
3807 if (!spec) {
3808 pr_err("Failed to allocate NUMA spec\n");
3809 ret = -ENOMEM;
3810 goto out_delete_topo;
3811 }
3812 for (s = 0; s < topo->nr; s++)
3813 spec[s] = topo->nodes[s].cpus;
3814
3815 ret = record__init_thread_masks_spec(rec, cpus, spec, spec, topo->nr);
3816
3817 zfree(&spec);
3818
3819out_delete_topo:
3820 numa_topology__delete(topo);
3821
3822 return ret;
3823}
3824
3825static int record__init_thread_user_masks(struct record *rec, struct perf_cpu_map *cpus)
3826{
3827 int t, ret;
3828 u32 s, nr_spec = 0;
3829 char **maps_spec = NULL, **affinity_spec = NULL, **tmp_spec;
3830 char *user_spec, *spec, *spec_ptr, *mask, *mask_ptr, *dup_mask = NULL;
3831
3832 for (t = 0, user_spec = (char *)rec->opts.threads_user_spec; ; t++, user_spec = NULL) {
3833 spec = strtok_r(user_spec, ":", &spec_ptr);
3834 if (spec == NULL)
3835 break;
3836 pr_debug2("threads_spec[%d]: %s\n", t, spec);
3837 mask = strtok_r(spec, "/", &mask_ptr);
3838 if (mask == NULL)
3839 break;
3840 pr_debug2(" maps mask: %s\n", mask);
3841 tmp_spec = realloc(maps_spec, (nr_spec + 1) * sizeof(char *));
3842 if (!tmp_spec) {
3843 pr_err("Failed to reallocate maps spec\n");
3844 ret = -ENOMEM;
3845 goto out_free;
3846 }
3847 maps_spec = tmp_spec;
3848 maps_spec[nr_spec] = dup_mask = strdup(mask);
3849 if (!maps_spec[nr_spec]) {
3850 pr_err("Failed to allocate maps spec[%d]\n", nr_spec);
3851 ret = -ENOMEM;
3852 goto out_free;
3853 }
3854 mask = strtok_r(NULL, "/", &mask_ptr);
3855 if (mask == NULL) {
3856 pr_err("Invalid thread maps or affinity specs\n");
3857 ret = -EINVAL;
3858 goto out_free;
3859 }
3860 pr_debug2(" affinity mask: %s\n", mask);
3861 tmp_spec = realloc(affinity_spec, (nr_spec + 1) * sizeof(char *));
3862 if (!tmp_spec) {
3863 pr_err("Failed to reallocate affinity spec\n");
3864 ret = -ENOMEM;
3865 goto out_free;
3866 }
3867 affinity_spec = tmp_spec;
3868 affinity_spec[nr_spec] = strdup(mask);
3869 if (!affinity_spec[nr_spec]) {
3870 pr_err("Failed to allocate affinity spec[%d]\n", nr_spec);
3871 ret = -ENOMEM;
3872 goto out_free;
3873 }
3874 dup_mask = NULL;
3875 nr_spec++;
3876 }
3877
3878 ret = record__init_thread_masks_spec(rec, cpus, (const char **)maps_spec,
3879 (const char **)affinity_spec, nr_spec);
3880
3881out_free:
3882 free(dup_mask);
3883 for (s = 0; s < nr_spec; s++) {
3884 if (maps_spec)
3885 free(maps_spec[s]);
3886 if (affinity_spec)
3887 free(affinity_spec[s]);
3888 }
3889 free(affinity_spec);
3890 free(maps_spec);
3891
3892 return ret;
3893}
3894
3895static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
3896{
3897 int ret;
3898
3899 ret = record__alloc_thread_masks(rec, 1, cpu__max_cpu().cpu);
3900 if (ret)
3901 return ret;
3902
3903 if (record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus))
3904 return -ENODEV;
3905
3906 rec->nr_threads = 1;
3907
3908 return 0;
3909}
3910
3911static int record__init_thread_masks(struct record *rec)
3912{
3913 int ret = 0;
3914 struct perf_cpu_map *cpus = rec->evlist->core.all_cpus;
3915
3916 if (!record__threads_enabled(rec))
3917 return record__init_thread_default_masks(rec, cpus);
3918
3919 if (evlist__per_thread(rec->evlist)) {
3920 pr_err("--per-thread option is mutually exclusive to parallel streaming mode.\n");
3921 return -EINVAL;
3922 }
3923
3924 switch (rec->opts.threads_spec) {
3925 case THREAD_SPEC__CPU:
3926 ret = record__init_thread_cpu_masks(rec, cpus);
3927 break;
3928 case THREAD_SPEC__CORE:
3929 ret = record__init_thread_core_masks(rec, cpus);
3930 break;
3931 case THREAD_SPEC__PACKAGE:
3932 ret = record__init_thread_package_masks(rec, cpus);
3933 break;
3934 case THREAD_SPEC__NUMA:
3935 ret = record__init_thread_numa_masks(rec, cpus);
3936 break;
3937 case THREAD_SPEC__USER:
3938 ret = record__init_thread_user_masks(rec, cpus);
3939 break;
3940 default:
3941 break;
3942 }
3943
3944 return ret;
3945}
3946
3947int cmd_record(int argc, const char **argv)
3948{
3949 int err;
3950 struct record *rec = &record;
3951 char errbuf[BUFSIZ];
3952
3953 setlocale(LC_ALL, "");
3954
3955#ifndef HAVE_LIBBPF_SUPPORT
3956# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, "NO_LIBBPF=1", c)
3957 set_nobuild('\0', "clang-path", true);
3958 set_nobuild('\0', "clang-opt", true);
3959# undef set_nobuild
3960#endif
3961
3962#ifndef HAVE_BPF_PROLOGUE
3963# if !defined (HAVE_DWARF_SUPPORT)
3964# define REASON "NO_DWARF=1"
3965# elif !defined (HAVE_LIBBPF_SUPPORT)
3966# define REASON "NO_LIBBPF=1"
3967# else
3968# define REASON "this architecture doesn't support BPF prologue"
3969# endif
3970# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, REASON, c)
3971 set_nobuild('\0', "vmlinux", true);
3972# undef set_nobuild
3973# undef REASON
3974#endif
3975
3976#ifndef HAVE_BPF_SKEL
3977# define set_nobuild(s, l, m, c) set_option_nobuild(record_options, s, l, m, c)
3978 set_nobuild('\0', "off-cpu", "no BUILD_BPF_SKEL=1", true);
3979# undef set_nobuild
3980#endif
3981
3982 rec->opts.affinity = PERF_AFFINITY_SYS;
3983
3984 rec->evlist = evlist__new();
3985 if (rec->evlist == NULL)
3986 return -ENOMEM;
3987
3988 err = perf_config(perf_record_config, rec);
3989 if (err)
3990 return err;
3991
3992 argc = parse_options(argc, argv, record_options, record_usage,
3993 PARSE_OPT_STOP_AT_NON_OPTION);
3994 if (quiet)
3995 perf_quiet_option();
3996
3997 err = symbol__validate_sym_arguments();
3998 if (err)
3999 return err;
4000
4001 perf_debuginfod_setup(&record.debuginfod);
4002
4003 /* Make system wide (-a) the default target. */
4004 if (!argc && target__none(&rec->opts.target))
4005 rec->opts.target.system_wide = true;
4006
4007 if (nr_cgroups && !rec->opts.target.system_wide) {
4008 usage_with_options_msg(record_usage, record_options,
4009 "cgroup monitoring only available in system-wide mode");
4010
4011 }
4012
4013 if (rec->buildid_mmap) {
4014 if (!perf_can_record_build_id()) {
4015 pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
4016 err = -EINVAL;
4017 goto out_opts;
4018 }
4019 pr_debug("Enabling build id in mmap2 events.\n");
4020 /* Enable mmap build id synthesizing. */
4021 symbol_conf.buildid_mmap2 = true;
4022 /* Enable perf_event_attr::build_id bit. */
4023 rec->opts.build_id = true;
4024 /* Disable build id cache. */
4025 rec->no_buildid = true;
4026 }
4027
4028 if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
4029 pr_err("Kernel has no cgroup sampling support.\n");
4030 err = -EINVAL;
4031 goto out_opts;
4032 }
4033
4034 if (rec->opts.kcore)
4035 rec->opts.text_poke = true;
4036
4037 if (rec->opts.kcore || record__threads_enabled(rec))
4038 rec->data.is_dir = true;
4039
4040 if (record__threads_enabled(rec)) {
4041 if (rec->opts.affinity != PERF_AFFINITY_SYS) {
4042 pr_err("--affinity option is mutually exclusive to parallel streaming mode.\n");
4043 goto out_opts;
4044 }
4045 if (record__aio_enabled(rec)) {
4046 pr_err("Asynchronous streaming mode (--aio) is mutually exclusive to parallel streaming mode.\n");
4047 goto out_opts;
4048 }
4049 }
4050
4051 if (rec->opts.comp_level != 0) {
4052 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
4053 rec->no_buildid = true;
4054 }
4055
4056 if (rec->opts.record_switch_events &&
4057 !perf_can_record_switch_events()) {
4058 ui__error("kernel does not support recording context switch events\n");
4059 parse_options_usage(record_usage, record_options, "switch-events", 0);
4060 err = -EINVAL;
4061 goto out_opts;
4062 }
4063
4064 if (switch_output_setup(rec)) {
4065 parse_options_usage(record_usage, record_options, "switch-output", 0);
4066 err = -EINVAL;
4067 goto out_opts;
4068 }
4069
4070 if (rec->switch_output.time) {
4071 signal(SIGALRM, alarm_sig_handler);
4072 alarm(rec->switch_output.time);
4073 }
4074
4075 if (rec->switch_output.num_files) {
4076 rec->switch_output.filenames = calloc(sizeof(char *),
4077 rec->switch_output.num_files);
4078 if (!rec->switch_output.filenames) {
4079 err = -EINVAL;
4080 goto out_opts;
4081 }
4082 }
4083
4084 if (rec->timestamp_filename && record__threads_enabled(rec)) {
4085 rec->timestamp_filename = false;
4086 pr_warning("WARNING: --timestamp-filename option is not available in parallel streaming mode.\n");
4087 }
4088
4089 /*
4090 * Allow aliases to facilitate the lookup of symbols for address
4091 * filters. Refer to auxtrace_parse_filters().
4092 */
4093 symbol_conf.allow_aliases = true;
4094
4095 symbol__init(NULL);
4096
4097 err = record__auxtrace_init(rec);
4098 if (err)
4099 goto out;
4100
4101 if (dry_run)
4102 goto out;
4103
4104 err = bpf__setup_stdout(rec->evlist);
4105 if (err) {
4106 bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
4107 pr_err("ERROR: Setup BPF stdout failed: %s\n",
4108 errbuf);
4109 goto out;
4110 }
4111
4112 err = -ENOMEM;
4113
4114 if (rec->no_buildid_cache || rec->no_buildid) {
4115 disable_buildid_cache();
4116 } else if (rec->switch_output.enabled) {
4117 /*
4118 * In 'perf record --switch-output', disable buildid
4119 * generation by default to reduce data file switching
4120 * overhead. Still generate buildid if they are required
4121 * explicitly using
4122 *
4123 * perf record --switch-output --no-no-buildid \
4124 * --no-no-buildid-cache
4125 *
4126 * Following code equals to:
4127 *
4128 * if ((rec->no_buildid || !rec->no_buildid_set) &&
4129 * (rec->no_buildid_cache || !rec->no_buildid_cache_set))
4130 * disable_buildid_cache();
4131 */
4132 bool disable = true;
4133
4134 if (rec->no_buildid_set && !rec->no_buildid)
4135 disable = false;
4136 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
4137 disable = false;
4138 if (disable) {
4139 rec->no_buildid = true;
4140 rec->no_buildid_cache = true;
4141 disable_buildid_cache();
4142 }
4143 }
4144
4145 if (record.opts.overwrite)
4146 record.opts.tail_synthesize = true;
4147
4148 if (rec->evlist->core.nr_entries == 0) {
4149 if (perf_pmu__has_hybrid()) {
4150 err = evlist__add_default_hybrid(rec->evlist,
4151 !record.opts.no_samples);
4152 } else {
4153 err = __evlist__add_default(rec->evlist,
4154 !record.opts.no_samples);
4155 }
4156
4157 if (err < 0) {
4158 pr_err("Not enough memory for event selector list\n");
4159 goto out;
4160 }
4161 }
4162
4163 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
4164 rec->opts.no_inherit = true;
4165
4166 err = target__validate(&rec->opts.target);
4167 if (err) {
4168 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
4169 ui__warning("%s\n", errbuf);
4170 }
4171
4172 err = target__parse_uid(&rec->opts.target);
4173 if (err) {
4174 int saved_errno = errno;
4175
4176 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
4177 ui__error("%s", errbuf);
4178
4179 err = -saved_errno;
4180 goto out;
4181 }
4182
4183 /* Enable ignoring missing threads when -u/-p option is defined. */
4184 rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
4185
4186 if (evlist__fix_hybrid_cpus(rec->evlist, rec->opts.target.cpu_list)) {
4187 pr_err("failed to use cpu list %s\n",
4188 rec->opts.target.cpu_list);
4189 goto out;
4190 }
4191
4192 rec->opts.target.hybrid = perf_pmu__has_hybrid();
4193
4194 if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP)
4195 arch__add_leaf_frame_record_opts(&rec->opts);
4196
4197 err = -ENOMEM;
4198 if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0) {
4199 if (rec->opts.target.pid != NULL) {
4200 pr_err("Couldn't create thread/CPU maps: %s\n",
4201 errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
4202 goto out;
4203 }
4204 else
4205 usage_with_options(record_usage, record_options);
4206 }
4207
4208 err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
4209 if (err)
4210 goto out;
4211
4212 /*
4213 * We take all buildids when the file contains
4214 * AUX area tracing data because we do not decode the
4215 * trace because it would take too long.
4216 */
4217 if (rec->opts.full_auxtrace)
4218 rec->buildid_all = true;
4219
4220 if (rec->opts.text_poke) {
4221 err = record__config_text_poke(rec->evlist);
4222 if (err) {
4223 pr_err("record__config_text_poke failed, error %d\n", err);
4224 goto out;
4225 }
4226 }
4227
4228 if (rec->off_cpu) {
4229 err = record__config_off_cpu(rec);
4230 if (err) {
4231 pr_err("record__config_off_cpu failed, error %d\n", err);
4232 goto out;
4233 }
4234 }
4235
4236 if (record_opts__config(&rec->opts)) {
4237 err = -EINVAL;
4238 goto out;
4239 }
4240
4241 err = record__init_thread_masks(rec);
4242 if (err) {
4243 pr_err("Failed to initialize parallel data streaming masks\n");
4244 goto out;
4245 }
4246
4247 if (rec->opts.nr_cblocks > nr_cblocks_max)
4248 rec->opts.nr_cblocks = nr_cblocks_max;
4249 pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
4250
4251 pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
4252 pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
4253
4254 if (rec->opts.comp_level > comp_level_max)
4255 rec->opts.comp_level = comp_level_max;
4256 pr_debug("comp level: %d\n", rec->opts.comp_level);
4257
4258 err = __cmd_record(&record, argc, argv);
4259out:
4260 evlist__delete(rec->evlist);
4261 symbol__exit();
4262 auxtrace_record__free(rec->itr);
4263out_opts:
4264 record__free_thread_masks(rec, rec->nr_threads);
4265 rec->nr_threads = 0;
4266 evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
4267 return err;
4268}
4269
4270static void snapshot_sig_handler(int sig __maybe_unused)
4271{
4272 struct record *rec = &record;
4273
4274 hit_auxtrace_snapshot_trigger(rec);
4275
4276 if (switch_output_signal(rec))
4277 trigger_hit(&switch_output_trigger);
4278}
4279
4280static void alarm_sig_handler(int sig __maybe_unused)
4281{
4282 struct record *rec = &record;
4283
4284 if (switch_output_time(rec))
4285 trigger_hit(&switch_output_trigger);
4286}