Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2019 Facebook */
4
5#include <assert.h>
6#include <limits.h>
7#include <unistd.h>
8#include <sys/file.h>
9#include <sys/time.h>
10#include <linux/err.h>
11#include <linux/zalloc.h>
12#include <api/fs/fs.h>
13#include <perf/bpf_perf.h>
14
15#include "bpf_counter.h"
16#include "counts.h"
17#include "debug.h"
18#include "evsel.h"
19#include "evlist.h"
20#include "target.h"
21#include "cgroup.h"
22#include "cpumap.h"
23#include "thread_map.h"
24
25#include "bpf_skel/bpf_prog_profiler.skel.h"
26#include "bpf_skel/bperf_u.h"
27#include "bpf_skel/bperf_leader.skel.h"
28#include "bpf_skel/bperf_follower.skel.h"
29
30#define ATTR_MAP_SIZE 16
31
32static inline void *u64_to_ptr(__u64 ptr)
33{
34 return (void *)(unsigned long)ptr;
35}
36
37static struct bpf_counter *bpf_counter_alloc(void)
38{
39 struct bpf_counter *counter;
40
41 counter = zalloc(sizeof(*counter));
42 if (counter)
43 INIT_LIST_HEAD(&counter->list);
44 return counter;
45}
46
47static int bpf_program_profiler__destroy(struct evsel *evsel)
48{
49 struct bpf_counter *counter, *tmp;
50
51 list_for_each_entry_safe(counter, tmp,
52 &evsel->bpf_counter_list, list) {
53 list_del_init(&counter->list);
54 bpf_prog_profiler_bpf__destroy(counter->skel);
55 free(counter);
56 }
57 assert(list_empty(&evsel->bpf_counter_list));
58
59 return 0;
60}
61
62static char *bpf_target_prog_name(int tgt_fd)
63{
64 struct bpf_prog_info_linear *info_linear;
65 struct bpf_func_info *func_info;
66 const struct btf_type *t;
67 struct btf *btf = NULL;
68 char *name = NULL;
69
70 info_linear = bpf_program__get_prog_info_linear(
71 tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO);
72 if (IS_ERR_OR_NULL(info_linear)) {
73 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd);
74 return NULL;
75 }
76
77 if (info_linear->info.btf_id == 0 ||
78 btf__get_from_id(info_linear->info.btf_id, &btf)) {
79 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd);
80 goto out;
81 }
82
83 func_info = u64_to_ptr(info_linear->info.func_info);
84 t = btf__type_by_id(btf, func_info[0].type_id);
85 if (!t) {
86 pr_debug("btf %d doesn't have type %d\n",
87 info_linear->info.btf_id, func_info[0].type_id);
88 goto out;
89 }
90 name = strdup(btf__name_by_offset(btf, t->name_off));
91out:
92 btf__free(btf);
93 free(info_linear);
94 return name;
95}
96
97static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id)
98{
99 struct bpf_prog_profiler_bpf *skel;
100 struct bpf_counter *counter;
101 struct bpf_program *prog;
102 char *prog_name;
103 int prog_fd;
104 int err;
105
106 prog_fd = bpf_prog_get_fd_by_id(prog_id);
107 if (prog_fd < 0) {
108 pr_err("Failed to open fd for bpf prog %u\n", prog_id);
109 return -1;
110 }
111 counter = bpf_counter_alloc();
112 if (!counter) {
113 close(prog_fd);
114 return -1;
115 }
116
117 skel = bpf_prog_profiler_bpf__open();
118 if (!skel) {
119 pr_err("Failed to open bpf skeleton\n");
120 goto err_out;
121 }
122
123 skel->rodata->num_cpu = evsel__nr_cpus(evsel);
124
125 bpf_map__resize(skel->maps.events, evsel__nr_cpus(evsel));
126 bpf_map__resize(skel->maps.fentry_readings, 1);
127 bpf_map__resize(skel->maps.accum_readings, 1);
128
129 prog_name = bpf_target_prog_name(prog_fd);
130 if (!prog_name) {
131 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id);
132 goto err_out;
133 }
134
135 bpf_object__for_each_program(prog, skel->obj) {
136 err = bpf_program__set_attach_target(prog, prog_fd, prog_name);
137 if (err) {
138 pr_err("bpf_program__set_attach_target failed.\n"
139 "Does bpf prog %u have BTF?\n", prog_id);
140 goto err_out;
141 }
142 }
143 set_max_rlimit();
144 err = bpf_prog_profiler_bpf__load(skel);
145 if (err) {
146 pr_err("bpf_prog_profiler_bpf__load failed\n");
147 goto err_out;
148 }
149
150 assert(skel != NULL);
151 counter->skel = skel;
152 list_add(&counter->list, &evsel->bpf_counter_list);
153 close(prog_fd);
154 return 0;
155err_out:
156 bpf_prog_profiler_bpf__destroy(skel);
157 free(counter);
158 close(prog_fd);
159 return -1;
160}
161
162static int bpf_program_profiler__load(struct evsel *evsel, struct target *target)
163{
164 char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p;
165 u32 prog_id;
166 int ret;
167
168 bpf_str_ = bpf_str = strdup(target->bpf_str);
169 if (!bpf_str)
170 return -1;
171
172 while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) {
173 prog_id = strtoul(tok, &p, 10);
174 if (prog_id == 0 || prog_id == UINT_MAX ||
175 (*p != '\0' && *p != ',')) {
176 pr_err("Failed to parse bpf prog ids %s\n",
177 target->bpf_str);
178 return -1;
179 }
180
181 ret = bpf_program_profiler_load_one(evsel, prog_id);
182 if (ret) {
183 bpf_program_profiler__destroy(evsel);
184 free(bpf_str_);
185 return -1;
186 }
187 bpf_str = NULL;
188 }
189 free(bpf_str_);
190 return 0;
191}
192
193static int bpf_program_profiler__enable(struct evsel *evsel)
194{
195 struct bpf_counter *counter;
196 int ret;
197
198 list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
199 assert(counter->skel != NULL);
200 ret = bpf_prog_profiler_bpf__attach(counter->skel);
201 if (ret) {
202 bpf_program_profiler__destroy(evsel);
203 return ret;
204 }
205 }
206 return 0;
207}
208
209static int bpf_program_profiler__disable(struct evsel *evsel)
210{
211 struct bpf_counter *counter;
212
213 list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
214 assert(counter->skel != NULL);
215 bpf_prog_profiler_bpf__detach(counter->skel);
216 }
217 return 0;
218}
219
220static int bpf_program_profiler__read(struct evsel *evsel)
221{
222 // perf_cpu_map uses /sys/devices/system/cpu/online
223 int num_cpu = evsel__nr_cpus(evsel);
224 // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible
225 // Sometimes possible > online, like on a Ryzen 3900X that has 24
226 // threads but its possible showed 0-31 -acme
227 int num_cpu_bpf = libbpf_num_possible_cpus();
228 struct bpf_perf_event_value values[num_cpu_bpf];
229 struct bpf_counter *counter;
230 int reading_map_fd;
231 __u32 key = 0;
232 int err, cpu;
233
234 if (list_empty(&evsel->bpf_counter_list))
235 return -EAGAIN;
236
237 for (cpu = 0; cpu < num_cpu; cpu++) {
238 perf_counts(evsel->counts, cpu, 0)->val = 0;
239 perf_counts(evsel->counts, cpu, 0)->ena = 0;
240 perf_counts(evsel->counts, cpu, 0)->run = 0;
241 }
242 list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
243 struct bpf_prog_profiler_bpf *skel = counter->skel;
244
245 assert(skel != NULL);
246 reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
247
248 err = bpf_map_lookup_elem(reading_map_fd, &key, values);
249 if (err) {
250 pr_err("failed to read value\n");
251 return err;
252 }
253
254 for (cpu = 0; cpu < num_cpu; cpu++) {
255 perf_counts(evsel->counts, cpu, 0)->val += values[cpu].counter;
256 perf_counts(evsel->counts, cpu, 0)->ena += values[cpu].enabled;
257 perf_counts(evsel->counts, cpu, 0)->run += values[cpu].running;
258 }
259 }
260 return 0;
261}
262
263static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu,
264 int fd)
265{
266 struct bpf_prog_profiler_bpf *skel;
267 struct bpf_counter *counter;
268 int ret;
269
270 list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
271 skel = counter->skel;
272 assert(skel != NULL);
273
274 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events),
275 &cpu, &fd, BPF_ANY);
276 if (ret)
277 return ret;
278 }
279 return 0;
280}
281
282struct bpf_counter_ops bpf_program_profiler_ops = {
283 .load = bpf_program_profiler__load,
284 .enable = bpf_program_profiler__enable,
285 .disable = bpf_program_profiler__disable,
286 .read = bpf_program_profiler__read,
287 .destroy = bpf_program_profiler__destroy,
288 .install_pe = bpf_program_profiler__install_pe,
289};
290
291static bool bperf_attr_map_compatible(int attr_map_fd)
292{
293 struct bpf_map_info map_info = {0};
294 __u32 map_info_len = sizeof(map_info);
295 int err;
296
297 err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len);
298
299 if (err)
300 return false;
301 return (map_info.key_size == sizeof(struct perf_event_attr)) &&
302 (map_info.value_size == sizeof(struct perf_event_attr_map_entry));
303}
304
305static int bperf_lock_attr_map(struct target *target)
306{
307 char path[PATH_MAX];
308 int map_fd, err;
309
310 if (target->attr_map) {
311 scnprintf(path, PATH_MAX, "%s", target->attr_map);
312 } else {
313 scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(),
314 BPF_PERF_DEFAULT_ATTR_MAP_PATH);
315 }
316
317 if (access(path, F_OK)) {
318 map_fd = bpf_create_map(BPF_MAP_TYPE_HASH,
319 sizeof(struct perf_event_attr),
320 sizeof(struct perf_event_attr_map_entry),
321 ATTR_MAP_SIZE, 0);
322 if (map_fd < 0)
323 return -1;
324
325 err = bpf_obj_pin(map_fd, path);
326 if (err) {
327 /* someone pinned the map in parallel? */
328 close(map_fd);
329 map_fd = bpf_obj_get(path);
330 if (map_fd < 0)
331 return -1;
332 }
333 } else {
334 map_fd = bpf_obj_get(path);
335 if (map_fd < 0)
336 return -1;
337 }
338
339 if (!bperf_attr_map_compatible(map_fd)) {
340 close(map_fd);
341 return -1;
342
343 }
344 err = flock(map_fd, LOCK_EX);
345 if (err) {
346 close(map_fd);
347 return -1;
348 }
349 return map_fd;
350}
351
352static int bperf_check_target(struct evsel *evsel,
353 struct target *target,
354 enum bperf_filter_type *filter_type,
355 __u32 *filter_entry_cnt)
356{
357 if (evsel->core.leader->nr_members > 1) {
358 pr_err("bpf managed perf events do not yet support groups.\n");
359 return -1;
360 }
361
362 /* determine filter type based on target */
363 if (target->system_wide) {
364 *filter_type = BPERF_FILTER_GLOBAL;
365 *filter_entry_cnt = 1;
366 } else if (target->cpu_list) {
367 *filter_type = BPERF_FILTER_CPU;
368 *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel));
369 } else if (target->tid) {
370 *filter_type = BPERF_FILTER_PID;
371 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
372 } else if (target->pid || evsel->evlist->workload.pid != -1) {
373 *filter_type = BPERF_FILTER_TGID;
374 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
375 } else {
376 pr_err("bpf managed perf events do not yet support these targets.\n");
377 return -1;
378 }
379
380 return 0;
381}
382
383static struct perf_cpu_map *all_cpu_map;
384
385static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd,
386 struct perf_event_attr_map_entry *entry)
387{
388 struct bperf_leader_bpf *skel = bperf_leader_bpf__open();
389 int link_fd, diff_map_fd, err;
390 struct bpf_link *link = NULL;
391
392 if (!skel) {
393 pr_err("Failed to open leader skeleton\n");
394 return -1;
395 }
396
397 bpf_map__resize(skel->maps.events, libbpf_num_possible_cpus());
398 err = bperf_leader_bpf__load(skel);
399 if (err) {
400 pr_err("Failed to load leader skeleton\n");
401 goto out;
402 }
403
404 link = bpf_program__attach(skel->progs.on_switch);
405 if (IS_ERR(link)) {
406 pr_err("Failed to attach leader program\n");
407 err = PTR_ERR(link);
408 goto out;
409 }
410
411 link_fd = bpf_link__fd(link);
412 diff_map_fd = bpf_map__fd(skel->maps.diff_readings);
413 entry->link_id = bpf_link_get_id(link_fd);
414 entry->diff_map_id = bpf_map_get_id(diff_map_fd);
415 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY);
416 assert(err == 0);
417
418 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id);
419 assert(evsel->bperf_leader_link_fd >= 0);
420
421 /*
422 * save leader_skel for install_pe, which is called within
423 * following evsel__open_per_cpu call
424 */
425 evsel->leader_skel = skel;
426 evsel__open_per_cpu(evsel, all_cpu_map, -1);
427
428out:
429 bperf_leader_bpf__destroy(skel);
430 bpf_link__destroy(link);
431 return err;
432}
433
434static int bperf__load(struct evsel *evsel, struct target *target)
435{
436 struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff};
437 int attr_map_fd, diff_map_fd = -1, err;
438 enum bperf_filter_type filter_type;
439 __u32 filter_entry_cnt, i;
440
441 if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt))
442 return -1;
443
444 if (!all_cpu_map) {
445 all_cpu_map = perf_cpu_map__new(NULL);
446 if (!all_cpu_map)
447 return -1;
448 }
449
450 evsel->bperf_leader_prog_fd = -1;
451 evsel->bperf_leader_link_fd = -1;
452
453 /*
454 * Step 1: hold a fd on the leader program and the bpf_link, if
455 * the program is not already gone, reload the program.
456 * Use flock() to ensure exclusive access to the perf_event_attr
457 * map.
458 */
459 attr_map_fd = bperf_lock_attr_map(target);
460 if (attr_map_fd < 0) {
461 pr_err("Failed to lock perf_event_attr map\n");
462 return -1;
463 }
464
465 err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry);
466 if (err) {
467 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY);
468 if (err)
469 goto out;
470 }
471
472 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id);
473 if (evsel->bperf_leader_link_fd < 0 &&
474 bperf_reload_leader_program(evsel, attr_map_fd, &entry)) {
475 err = -1;
476 goto out;
477 }
478 /*
479 * The bpf_link holds reference to the leader program, and the
480 * leader program holds reference to the maps. Therefore, if
481 * link_id is valid, diff_map_id should also be valid.
482 */
483 evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id(
484 bpf_link_get_prog_id(evsel->bperf_leader_link_fd));
485 assert(evsel->bperf_leader_prog_fd >= 0);
486
487 diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id);
488 assert(diff_map_fd >= 0);
489
490 /*
491 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
492 * whether the kernel support it
493 */
494 err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0);
495 if (err) {
496 pr_err("The kernel does not support test_run for raw_tp BPF programs.\n"
497 "Therefore, --use-bpf might show inaccurate readings\n");
498 goto out;
499 }
500
501 /* Step 2: load the follower skeleton */
502 evsel->follower_skel = bperf_follower_bpf__open();
503 if (!evsel->follower_skel) {
504 err = -1;
505 pr_err("Failed to open follower skeleton\n");
506 goto out;
507 }
508
509 /* attach fexit program to the leader program */
510 bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX,
511 evsel->bperf_leader_prog_fd, "on_switch");
512
513 /* connect to leader diff_reading map */
514 bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd);
515
516 /* set up reading map */
517 bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings,
518 filter_entry_cnt);
519 /* set up follower filter based on target */
520 bpf_map__set_max_entries(evsel->follower_skel->maps.filter,
521 filter_entry_cnt);
522 err = bperf_follower_bpf__load(evsel->follower_skel);
523 if (err) {
524 pr_err("Failed to load follower skeleton\n");
525 bperf_follower_bpf__destroy(evsel->follower_skel);
526 evsel->follower_skel = NULL;
527 goto out;
528 }
529
530 for (i = 0; i < filter_entry_cnt; i++) {
531 int filter_map_fd;
532 __u32 key;
533
534 if (filter_type == BPERF_FILTER_PID ||
535 filter_type == BPERF_FILTER_TGID)
536 key = evsel->core.threads->map[i].pid;
537 else if (filter_type == BPERF_FILTER_CPU)
538 key = evsel->core.cpus->map[i];
539 else
540 break;
541
542 filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter);
543 bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY);
544 }
545
546 evsel->follower_skel->bss->type = filter_type;
547
548 err = bperf_follower_bpf__attach(evsel->follower_skel);
549
550out:
551 if (err && evsel->bperf_leader_link_fd >= 0)
552 close(evsel->bperf_leader_link_fd);
553 if (err && evsel->bperf_leader_prog_fd >= 0)
554 close(evsel->bperf_leader_prog_fd);
555 if (diff_map_fd >= 0)
556 close(diff_map_fd);
557
558 flock(attr_map_fd, LOCK_UN);
559 close(attr_map_fd);
560
561 return err;
562}
563
564static int bperf__install_pe(struct evsel *evsel, int cpu, int fd)
565{
566 struct bperf_leader_bpf *skel = evsel->leader_skel;
567
568 return bpf_map_update_elem(bpf_map__fd(skel->maps.events),
569 &cpu, &fd, BPF_ANY);
570}
571
572/*
573 * trigger the leader prog on each cpu, so the accum_reading map could get
574 * the latest readings.
575 */
576static int bperf_sync_counters(struct evsel *evsel)
577{
578 int num_cpu, i, cpu;
579
580 num_cpu = all_cpu_map->nr;
581 for (i = 0; i < num_cpu; i++) {
582 cpu = all_cpu_map->map[i];
583 bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu);
584 }
585 return 0;
586}
587
588static int bperf__enable(struct evsel *evsel)
589{
590 evsel->follower_skel->bss->enabled = 1;
591 return 0;
592}
593
594static int bperf__disable(struct evsel *evsel)
595{
596 evsel->follower_skel->bss->enabled = 0;
597 return 0;
598}
599
600static int bperf__read(struct evsel *evsel)
601{
602 struct bperf_follower_bpf *skel = evsel->follower_skel;
603 __u32 num_cpu_bpf = cpu__max_cpu();
604 struct bpf_perf_event_value values[num_cpu_bpf];
605 int reading_map_fd, err = 0;
606 __u32 i, j, num_cpu;
607
608 bperf_sync_counters(evsel);
609 reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
610
611 for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) {
612 __u32 cpu;
613
614 err = bpf_map_lookup_elem(reading_map_fd, &i, values);
615 if (err)
616 goto out;
617 switch (evsel->follower_skel->bss->type) {
618 case BPERF_FILTER_GLOBAL:
619 assert(i == 0);
620
621 num_cpu = all_cpu_map->nr;
622 for (j = 0; j < num_cpu; j++) {
623 cpu = all_cpu_map->map[j];
624 perf_counts(evsel->counts, cpu, 0)->val = values[cpu].counter;
625 perf_counts(evsel->counts, cpu, 0)->ena = values[cpu].enabled;
626 perf_counts(evsel->counts, cpu, 0)->run = values[cpu].running;
627 }
628 break;
629 case BPERF_FILTER_CPU:
630 cpu = evsel->core.cpus->map[i];
631 perf_counts(evsel->counts, i, 0)->val = values[cpu].counter;
632 perf_counts(evsel->counts, i, 0)->ena = values[cpu].enabled;
633 perf_counts(evsel->counts, i, 0)->run = values[cpu].running;
634 break;
635 case BPERF_FILTER_PID:
636 case BPERF_FILTER_TGID:
637 perf_counts(evsel->counts, 0, i)->val = 0;
638 perf_counts(evsel->counts, 0, i)->ena = 0;
639 perf_counts(evsel->counts, 0, i)->run = 0;
640
641 for (cpu = 0; cpu < num_cpu_bpf; cpu++) {
642 perf_counts(evsel->counts, 0, i)->val += values[cpu].counter;
643 perf_counts(evsel->counts, 0, i)->ena += values[cpu].enabled;
644 perf_counts(evsel->counts, 0, i)->run += values[cpu].running;
645 }
646 break;
647 default:
648 break;
649 }
650 }
651out:
652 return err;
653}
654
655static int bperf__destroy(struct evsel *evsel)
656{
657 bperf_follower_bpf__destroy(evsel->follower_skel);
658 close(evsel->bperf_leader_prog_fd);
659 close(evsel->bperf_leader_link_fd);
660 return 0;
661}
662
663/*
664 * bperf: share hardware PMCs with BPF
665 *
666 * perf uses performance monitoring counters (PMC) to monitor system
667 * performance. The PMCs are limited hardware resources. For example,
668 * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu.
669 *
670 * Modern data center systems use these PMCs in many different ways:
671 * system level monitoring, (maybe nested) container level monitoring, per
672 * process monitoring, profiling (in sample mode), etc. In some cases,
673 * there are more active perf_events than available hardware PMCs. To allow
674 * all perf_events to have a chance to run, it is necessary to do expensive
675 * time multiplexing of events.
676 *
677 * On the other hand, many monitoring tools count the common metrics
678 * (cycles, instructions). It is a waste to have multiple tools create
679 * multiple perf_events of "cycles" and occupy multiple PMCs.
680 *
681 * bperf tries to reduce such wastes by allowing multiple perf_events of
682 * "cycles" or "instructions" (at different scopes) to share PMUs. Instead
683 * of having each perf-stat session to read its own perf_events, bperf uses
684 * BPF programs to read the perf_events and aggregate readings to BPF maps.
685 * Then, the perf-stat session(s) reads the values from these BPF maps.
686 *
687 * ||
688 * shared progs and maps <- || -> per session progs and maps
689 * ||
690 * --------------- ||
691 * | perf_events | ||
692 * --------------- fexit || -----------------
693 * | --------||----> | follower prog |
694 * --------------- / || --- -----------------
695 * cs -> | leader prog |/ ||/ | |
696 * --> --------------- /|| -------------- ------------------
697 * / | | / || | filter map | | accum_readings |
698 * / ------------ ------------ || -------------- ------------------
699 * | | prev map | | diff map | || |
700 * | ------------ ------------ || |
701 * \ || |
702 * = \ ==================================================== | ============
703 * \ / user space
704 * \ /
705 * \ /
706 * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM
707 * \ /
708 * \ /
709 * \------ perf-stat ----------------------/
710 *
711 * The figure above shows the architecture of bperf. Note that the figure
712 * is divided into 3 regions: shared progs and maps (top left), per session
713 * progs and maps (top right), and user space (bottom).
714 *
715 * The leader prog is triggered on each context switch (cs). The leader
716 * prog reads perf_events and stores the difference (current_reading -
717 * previous_reading) to the diff map. For the same metric, e.g. "cycles",
718 * multiple perf-stat sessions share the same leader prog.
719 *
720 * Each perf-stat session creates a follower prog as fexit program to the
721 * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38)
722 * follower progs to the same leader prog. The follower prog checks current
723 * task and processor ID to decide whether to add the value from the diff
724 * map to its accumulated reading map (accum_readings).
725 *
726 * Finally, perf-stat user space reads the value from accum_reading map.
727 *
728 * Besides context switch, it is also necessary to trigger the leader prog
729 * before perf-stat reads the value. Otherwise, the accum_reading map may
730 * not have the latest reading from the perf_events. This is achieved by
731 * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU.
732 *
733 * Comment before the definition of struct perf_event_attr_map_entry
734 * describes how different sessions of perf-stat share information about
735 * the leader prog.
736 */
737
738struct bpf_counter_ops bperf_ops = {
739 .load = bperf__load,
740 .enable = bperf__enable,
741 .disable = bperf__disable,
742 .read = bperf__read,
743 .install_pe = bperf__install_pe,
744 .destroy = bperf__destroy,
745};
746
747extern struct bpf_counter_ops bperf_cgrp_ops;
748
749static inline bool bpf_counter_skip(struct evsel *evsel)
750{
751 return list_empty(&evsel->bpf_counter_list) &&
752 evsel->follower_skel == NULL;
753}
754
755int bpf_counter__install_pe(struct evsel *evsel, int cpu, int fd)
756{
757 if (bpf_counter_skip(evsel))
758 return 0;
759 return evsel->bpf_counter_ops->install_pe(evsel, cpu, fd);
760}
761
762int bpf_counter__load(struct evsel *evsel, struct target *target)
763{
764 if (target->bpf_str)
765 evsel->bpf_counter_ops = &bpf_program_profiler_ops;
766 else if (cgrp_event_expanded && target->use_bpf)
767 evsel->bpf_counter_ops = &bperf_cgrp_ops;
768 else if (target->use_bpf || evsel->bpf_counter ||
769 evsel__match_bpf_counter_events(evsel->name))
770 evsel->bpf_counter_ops = &bperf_ops;
771
772 if (evsel->bpf_counter_ops)
773 return evsel->bpf_counter_ops->load(evsel, target);
774 return 0;
775}
776
777int bpf_counter__enable(struct evsel *evsel)
778{
779 if (bpf_counter_skip(evsel))
780 return 0;
781 return evsel->bpf_counter_ops->enable(evsel);
782}
783
784int bpf_counter__disable(struct evsel *evsel)
785{
786 if (bpf_counter_skip(evsel))
787 return 0;
788 return evsel->bpf_counter_ops->disable(evsel);
789}
790
791int bpf_counter__read(struct evsel *evsel)
792{
793 if (bpf_counter_skip(evsel))
794 return -EAGAIN;
795 return evsel->bpf_counter_ops->read(evsel);
796}
797
798void bpf_counter__destroy(struct evsel *evsel)
799{
800 if (bpf_counter_skip(evsel))
801 return;
802 evsel->bpf_counter_ops->destroy(evsel);
803 evsel->bpf_counter_ops = NULL;
804}