1// SPDX-License-Identifier: GPL-2.0
  2
  3/* Copyright (c) 2019 Facebook */
  4
  5#include <assert.h>
  6#include <limits.h>
  7#include <unistd.h>
  8#include <sys/file.h>
  9#include <sys/time.h>
 10#include <linux/err.h>
 11#include <linux/zalloc.h>
 12#include <api/fs/fs.h>
 13#include <perf/bpf_perf.h>
 14
 15#include "bpf_counter.h"
 16#include "bpf-utils.h"
 17#include "counts.h"
 18#include "debug.h"
 19#include "evsel.h"
 20#include "evlist.h"
 21#include "target.h"
 22#include "cgroup.h"
 23#include "cpumap.h"
 24#include "thread_map.h"
 25
 26#include "bpf_skel/bpf_prog_profiler.skel.h"
 27#include "bpf_skel/bperf_u.h"
 28#include "bpf_skel/bperf_leader.skel.h"
 29#include "bpf_skel/bperf_follower.skel.h"
 30
 31#define ATTR_MAP_SIZE 16
 32
 33static inline void *u64_to_ptr(__u64 ptr)
 34{
 35	return (void *)(unsigned long)ptr;
 36}
 37
 38static struct bpf_counter *bpf_counter_alloc(void)
 39{
 40	struct bpf_counter *counter;
 41
 42	counter = zalloc(sizeof(*counter));
 43	if (counter)
 44		INIT_LIST_HEAD(&counter->list);
 45	return counter;
 46}
 47
 48static int bpf_program_profiler__destroy(struct evsel *evsel)
 49{
 50	struct bpf_counter *counter, *tmp;
 51
 52	list_for_each_entry_safe(counter, tmp,
 53				 &evsel->bpf_counter_list, list) {
 54		list_del_init(&counter->list);
 55		bpf_prog_profiler_bpf__destroy(counter->skel);
 56		free(counter);
 57	}
 58	assert(list_empty(&evsel->bpf_counter_list));
 59
 60	return 0;
 61}
 62
 63static char *bpf_target_prog_name(int tgt_fd)
 64{
 65	struct bpf_func_info *func_info;
 66	struct perf_bpil *info_linear;
 67	const struct btf_type *t;
 68	struct btf *btf = NULL;
 69	char *name = NULL;
 70
 71	info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO);
 72	if (IS_ERR_OR_NULL(info_linear)) {
 73		pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd);
 74		return NULL;
 75	}
 76
 77	if (info_linear->info.btf_id == 0) {
 78		pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd);
 79		goto out;
 80	}
 81
 82	btf = btf__load_from_kernel_by_id(info_linear->info.btf_id);
 83	if (libbpf_get_error(btf)) {
 84		pr_debug("failed to load btf for prog FD %d\n", tgt_fd);
 85		goto out;
 86	}
 87
 88	func_info = u64_to_ptr(info_linear->info.func_info);
 89	t = btf__type_by_id(btf, func_info[0].type_id);
 90	if (!t) {
 91		pr_debug("btf %d doesn't have type %d\n",
 92			 info_linear->info.btf_id, func_info[0].type_id);
 93		goto out;
 94	}
 95	name = strdup(btf__name_by_offset(btf, t->name_off));
 96out:
 97	btf__free(btf);
 98	free(info_linear);
 99	return name;
100}
101
102static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id)
103{
104	struct bpf_prog_profiler_bpf *skel;
105	struct bpf_counter *counter;
106	struct bpf_program *prog;
107	char *prog_name = NULL;
108	int prog_fd;
109	int err;
110
111	prog_fd = bpf_prog_get_fd_by_id(prog_id);
112	if (prog_fd < 0) {
113		pr_err("Failed to open fd for bpf prog %u\n", prog_id);
114		return -1;
115	}
116	counter = bpf_counter_alloc();
117	if (!counter) {
118		close(prog_fd);
119		return -1;
120	}
121
122	skel = bpf_prog_profiler_bpf__open();
123	if (!skel) {
124		pr_err("Failed to open bpf skeleton\n");
125		goto err_out;
126	}
127
128	skel->rodata->num_cpu = evsel__nr_cpus(evsel);
129
130	bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel));
131	bpf_map__set_max_entries(skel->maps.fentry_readings, 1);
132	bpf_map__set_max_entries(skel->maps.accum_readings, 1);
133
134	prog_name = bpf_target_prog_name(prog_fd);
135	if (!prog_name) {
136		pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id);
137		goto err_out;
138	}
139
140	bpf_object__for_each_program(prog, skel->obj) {
141		err = bpf_program__set_attach_target(prog, prog_fd, prog_name);
142		if (err) {
143			pr_err("bpf_program__set_attach_target failed.\n"
144			       "Does bpf prog %u have BTF?\n", prog_id);
145			goto err_out;
146		}
147	}
148	set_max_rlimit();
149	err = bpf_prog_profiler_bpf__load(skel);
150	if (err) {
151		pr_err("bpf_prog_profiler_bpf__load failed\n");
152		goto err_out;
153	}
154
155	assert(skel != NULL);
156	counter->skel = skel;
157	list_add(&counter->list, &evsel->bpf_counter_list);
158	free(prog_name);
159	close(prog_fd);
160	return 0;
161err_out:
162	bpf_prog_profiler_bpf__destroy(skel);
163	free(prog_name);
164	free(counter);
165	close(prog_fd);
166	return -1;
167}
168
169static int bpf_program_profiler__load(struct evsel *evsel, struct target *target)
170{
171	char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p;
172	u32 prog_id;
173	int ret;
174
175	bpf_str_ = bpf_str = strdup(target->bpf_str);
176	if (!bpf_str)
177		return -1;
178
179	while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) {
180		prog_id = strtoul(tok, &p, 10);
181		if (prog_id == 0 || prog_id == UINT_MAX ||
182		    (*p != '\0' && *p != ',')) {
183			pr_err("Failed to parse bpf prog ids %s\n",
184			       target->bpf_str);
185			free(bpf_str_);
186			return -1;
187		}
188
189		ret = bpf_program_profiler_load_one(evsel, prog_id);
190		if (ret) {
191			bpf_program_profiler__destroy(evsel);
192			free(bpf_str_);
193			return -1;
194		}
195		bpf_str = NULL;
196	}
197	free(bpf_str_);
198	return 0;
199}
200
201static int bpf_program_profiler__enable(struct evsel *evsel)
202{
203	struct bpf_counter *counter;
204	int ret;
205
206	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
207		assert(counter->skel != NULL);
208		ret = bpf_prog_profiler_bpf__attach(counter->skel);
209		if (ret) {
210			bpf_program_profiler__destroy(evsel);
211			return ret;
212		}
213	}
214	return 0;
215}
216
217static int bpf_program_profiler__disable(struct evsel *evsel)
218{
219	struct bpf_counter *counter;
220
221	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
222		assert(counter->skel != NULL);
223		bpf_prog_profiler_bpf__detach(counter->skel);
224	}
225	return 0;
226}
227
228static int bpf_program_profiler__read(struct evsel *evsel)
229{
230	// BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible
231	// Sometimes possible > online, like on a Ryzen 3900X that has 24
232	// threads but its possible showed 0-31 -acme
233	int num_cpu_bpf = libbpf_num_possible_cpus();
234	struct bpf_perf_event_value values[num_cpu_bpf];
235	struct bpf_counter *counter;
236	struct perf_counts_values *counts;
237	int reading_map_fd;
238	__u32 key = 0;
239	int err, idx, bpf_cpu;
240
241	if (list_empty(&evsel->bpf_counter_list))
242		return -EAGAIN;
243
244	perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) {
245		counts = perf_counts(evsel->counts, idx, 0);
246		counts->val = 0;
247		counts->ena = 0;
248		counts->run = 0;
249	}
250	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
251		struct bpf_prog_profiler_bpf *skel = counter->skel;
252
253		assert(skel != NULL);
254		reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
255
256		err = bpf_map_lookup_elem(reading_map_fd, &key, values);
257		if (err) {
258			pr_err("failed to read value\n");
259			return err;
260		}
261
262		for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) {
263			idx = perf_cpu_map__idx(evsel__cpus(evsel),
264						(struct perf_cpu){.cpu = bpf_cpu});
265			if (idx == -1)
266				continue;
267			counts = perf_counts(evsel->counts, idx, 0);
268			counts->val += values[bpf_cpu].counter;
269			counts->ena += values[bpf_cpu].enabled;
270			counts->run += values[bpf_cpu].running;
271		}
272	}
273	return 0;
274}
275
276static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx,
277					    int fd)
278{
279	struct bpf_prog_profiler_bpf *skel;
280	struct bpf_counter *counter;
281	int ret;
282
283	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
284		skel = counter->skel;
285		assert(skel != NULL);
286
287		ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events),
288					  &cpu_map_idx, &fd, BPF_ANY);
289		if (ret)
290			return ret;
291	}
292	return 0;
293}
294
295struct bpf_counter_ops bpf_program_profiler_ops = {
296	.load       = bpf_program_profiler__load,
297	.enable	    = bpf_program_profiler__enable,
298	.disable    = bpf_program_profiler__disable,
299	.read       = bpf_program_profiler__read,
300	.destroy    = bpf_program_profiler__destroy,
301	.install_pe = bpf_program_profiler__install_pe,
302};
303
304static bool bperf_attr_map_compatible(int attr_map_fd)
305{
306	struct bpf_map_info map_info = {0};
307	__u32 map_info_len = sizeof(map_info);
308	int err;
309
310	err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len);
311
312	if (err)
313		return false;
314	return (map_info.key_size == sizeof(struct perf_event_attr)) &&
315		(map_info.value_size == sizeof(struct perf_event_attr_map_entry));
316}
317
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318static int bperf_lock_attr_map(struct target *target)
319{
320	char path[PATH_MAX];
321	int map_fd, err;
322
323	if (target->attr_map) {
324		scnprintf(path, PATH_MAX, "%s", target->attr_map);
325	} else {
326		scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(),
327			  BPF_PERF_DEFAULT_ATTR_MAP_PATH);
328	}
329
330	if (access(path, F_OK)) {
331		map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL,
332					sizeof(struct perf_event_attr),
333					sizeof(struct perf_event_attr_map_entry),
334					ATTR_MAP_SIZE, NULL);
335		if (map_fd < 0)
336			return -1;
337
338		err = bpf_obj_pin(map_fd, path);
339		if (err) {
340			/* someone pinned the map in parallel? */
341			close(map_fd);
342			map_fd = bpf_obj_get(path);
343			if (map_fd < 0)
344				return -1;
345		}
346	} else {
347		map_fd = bpf_obj_get(path);
348		if (map_fd < 0)
349			return -1;
350	}
351
352	if (!bperf_attr_map_compatible(map_fd)) {
353		close(map_fd);
354		return -1;
355
356	}
357	err = flock(map_fd, LOCK_EX);
358	if (err) {
359		close(map_fd);
360		return -1;
361	}
362	return map_fd;
363}
364
365static int bperf_check_target(struct evsel *evsel,
366			      struct target *target,
367			      enum bperf_filter_type *filter_type,
368			      __u32 *filter_entry_cnt)
369{
370	if (evsel->core.leader->nr_members > 1) {
371		pr_err("bpf managed perf events do not yet support groups.\n");
372		return -1;
373	}
374
375	/* determine filter type based on target */
376	if (target->system_wide) {
377		*filter_type = BPERF_FILTER_GLOBAL;
378		*filter_entry_cnt = 1;
379	} else if (target->cpu_list) {
380		*filter_type = BPERF_FILTER_CPU;
381		*filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel));
382	} else if (target->tid) {
383		*filter_type = BPERF_FILTER_PID;
384		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
385	} else if (target->pid || evsel->evlist->workload.pid != -1) {
386		*filter_type = BPERF_FILTER_TGID;
387		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
388	} else {
389		pr_err("bpf managed perf events do not yet support these targets.\n");
390		return -1;
391	}
392
393	return 0;
394}
395
396static	struct perf_cpu_map *all_cpu_map;
397
398static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd,
399				       struct perf_event_attr_map_entry *entry)
400{
401	struct bperf_leader_bpf *skel = bperf_leader_bpf__open();
402	int link_fd, diff_map_fd, err;
403	struct bpf_link *link = NULL;
404
405	if (!skel) {
406		pr_err("Failed to open leader skeleton\n");
407		return -1;
408	}
409
410	bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus());
411	err = bperf_leader_bpf__load(skel);
412	if (err) {
413		pr_err("Failed to load leader skeleton\n");
414		goto out;
415	}
416
417	link = bpf_program__attach(skel->progs.on_switch);
418	if (IS_ERR(link)) {
419		pr_err("Failed to attach leader program\n");
420		err = PTR_ERR(link);
421		goto out;
422	}
423
424	link_fd = bpf_link__fd(link);
425	diff_map_fd = bpf_map__fd(skel->maps.diff_readings);
426	entry->link_id = bpf_link_get_id(link_fd);
427	entry->diff_map_id = bpf_map_get_id(diff_map_fd);
428	err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY);
429	assert(err == 0);
430
431	evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id);
432	assert(evsel->bperf_leader_link_fd >= 0);
433
434	/*
435	 * save leader_skel for install_pe, which is called within
436	 * following evsel__open_per_cpu call
437	 */
438	evsel->leader_skel = skel;
439	evsel__open_per_cpu(evsel, all_cpu_map, -1);
440
441out:
442	bperf_leader_bpf__destroy(skel);
443	bpf_link__destroy(link);
444	return err;
445}
446
447static int bperf__load(struct evsel *evsel, struct target *target)
448{
449	struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff};
450	int attr_map_fd, diff_map_fd = -1, err;
451	enum bperf_filter_type filter_type;
452	__u32 filter_entry_cnt, i;
453
454	if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt))
455		return -1;
456
457	if (!all_cpu_map) {
458		all_cpu_map = perf_cpu_map__new_online_cpus();
459		if (!all_cpu_map)
460			return -1;
461	}
462
463	evsel->bperf_leader_prog_fd = -1;
464	evsel->bperf_leader_link_fd = -1;
465
466	/*
467	 * Step 1: hold a fd on the leader program and the bpf_link, if
468	 * the program is not already gone, reload the program.
469	 * Use flock() to ensure exclusive access to the perf_event_attr
470	 * map.
471	 */
472	attr_map_fd = bperf_lock_attr_map(target);
473	if (attr_map_fd < 0) {
474		pr_err("Failed to lock perf_event_attr map\n");
475		return -1;
476	}
477
478	err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry);
479	if (err) {
480		err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY);
481		if (err)
482			goto out;
483	}
484
485	evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id);
486	if (evsel->bperf_leader_link_fd < 0 &&
487	    bperf_reload_leader_program(evsel, attr_map_fd, &entry)) {
488		err = -1;
489		goto out;
490	}
491	/*
492	 * The bpf_link holds reference to the leader program, and the
493	 * leader program holds reference to the maps. Therefore, if
494	 * link_id is valid, diff_map_id should also be valid.
495	 */
496	evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id(
497		bpf_link_get_prog_id(evsel->bperf_leader_link_fd));
498	assert(evsel->bperf_leader_prog_fd >= 0);
499
500	diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id);
501	assert(diff_map_fd >= 0);
502
503	/*
504	 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
505	 * whether the kernel support it
506	 */
507	err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0);
508	if (err) {
509		pr_err("The kernel does not support test_run for raw_tp BPF programs.\n"
510		       "Therefore, --use-bpf might show inaccurate readings\n");
511		goto out;
512	}
513
514	/* Step 2: load the follower skeleton */
515	evsel->follower_skel = bperf_follower_bpf__open();
516	if (!evsel->follower_skel) {
517		err = -1;
518		pr_err("Failed to open follower skeleton\n");
519		goto out;
520	}
521
522	/* attach fexit program to the leader program */
523	bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX,
524				       evsel->bperf_leader_prog_fd, "on_switch");
525
526	/* connect to leader diff_reading map */
527	bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd);
528
529	/* set up reading map */
530	bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings,
531				 filter_entry_cnt);
532	/* set up follower filter based on target */
533	bpf_map__set_max_entries(evsel->follower_skel->maps.filter,
534				 filter_entry_cnt);
535	err = bperf_follower_bpf__load(evsel->follower_skel);
536	if (err) {
537		pr_err("Failed to load follower skeleton\n");
538		bperf_follower_bpf__destroy(evsel->follower_skel);
539		evsel->follower_skel = NULL;
540		goto out;
541	}
542
543	for (i = 0; i < filter_entry_cnt; i++) {
544		int filter_map_fd;
545		__u32 key;
546
547		if (filter_type == BPERF_FILTER_PID ||
548		    filter_type == BPERF_FILTER_TGID)
549			key = perf_thread_map__pid(evsel->core.threads, i);
550		else if (filter_type == BPERF_FILTER_CPU)
551			key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu;
552		else
553			break;
554
555		filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter);
556		bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY);
557	}
558
559	evsel->follower_skel->bss->type = filter_type;
560
561	err = bperf_follower_bpf__attach(evsel->follower_skel);
562
563out:
564	if (err && evsel->bperf_leader_link_fd >= 0)
565		close(evsel->bperf_leader_link_fd);
566	if (err && evsel->bperf_leader_prog_fd >= 0)
567		close(evsel->bperf_leader_prog_fd);
568	if (diff_map_fd >= 0)
569		close(diff_map_fd);
570
571	flock(attr_map_fd, LOCK_UN);
572	close(attr_map_fd);
573
574	return err;
575}
576
577static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)
578{
579	struct bperf_leader_bpf *skel = evsel->leader_skel;
580
581	return bpf_map_update_elem(bpf_map__fd(skel->maps.events),
582				   &cpu_map_idx, &fd, BPF_ANY);
583}
584
585/*
586 * trigger the leader prog on each cpu, so the accum_reading map could get
587 * the latest readings.
588 */
589static int bperf_sync_counters(struct evsel *evsel)
590{
591	int num_cpu, i, cpu;
592
593	num_cpu = perf_cpu_map__nr(all_cpu_map);
594	for (i = 0; i < num_cpu; i++) {
595		cpu = perf_cpu_map__cpu(all_cpu_map, i).cpu;
596		bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu);
597	}
598	return 0;
599}
600
601static int bperf__enable(struct evsel *evsel)
602{
603	evsel->follower_skel->bss->enabled = 1;
604	return 0;
605}
606
607static int bperf__disable(struct evsel *evsel)
608{
609	evsel->follower_skel->bss->enabled = 0;
610	return 0;
611}
612
613static int bperf__read(struct evsel *evsel)
614{
615	struct bperf_follower_bpf *skel = evsel->follower_skel;
616	__u32 num_cpu_bpf = cpu__max_cpu().cpu;
617	struct bpf_perf_event_value values[num_cpu_bpf];
618	struct perf_counts_values *counts;
619	int reading_map_fd, err = 0;
620	__u32 i;
621	int j;
622
623	bperf_sync_counters(evsel);
624	reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
625
626	for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) {
627		struct perf_cpu entry;
628		__u32 cpu;
629
630		err = bpf_map_lookup_elem(reading_map_fd, &i, values);
631		if (err)
632			goto out;
633		switch (evsel->follower_skel->bss->type) {
634		case BPERF_FILTER_GLOBAL:
635			assert(i == 0);
636
637			perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) {
638				counts = perf_counts(evsel->counts, j, 0);
639				counts->val = values[entry.cpu].counter;
640				counts->ena = values[entry.cpu].enabled;
641				counts->run = values[entry.cpu].running;
642			}
643			break;
644		case BPERF_FILTER_CPU:
645			cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu;
646			assert(cpu >= 0);
647			counts = perf_counts(evsel->counts, i, 0);
648			counts->val = values[cpu].counter;
649			counts->ena = values[cpu].enabled;
650			counts->run = values[cpu].running;
651			break;
652		case BPERF_FILTER_PID:
653		case BPERF_FILTER_TGID:
654			counts = perf_counts(evsel->counts, 0, i);
655			counts->val = 0;
656			counts->ena = 0;
657			counts->run = 0;
658
659			for (cpu = 0; cpu < num_cpu_bpf; cpu++) {
660				counts->val += values[cpu].counter;
661				counts->ena += values[cpu].enabled;
662				counts->run += values[cpu].running;
663			}
664			break;
665		default:
666			break;
667		}
668	}
669out:
670	return err;
671}
672
673static int bperf__destroy(struct evsel *evsel)
674{
675	bperf_follower_bpf__destroy(evsel->follower_skel);
676	close(evsel->bperf_leader_prog_fd);
677	close(evsel->bperf_leader_link_fd);
678	return 0;
679}
680
681/*
682 * bperf: share hardware PMCs with BPF
683 *
684 * perf uses performance monitoring counters (PMC) to monitor system
685 * performance. The PMCs are limited hardware resources. For example,
686 * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu.
687 *
688 * Modern data center systems use these PMCs in many different ways:
689 * system level monitoring, (maybe nested) container level monitoring, per
690 * process monitoring, profiling (in sample mode), etc. In some cases,
691 * there are more active perf_events than available hardware PMCs. To allow
692 * all perf_events to have a chance to run, it is necessary to do expensive
693 * time multiplexing of events.
694 *
695 * On the other hand, many monitoring tools count the common metrics
696 * (cycles, instructions). It is a waste to have multiple tools create
697 * multiple perf_events of "cycles" and occupy multiple PMCs.
698 *
699 * bperf tries to reduce such wastes by allowing multiple perf_events of
700 * "cycles" or "instructions" (at different scopes) to share PMUs. Instead
701 * of having each perf-stat session to read its own perf_events, bperf uses
702 * BPF programs to read the perf_events and aggregate readings to BPF maps.
703 * Then, the perf-stat session(s) reads the values from these BPF maps.
704 *
705 *                                ||
706 *       shared progs and maps <- || -> per session progs and maps
707 *                                ||
708 *   ---------------              ||
709 *   | perf_events |              ||
710 *   ---------------       fexit  ||      -----------------
711 *          |             --------||----> | follower prog |
712 *       --------------- /        || ---  -----------------
713 * cs -> | leader prog |/         ||/        |         |
714 *   --> ---------------         /||  --------------  ------------------
715 *  /       |         |         / ||  | filter map |  | accum_readings |
716 * /  ------------  ------------  ||  --------------  ------------------
717 * |  | prev map |  | diff map |  ||                        |
718 * |  ------------  ------------  ||                        |
719 *  \                             ||                        |
720 * = \ ==================================================== | ============
721 *    \                                                    /   user space
722 *     \                                                  /
723 *      \                                                /
724 *    BPF_PROG_TEST_RUN                    BPF_MAP_LOOKUP_ELEM
725 *        \                                            /
726 *         \                                          /
727 *          \------  perf-stat ----------------------/
728 *
729 * The figure above shows the architecture of bperf. Note that the figure
730 * is divided into 3 regions: shared progs and maps (top left), per session
731 * progs and maps (top right), and user space (bottom).
732 *
733 * The leader prog is triggered on each context switch (cs). The leader
734 * prog reads perf_events and stores the difference (current_reading -
735 * previous_reading) to the diff map. For the same metric, e.g. "cycles",
736 * multiple perf-stat sessions share the same leader prog.
737 *
738 * Each perf-stat session creates a follower prog as fexit program to the
739 * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38)
740 * follower progs to the same leader prog. The follower prog checks current
741 * task and processor ID to decide whether to add the value from the diff
742 * map to its accumulated reading map (accum_readings).
743 *
744 * Finally, perf-stat user space reads the value from accum_reading map.
745 *
746 * Besides context switch, it is also necessary to trigger the leader prog
747 * before perf-stat reads the value. Otherwise, the accum_reading map may
748 * not have the latest reading from the perf_events. This is achieved by
749 * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU.
750 *
751 * Comment before the definition of struct perf_event_attr_map_entry
752 * describes how different sessions of perf-stat share information about
753 * the leader prog.
754 */
755
756struct bpf_counter_ops bperf_ops = {
757	.load       = bperf__load,
758	.enable     = bperf__enable,
759	.disable    = bperf__disable,
760	.read       = bperf__read,
761	.install_pe = bperf__install_pe,
762	.destroy    = bperf__destroy,
763};
764
765extern struct bpf_counter_ops bperf_cgrp_ops;
766
767static inline bool bpf_counter_skip(struct evsel *evsel)
768{
769	return evsel->bpf_counter_ops == NULL;
 
770}
771
772int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)
773{
774	if (bpf_counter_skip(evsel))
775		return 0;
776	return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd);
777}
778
779int bpf_counter__load(struct evsel *evsel, struct target *target)
780{
781	if (target->bpf_str)
782		evsel->bpf_counter_ops = &bpf_program_profiler_ops;
783	else if (cgrp_event_expanded && target->use_bpf)
784		evsel->bpf_counter_ops = &bperf_cgrp_ops;
785	else if (target->use_bpf || evsel->bpf_counter ||
786		 evsel__match_bpf_counter_events(evsel->name))
787		evsel->bpf_counter_ops = &bperf_ops;
788
789	if (evsel->bpf_counter_ops)
790		return evsel->bpf_counter_ops->load(evsel, target);
791	return 0;
792}
793
794int bpf_counter__enable(struct evsel *evsel)
795{
796	if (bpf_counter_skip(evsel))
797		return 0;
798	return evsel->bpf_counter_ops->enable(evsel);
799}
800
801int bpf_counter__disable(struct evsel *evsel)
802{
803	if (bpf_counter_skip(evsel))
804		return 0;
805	return evsel->bpf_counter_ops->disable(evsel);
806}
807
808int bpf_counter__read(struct evsel *evsel)
809{
810	if (bpf_counter_skip(evsel))
811		return -EAGAIN;
812	return evsel->bpf_counter_ops->read(evsel);
813}
814
815void bpf_counter__destroy(struct evsel *evsel)
816{
817	if (bpf_counter_skip(evsel))
818		return;
819	evsel->bpf_counter_ops->destroy(evsel);
820	evsel->bpf_counter_ops = NULL;
821	evsel->bpf_skel = NULL;
822}