1// SPDX-License-Identifier: GPL-2.0
  2
  3/* Copyright (c) 2021 Facebook */
  4/* Copyright (c) 2021 Google */
  5
  6#include <assert.h>
  7#include <limits.h>
  8#include <unistd.h>
  9#include <sys/file.h>
 10#include <sys/time.h>
 11#include <sys/resource.h>
 12#include <linux/err.h>
 13#include <linux/zalloc.h>
 14#include <linux/perf_event.h>
 15#include <api/fs/fs.h>
 16#include <perf/bpf_perf.h>
 17
 18#include "affinity.h"
 19#include "bpf_counter.h"
 20#include "cgroup.h"
 21#include "counts.h"
 22#include "debug.h"
 23#include "evsel.h"
 24#include "evlist.h"
 25#include "target.h"
 26#include "cpumap.h"
 27#include "thread_map.h"
 28
 29#include "bpf_skel/bperf_cgroup.skel.h"
 30
 31static struct perf_event_attr cgrp_switch_attr = {
 32	.type = PERF_TYPE_SOFTWARE,
 33	.config = PERF_COUNT_SW_CGROUP_SWITCHES,
 34	.size = sizeof(cgrp_switch_attr),
 35	.sample_period = 1,
 36	.disabled = 1,
 37};
 38
 39static struct evsel *cgrp_switch;
 40static struct bperf_cgroup_bpf *skel;
 41
 42#define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0))
 43
 44static int bperf_load_program(struct evlist *evlist)
 45{
 46	struct bpf_link *link;
 47	struct evsel *evsel;
 48	struct cgroup *cgrp, *leader_cgrp;
 49	__u32 i, cpu;
 50	__u32 nr_cpus = evlist->core.all_cpus->nr;
 51	int total_cpus = cpu__max_cpu();
 52	int map_size, map_fd;
 53	int prog_fd, err;
 54
 55	skel = bperf_cgroup_bpf__open();
 56	if (!skel) {
 57		pr_err("Failed to open cgroup skeleton\n");
 58		return -1;
 59	}
 60
 61	skel->rodata->num_cpus = total_cpus;
 62	skel->rodata->num_events = evlist->core.nr_entries / nr_cgroups;
 63
 64	BUG_ON(evlist->core.nr_entries % nr_cgroups != 0);
 65
 66	/* we need one copy of events per cpu for reading */
 67	map_size = total_cpus * evlist->core.nr_entries / nr_cgroups;
 68	bpf_map__resize(skel->maps.events, map_size);
 69	bpf_map__resize(skel->maps.cgrp_idx, nr_cgroups);
 70	/* previous result is saved in a per-cpu array */
 71	map_size = evlist->core.nr_entries / nr_cgroups;
 72	bpf_map__resize(skel->maps.prev_readings, map_size);
 73	/* cgroup result needs all events (per-cpu) */
 74	map_size = evlist->core.nr_entries;
 75	bpf_map__resize(skel->maps.cgrp_readings, map_size);
 76
 77	set_max_rlimit();
 78
 79	err = bperf_cgroup_bpf__load(skel);
 80	if (err) {
 81		pr_err("Failed to load cgroup skeleton\n");
 82		goto out;
 83	}
 84
 85	if (cgroup_is_v2("perf_event") > 0)
 86		skel->bss->use_cgroup_v2 = 1;
 87
 88	err = -1;
 89
 90	cgrp_switch = evsel__new(&cgrp_switch_attr);
 91	if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) {
 92		pr_err("Failed to open cgroup switches event\n");
 93		goto out;
 94	}
 95
 96	for (i = 0; i < nr_cpus; i++) {
 97		link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch,
 98						      FD(cgrp_switch, i));
 99		if (IS_ERR(link)) {
100			pr_err("Failed to attach cgroup program\n");
101			err = PTR_ERR(link);
102			goto out;
103		}
104	}
105
106	/*
107	 * Update cgrp_idx map from cgroup-id to event index.
108	 */
109	cgrp = NULL;
110	i = 0;
111
112	evlist__for_each_entry(evlist, evsel) {
113		if (cgrp == NULL || evsel->cgrp == leader_cgrp) {
114			leader_cgrp = evsel->cgrp;
115			evsel->cgrp = NULL;
116
117			/* open single copy of the events w/o cgroup */
118			err = evsel__open_per_cpu(evsel, evlist->core.all_cpus, -1);
119			if (err) {
120				pr_err("Failed to open first cgroup events\n");
121				goto out;
122			}
123
124			map_fd = bpf_map__fd(skel->maps.events);
125			for (cpu = 0; cpu < nr_cpus; cpu++) {
126				int fd = FD(evsel, cpu);
127				__u32 idx = evsel->core.idx * total_cpus +
128					evlist->core.all_cpus->map[cpu];
129
130				err = bpf_map_update_elem(map_fd, &idx, &fd,
131							  BPF_ANY);
132				if (err < 0) {
133					pr_err("Failed to update perf_event fd\n");
134					goto out;
135				}
136			}
137
138			evsel->cgrp = leader_cgrp;
139		}
140		evsel->supported = true;
141
142		if (evsel->cgrp == cgrp)
143			continue;
144
145		cgrp = evsel->cgrp;
146
147		if (read_cgroup_id(cgrp) < 0) {
148			pr_err("Failed to get cgroup id\n");
149			err = -1;
150			goto out;
151		}
152
153		map_fd = bpf_map__fd(skel->maps.cgrp_idx);
154		err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY);
155		if (err < 0) {
156			pr_err("Failed to update cgroup index map\n");
157			goto out;
158		}
159
160		i++;
161	}
162
163	/*
164	 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
165	 * whether the kernel support it
166	 */
167	prog_fd = bpf_program__fd(skel->progs.trigger_read);
168	err = bperf_trigger_reading(prog_fd, 0);
169	if (err) {
170		pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n"
171			   "Therefore, --for-each-cgroup might show inaccurate readings\n");
172		err = 0;
173	}
174
175out:
176	return err;
177}
178
179static int bperf_cgrp__load(struct evsel *evsel,
180			    struct target *target __maybe_unused)
181{
182	static bool bperf_loaded = false;
183
184	evsel->bperf_leader_prog_fd = -1;
185	evsel->bperf_leader_link_fd = -1;
186
187	if (!bperf_loaded && bperf_load_program(evsel->evlist))
188		return -1;
189
190	bperf_loaded = true;
191	/* just to bypass bpf_counter_skip() */
192	evsel->follower_skel = (struct bperf_follower_bpf *)skel;
193
194	return 0;
195}
196
197static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused,
198				  int cpu __maybe_unused, int fd __maybe_unused)
199{
200	/* nothing to do */
201	return 0;
202}
203
204/*
205 * trigger the leader prog on each cpu, so the cgrp_reading map could get
206 * the latest results.
207 */
208static int bperf_cgrp__sync_counters(struct evlist *evlist)
209{
210	int i, cpu;
211	int nr_cpus = evlist->core.all_cpus->nr;
212	int prog_fd = bpf_program__fd(skel->progs.trigger_read);
213
214	for (i = 0; i < nr_cpus; i++) {
215		cpu = evlist->core.all_cpus->map[i];
216		bperf_trigger_reading(prog_fd, cpu);
217	}
218
219	return 0;
220}
221
222static int bperf_cgrp__enable(struct evsel *evsel)
223{
224	if (evsel->core.idx)
225		return 0;
226
227	bperf_cgrp__sync_counters(evsel->evlist);
228
229	skel->bss->enabled = 1;
230	return 0;
231}
232
233static int bperf_cgrp__disable(struct evsel *evsel)
234{
235	if (evsel->core.idx)
236		return 0;
237
238	bperf_cgrp__sync_counters(evsel->evlist);
239
240	skel->bss->enabled = 0;
241	return 0;
242}
243
244static int bperf_cgrp__read(struct evsel *evsel)
245{
246	struct evlist *evlist = evsel->evlist;
247	int i, cpu, nr_cpus = evlist->core.all_cpus->nr;
248	int total_cpus = cpu__max_cpu();
249	struct perf_counts_values *counts;
250	struct bpf_perf_event_value *values;
251	int reading_map_fd, err = 0;
252	__u32 idx;
253
254	if (evsel->core.idx)
255		return 0;
256
257	bperf_cgrp__sync_counters(evsel->evlist);
258
259	values = calloc(total_cpus, sizeof(*values));
260	if (values == NULL)
261		return -ENOMEM;
262
263	reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings);
264
265	evlist__for_each_entry(evlist, evsel) {
266		idx = evsel->core.idx;
267		err = bpf_map_lookup_elem(reading_map_fd, &idx, values);
268		if (err) {
269			pr_err("bpf map lookup falied: idx=%u, event=%s, cgrp=%s\n",
270			       idx, evsel__name(evsel), evsel->cgrp->name);
271			goto out;
272		}
273
274		for (i = 0; i < nr_cpus; i++) {
275			cpu = evlist->core.all_cpus->map[i];
276
277			counts = perf_counts(evsel->counts, i, 0);
278			counts->val = values[cpu].counter;
279			counts->ena = values[cpu].enabled;
280			counts->run = values[cpu].running;
281		}
282	}
283
284out:
285	free(values);
286	return err;
287}
288
289static int bperf_cgrp__destroy(struct evsel *evsel)
290{
291	if (evsel->core.idx)
292		return 0;
293
294	bperf_cgroup_bpf__destroy(skel);
295	evsel__delete(cgrp_switch);  // it'll destroy on_switch progs too
296
297	return 0;
298}
299
300struct bpf_counter_ops bperf_cgrp_ops = {
301	.load       = bperf_cgrp__load,
302	.enable     = bperf_cgrp__enable,
303	.disable    = bperf_cgrp__disable,
304	.read       = bperf_cgrp__read,
305	.install_pe = bperf_cgrp__install_pe,
306	.destroy    = bperf_cgrp__destroy,
307};