1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * bpf_kwork.c
  4 *
  5 * Copyright (c) 2022  Huawei Inc,  Yang Jihong <yangjihong1@huawei.com>
  6 */
  7
  8#include <time.h>
  9#include <fcntl.h>
 10#include <signal.h>
 11#include <stdio.h>
 12#include <unistd.h>
 13
 14#include <linux/time64.h>
 15
 16#include "util/debug.h"
 17#include "util/evsel.h"
 18#include "util/kwork.h"
 19
 20#include <bpf/bpf.h>
 21#include <perf/cpumap.h>
 22
 23#include "util/bpf_skel/kwork_trace.skel.h"
 24
 25/*
 26 * This should be in sync with "util/kwork_trace.bpf.c"
 27 */
 28#define MAX_KWORKNAME 128
 29
 30struct work_key {
 31	u32 type;
 32	u32 cpu;
 33	u64 id;
 34};
 35
 36struct report_data {
 37	u64 nr;
 38	u64 total_time;
 39	u64 max_time;
 40	u64 max_time_start;
 41	u64 max_time_end;
 42};
 43
 44struct kwork_class_bpf {
 45	struct kwork_class *class;
 46
 47	void (*load_prepare)(struct perf_kwork *kwork);
 48	int  (*get_work_name)(struct work_key *key, char **ret_name);
 49};
 50
 51static struct kwork_trace_bpf *skel;
 52
 53static struct timespec ts_start;
 54static struct timespec ts_end;
 55
 56void perf_kwork__trace_start(void)
 57{
 58	clock_gettime(CLOCK_MONOTONIC, &ts_start);
 59	skel->bss->enabled = 1;
 60}
 61
 62void perf_kwork__trace_finish(void)
 63{
 64	clock_gettime(CLOCK_MONOTONIC, &ts_end);
 65	skel->bss->enabled = 0;
 66}
 67
 68static int get_work_name_from_map(struct work_key *key, char **ret_name)
 69{
 70	char name[MAX_KWORKNAME] = { 0 };
 71	int fd = bpf_map__fd(skel->maps.perf_kwork_names);
 72
 73	*ret_name = NULL;
 74
 75	if (fd < 0) {
 76		pr_debug("Invalid names map fd\n");
 77		return 0;
 78	}
 79
 80	if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) {
 81		*ret_name = strdup(name);
 82		if (*ret_name == NULL) {
 83			pr_err("Failed to copy work name\n");
 84			return -1;
 85		}
 86	}
 87
 88	return 0;
 89}
 90
 91static void irq_load_prepare(struct perf_kwork *kwork)
 92{
 93	if (kwork->report == KWORK_REPORT_RUNTIME) {
 94		bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true);
 95		bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true);
 96	}
 97}
 98
 99static struct kwork_class_bpf kwork_irq_bpf = {
100	.load_prepare  = irq_load_prepare,
101	.get_work_name = get_work_name_from_map,
102};
103
104static void softirq_load_prepare(struct perf_kwork *kwork)
105{
106	if (kwork->report == KWORK_REPORT_RUNTIME) {
107		bpf_program__set_autoload(skel->progs.report_softirq_entry, true);
108		bpf_program__set_autoload(skel->progs.report_softirq_exit, true);
109	} else if (kwork->report == KWORK_REPORT_LATENCY) {
110		bpf_program__set_autoload(skel->progs.latency_softirq_raise, true);
111		bpf_program__set_autoload(skel->progs.latency_softirq_entry, true);
112	}
113}
114
115static struct kwork_class_bpf kwork_softirq_bpf = {
116	.load_prepare  = softirq_load_prepare,
117	.get_work_name = get_work_name_from_map,
118};
119
120static void workqueue_load_prepare(struct perf_kwork *kwork)
121{
122	if (kwork->report == KWORK_REPORT_RUNTIME) {
123		bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true);
124		bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true);
125	} else if (kwork->report == KWORK_REPORT_LATENCY) {
126		bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true);
127		bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true);
128	}
129}
130
131static struct kwork_class_bpf kwork_workqueue_bpf = {
132	.load_prepare  = workqueue_load_prepare,
133	.get_work_name = get_work_name_from_map,
134};
135
136static struct kwork_class_bpf *
137kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = {
138	[KWORK_CLASS_IRQ]       = &kwork_irq_bpf,
139	[KWORK_CLASS_SOFTIRQ]   = &kwork_softirq_bpf,
140	[KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf,
141};
142
143static bool valid_kwork_class_type(enum kwork_class_type type)
144{
145	return type >= 0 && type < KWORK_CLASS_MAX ? true : false;
146}
147
148static int setup_filters(struct perf_kwork *kwork)
149{
150	if (kwork->cpu_list != NULL) {
151		int idx, nr_cpus;
152		struct perf_cpu_map *map;
153		struct perf_cpu cpu;
154		int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter);
155
156		if (fd < 0) {
157			pr_debug("Invalid cpu filter fd\n");
158			return -1;
159		}
160
161		map = perf_cpu_map__new(kwork->cpu_list);
162		if (map == NULL) {
163			pr_debug("Invalid cpu_list\n");
164			return -1;
165		}
166
167		nr_cpus = libbpf_num_possible_cpus();
168		perf_cpu_map__for_each_cpu(cpu, idx, map) {
169			u8 val = 1;
170
171			if (cpu.cpu >= nr_cpus) {
172				perf_cpu_map__put(map);
173				pr_err("Requested cpu %d too large\n", cpu.cpu);
174				return -1;
175			}
176			bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY);
177		}
178		perf_cpu_map__put(map);
179	}
180
181	if (kwork->profile_name != NULL) {
182		int key, fd;
183
184		if (strlen(kwork->profile_name) >= MAX_KWORKNAME) {
185			pr_err("Requested name filter %s too large, limit to %d\n",
186			       kwork->profile_name, MAX_KWORKNAME - 1);
187			return -1;
188		}
189
190		fd = bpf_map__fd(skel->maps.perf_kwork_name_filter);
191		if (fd < 0) {
192			pr_debug("Invalid name filter fd\n");
193			return -1;
194		}
195
196		key = 0;
197		bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY);
198	}
199
200	return 0;
201}
202
203int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork)
204{
205	struct bpf_program *prog;
206	struct kwork_class *class;
207	struct kwork_class_bpf *class_bpf;
208	enum kwork_class_type type;
209
210	skel = kwork_trace_bpf__open();
211	if (!skel) {
212		pr_debug("Failed to open kwork trace skeleton\n");
213		return -1;
214	}
215
216	/*
217	 * set all progs to non-autoload,
218	 * then set corresponding progs according to config
219	 */
220	bpf_object__for_each_program(prog, skel->obj)
221		bpf_program__set_autoload(prog, false);
222
223	list_for_each_entry(class, &kwork->class_list, list) {
224		type = class->type;
225		if (!valid_kwork_class_type(type) ||
226		    (kwork_class_bpf_supported_list[type] == NULL)) {
227			pr_err("Unsupported bpf trace class %s\n", class->name);
228			goto out;
229		}
230
231		class_bpf = kwork_class_bpf_supported_list[type];
232		class_bpf->class = class;
233
234		if (class_bpf->load_prepare != NULL)
235			class_bpf->load_prepare(kwork);
236	}
237
238	if (kwork->cpu_list != NULL)
239		skel->rodata->has_cpu_filter = 1;
240	if (kwork->profile_name != NULL)
241		skel->rodata->has_name_filter = 1;
242
243	if (kwork_trace_bpf__load(skel)) {
244		pr_debug("Failed to load kwork trace skeleton\n");
245		goto out;
246	}
247
248	if (setup_filters(kwork))
249		goto out;
250
251	if (kwork_trace_bpf__attach(skel)) {
252		pr_debug("Failed to attach kwork trace skeleton\n");
253		goto out;
254	}
255
256	return 0;
257
258out:
259	kwork_trace_bpf__destroy(skel);
260	return -1;
261}
262
263static int add_work(struct perf_kwork *kwork,
264		    struct work_key *key,
265		    struct report_data *data)
266{
267	struct kwork_work *work;
268	struct kwork_class_bpf *bpf_trace;
269	struct kwork_work tmp = {
270		.id = key->id,
271		.name = NULL,
272		.cpu = key->cpu,
273	};
274	enum kwork_class_type type = key->type;
275
276	if (!valid_kwork_class_type(type)) {
277		pr_debug("Invalid class type %d to add work\n", type);
278		return -1;
279	}
280
281	bpf_trace = kwork_class_bpf_supported_list[type];
282	tmp.class = bpf_trace->class;
283
284	if ((bpf_trace->get_work_name != NULL) &&
285	    (bpf_trace->get_work_name(key, &tmp.name)))
286		return -1;
287
288	work = perf_kwork_add_work(kwork, tmp.class, &tmp);
289	if (work == NULL)
290		return -1;
291
292	if (kwork->report == KWORK_REPORT_RUNTIME) {
293		work->nr_atoms = data->nr;
294		work->total_runtime = data->total_time;
295		work->max_runtime = data->max_time;
296		work->max_runtime_start = data->max_time_start;
297		work->max_runtime_end = data->max_time_end;
298	} else if (kwork->report == KWORK_REPORT_LATENCY) {
299		work->nr_atoms = data->nr;
300		work->total_latency = data->total_time;
301		work->max_latency = data->max_time;
302		work->max_latency_start = data->max_time_start;
303		work->max_latency_end = data->max_time_end;
304	} else {
305		pr_debug("Invalid bpf report type %d\n", kwork->report);
306		return -1;
307	}
308
309	kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec;
310	kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec;
311
312	return 0;
313}
314
315int perf_kwork__report_read_bpf(struct perf_kwork *kwork)
316{
317	struct report_data data;
318	struct work_key key = {
319		.type = 0,
320		.cpu  = 0,
321		.id   = 0,
322	};
323	struct work_key prev = {
324		.type = 0,
325		.cpu  = 0,
326		.id   = 0,
327	};
328	int fd = bpf_map__fd(skel->maps.perf_kwork_report);
329
330	if (fd < 0) {
331		pr_debug("Invalid report fd\n");
332		return -1;
333	}
334
335	while (!bpf_map_get_next_key(fd, &prev, &key)) {
336		if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) {
337			pr_debug("Failed to lookup report elem\n");
338			return -1;
339		}
340
341		if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0))
342			return -1;
343
344		prev = key;
345	}
346	return 0;
347}
348
349void perf_kwork__report_cleanup_bpf(void)
350{
351	kwork_trace_bpf__destroy(skel);
352}