1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
  2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
  3#ifndef __USDT_BPF_H__
  4#define __USDT_BPF_H__
  5
  6#include <linux/errno.h>
  7#include "bpf_helpers.h"
  8#include "bpf_tracing.h"
  9
 10/* Below types and maps are internal implementation details of libbpf's USDT
 11 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
 12 * be considered an unstable API as well and might be adjusted based on user
 13 * feedback from using libbpf's USDT support in production.
 14 */
 15
 16/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
 17 * map that keeps track of USDT argument specifications. This might be
 18 * necessary if there are a lot of USDT attachments.
 19 */
 20#ifndef BPF_USDT_MAX_SPEC_CNT
 21#define BPF_USDT_MAX_SPEC_CNT 256
 22#endif
 23/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
 24 * map that keeps track of IP (memory address) mapping to USDT argument
 25 * specification.
 26 * Note, if kernel supports BPF cookies, this map is not used and could be
 27 * resized all the way to 1 to save a bit of memory.
 28 */
 29#ifndef BPF_USDT_MAX_IP_CNT
 30#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
 31#endif
 32
 33enum __bpf_usdt_arg_type {
 34	BPF_USDT_ARG_CONST,
 35	BPF_USDT_ARG_REG,
 36	BPF_USDT_ARG_REG_DEREF,
 37};
 38
 39struct __bpf_usdt_arg_spec {
 40	/* u64 scalar interpreted depending on arg_type, see below */
 41	__u64 val_off;
 42	/* arg location case, see bpf_udst_arg() for details */
 43	enum __bpf_usdt_arg_type arg_type;
 44	/* offset of referenced register within struct pt_regs */
 45	short reg_off;
 46	/* whether arg should be interpreted as signed value */
 47	bool arg_signed;
 48	/* number of bits that need to be cleared and, optionally,
 49	 * sign-extended to cast arguments that are 1, 2, or 4 bytes
 50	 * long into final 8-byte u64/s64 value returned to user
 51	 */
 52	char arg_bitshift;
 53};
 54
 55/* should match USDT_MAX_ARG_CNT in usdt.c exactly */
 56#define BPF_USDT_MAX_ARG_CNT 12
 57struct __bpf_usdt_spec {
 58	struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
 59	__u64 usdt_cookie;
 60	short arg_cnt;
 61};
 62
 63struct {
 64	__uint(type, BPF_MAP_TYPE_ARRAY);
 65	__uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
 66	__type(key, int);
 67	__type(value, struct __bpf_usdt_spec);
 68} __bpf_usdt_specs SEC(".maps") __weak;
 69
 70struct {
 71	__uint(type, BPF_MAP_TYPE_HASH);
 72	__uint(max_entries, BPF_USDT_MAX_IP_CNT);
 73	__type(key, long);
 74	__type(value, __u32);
 75} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
 76
 77extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
 78
 79static __always_inline
 80int __bpf_usdt_spec_id(struct pt_regs *ctx)
 81{
 82	if (!LINUX_HAS_BPF_COOKIE) {
 83		long ip = PT_REGS_IP(ctx);
 84		int *spec_id_ptr;
 85
 86		spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
 87		return spec_id_ptr ? *spec_id_ptr : -ESRCH;
 88	}
 89
 90	return bpf_get_attach_cookie(ctx);
 91}
 92
 93/* Return number of USDT arguments defined for currently traced USDT. */
 94__weak __hidden
 95int bpf_usdt_arg_cnt(struct pt_regs *ctx)
 96{
 97	struct __bpf_usdt_spec *spec;
 98	int spec_id;
 99
100	spec_id = __bpf_usdt_spec_id(ctx);
101	if (spec_id < 0)
102		return -ESRCH;
103
104	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
105	if (!spec)
106		return -ESRCH;
107
108	return spec->arg_cnt;
109}
110
111/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
112 * Returns 0 on success; negative error, otherwise.
113 * On error *res is guaranteed to be set to zero.
114 */
115__weak __hidden
116int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
117{
118	struct __bpf_usdt_spec *spec;
119	struct __bpf_usdt_arg_spec *arg_spec;
120	unsigned long val;
121	int err, spec_id;
122
123	*res = 0;
124
125	spec_id = __bpf_usdt_spec_id(ctx);
126	if (spec_id < 0)
127		return -ESRCH;
128
129	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
130	if (!spec)
131		return -ESRCH;
132
133	if (arg_num >= BPF_USDT_MAX_ARG_CNT)
134		return -ENOENT;
135	barrier_var(arg_num);
136	if (arg_num >= spec->arg_cnt)
137		return -ENOENT;
138
139	arg_spec = &spec->args[arg_num];
140	switch (arg_spec->arg_type) {
141	case BPF_USDT_ARG_CONST:
142		/* Arg is just a constant ("-4@$-9" in USDT arg spec).
143		 * value is recorded in arg_spec->val_off directly.
144		 */
145		val = arg_spec->val_off;
146		break;
147	case BPF_USDT_ARG_REG:
148		/* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
149		 * so we read the contents of that register directly from
150		 * struct pt_regs. To keep things simple user-space parts
151		 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
152		 */
153		err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
154		if (err)
155			return err;
156		break;
157	case BPF_USDT_ARG_REG_DEREF:
158		/* Arg is in memory addressed by register, plus some offset
159		 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
160		 * identified like with BPF_USDT_ARG_REG case, and the offset
161		 * is in arg_spec->val_off. We first fetch register contents
162		 * from pt_regs, then do another user-space probe read to
163		 * fetch argument value itself.
164		 */
165		err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
166		if (err)
167			return err;
168		err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
169		if (err)
170			return err;
171#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
172		val >>= arg_spec->arg_bitshift;
173#endif
174		break;
175	default:
176		return -EINVAL;
177	}
178
179	/* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
180	 * necessary upper arg_bitshift bits, with sign extension if argument
181	 * is signed
182	 */
183	val <<= arg_spec->arg_bitshift;
184	if (arg_spec->arg_signed)
185		val = ((long)val) >> arg_spec->arg_bitshift;
186	else
187		val = val >> arg_spec->arg_bitshift;
188	*res = val;
189	return 0;
190}
191
192/* Retrieve user-specified cookie value provided during attach as
193 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
194 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
195 * utilizing BPF cookies internally, so user can't use BPF cookie directly
196 * for USDT programs and has to use bpf_usdt_cookie() API instead.
197 */
198__weak __hidden
199long bpf_usdt_cookie(struct pt_regs *ctx)
200{
201	struct __bpf_usdt_spec *spec;
202	int spec_id;
203
204	spec_id = __bpf_usdt_spec_id(ctx);
205	if (spec_id < 0)
206		return 0;
207
208	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
209	if (!spec)
210		return 0;
211
212	return spec->usdt_cookie;
213}
214
215/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
216#define ___bpf_usdt_args0() ctx
217#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); (void *)_x; })
218#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); (void *)_x; })
219#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); (void *)_x; })
220#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); (void *)_x; })
221#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); (void *)_x; })
222#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); (void *)_x; })
223#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); (void *)_x; })
224#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); (void *)_x; })
225#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); (void *)_x; })
226#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); (void *)_x; })
227#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); (void *)_x; })
228#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); (void *)_x; })
229#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
230
231/*
232 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
233 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
234 * Original struct pt_regs * context is preserved as 'ctx' argument.
235 */
236#define BPF_USDT(name, args...)						    \
237name(struct pt_regs *ctx);						    \
238static __always_inline typeof(name(0))					    \
239____##name(struct pt_regs *ctx, ##args);				    \
240typeof(name(0)) name(struct pt_regs *ctx)				    \
241{									    \
242        _Pragma("GCC diagnostic push")					    \
243        _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
244        return ____##name(___bpf_usdt_args(args));			    \
245        _Pragma("GCC diagnostic pop")					    \
246}									    \
247static __always_inline typeof(name(0))					    \
248____##name(struct pt_regs *ctx, ##args)
249
250#endif /* __USDT_BPF_H__ */