1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
    2
    3/*
    4 * Common eBPF ELF object loading operations.
    5 *
    6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
    7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
    8 * Copyright (C) 2015 Huawei Inc.
    9 * Copyright (C) 2017 Nicira, Inc.
   10 * Copyright (C) 2019 Isovalent, Inc.
   11 */
   12
   13#ifndef _GNU_SOURCE
   14#define _GNU_SOURCE
   15#endif
   16#include <stdlib.h>
   17#include <stdio.h>
   18#include <stdarg.h>
   19#include <libgen.h>
   20#include <inttypes.h>
   21#include <limits.h>
   22#include <string.h>
   23#include <unistd.h>
   24#include <endian.h>
   25#include <fcntl.h>
   26#include <errno.h>
   27#include <ctype.h>
   28#include <asm/unistd.h>
   29#include <linux/err.h>
   30#include <linux/kernel.h>
   31#include <linux/bpf.h>
   32#include <linux/btf.h>
   33#include <linux/filter.h>
   34#include <linux/limits.h>
   35#include <linux/perf_event.h>
   36#include <linux/ring_buffer.h>
   37#include <linux/version.h>
   38#include <sys/epoll.h>
   39#include <sys/ioctl.h>
   40#include <sys/mman.h>
   41#include <sys/stat.h>
   42#include <sys/types.h>
   43#include <sys/vfs.h>
   44#include <sys/utsname.h>
   45#include <sys/resource.h>
   46#include <libelf.h>
   47#include <gelf.h>
   48#include <zlib.h>
   49
   50#include "libbpf.h"
   51#include "bpf.h"
   52#include "btf.h"
   53#include "str_error.h"
   54#include "libbpf_internal.h"
   55#include "hashmap.h"
   56#include "bpf_gen_internal.h"
   57
   58#ifndef BPF_FS_MAGIC
   59#define BPF_FS_MAGIC		0xcafe4a11
   60#endif
   61
   62#define BPF_INSN_SZ (sizeof(struct bpf_insn))
   63
   64/* vsprintf() in __base_pr() uses nonliteral format string. It may break
   65 * compilation if user enables corresponding warning. Disable it explicitly.
   66 */
   67#pragma GCC diagnostic ignored "-Wformat-nonliteral"
   68
   69#define __printf(a, b)	__attribute__((format(printf, a, b)))
   70
   71static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
   72static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
   73
   74static const char * const attach_type_name[] = {
   75	[BPF_CGROUP_INET_INGRESS]	= "cgroup_inet_ingress",
   76	[BPF_CGROUP_INET_EGRESS]	= "cgroup_inet_egress",
   77	[BPF_CGROUP_INET_SOCK_CREATE]	= "cgroup_inet_sock_create",
   78	[BPF_CGROUP_INET_SOCK_RELEASE]	= "cgroup_inet_sock_release",
   79	[BPF_CGROUP_SOCK_OPS]		= "cgroup_sock_ops",
   80	[BPF_CGROUP_DEVICE]		= "cgroup_device",
   81	[BPF_CGROUP_INET4_BIND]		= "cgroup_inet4_bind",
   82	[BPF_CGROUP_INET6_BIND]		= "cgroup_inet6_bind",
   83	[BPF_CGROUP_INET4_CONNECT]	= "cgroup_inet4_connect",
   84	[BPF_CGROUP_INET6_CONNECT]	= "cgroup_inet6_connect",
   85	[BPF_CGROUP_INET4_POST_BIND]	= "cgroup_inet4_post_bind",
   86	[BPF_CGROUP_INET6_POST_BIND]	= "cgroup_inet6_post_bind",
   87	[BPF_CGROUP_INET4_GETPEERNAME]	= "cgroup_inet4_getpeername",
   88	[BPF_CGROUP_INET6_GETPEERNAME]	= "cgroup_inet6_getpeername",
   89	[BPF_CGROUP_INET4_GETSOCKNAME]	= "cgroup_inet4_getsockname",
   90	[BPF_CGROUP_INET6_GETSOCKNAME]	= "cgroup_inet6_getsockname",
   91	[BPF_CGROUP_UDP4_SENDMSG]	= "cgroup_udp4_sendmsg",
   92	[BPF_CGROUP_UDP6_SENDMSG]	= "cgroup_udp6_sendmsg",
   93	[BPF_CGROUP_SYSCTL]		= "cgroup_sysctl",
   94	[BPF_CGROUP_UDP4_RECVMSG]	= "cgroup_udp4_recvmsg",
   95	[BPF_CGROUP_UDP6_RECVMSG]	= "cgroup_udp6_recvmsg",
   96	[BPF_CGROUP_GETSOCKOPT]		= "cgroup_getsockopt",
   97	[BPF_CGROUP_SETSOCKOPT]		= "cgroup_setsockopt",
   98	[BPF_SK_SKB_STREAM_PARSER]	= "sk_skb_stream_parser",
   99	[BPF_SK_SKB_STREAM_VERDICT]	= "sk_skb_stream_verdict",
  100	[BPF_SK_SKB_VERDICT]		= "sk_skb_verdict",
  101	[BPF_SK_MSG_VERDICT]		= "sk_msg_verdict",
  102	[BPF_LIRC_MODE2]		= "lirc_mode2",
  103	[BPF_FLOW_DISSECTOR]		= "flow_dissector",
  104	[BPF_TRACE_RAW_TP]		= "trace_raw_tp",
  105	[BPF_TRACE_FENTRY]		= "trace_fentry",
  106	[BPF_TRACE_FEXIT]		= "trace_fexit",
  107	[BPF_MODIFY_RETURN]		= "modify_return",
  108	[BPF_LSM_MAC]			= "lsm_mac",
  109	[BPF_LSM_CGROUP]		= "lsm_cgroup",
  110	[BPF_SK_LOOKUP]			= "sk_lookup",
  111	[BPF_TRACE_ITER]		= "trace_iter",
  112	[BPF_XDP_DEVMAP]		= "xdp_devmap",
  113	[BPF_XDP_CPUMAP]		= "xdp_cpumap",
  114	[BPF_XDP]			= "xdp",
  115	[BPF_SK_REUSEPORT_SELECT]	= "sk_reuseport_select",
  116	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]	= "sk_reuseport_select_or_migrate",
  117	[BPF_PERF_EVENT]		= "perf_event",
  118	[BPF_TRACE_KPROBE_MULTI]	= "trace_kprobe_multi",
  119};
  120
  121static const char * const link_type_name[] = {
  122	[BPF_LINK_TYPE_UNSPEC]			= "unspec",
  123	[BPF_LINK_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",
  124	[BPF_LINK_TYPE_TRACING]			= "tracing",
  125	[BPF_LINK_TYPE_CGROUP]			= "cgroup",
  126	[BPF_LINK_TYPE_ITER]			= "iter",
  127	[BPF_LINK_TYPE_NETNS]			= "netns",
  128	[BPF_LINK_TYPE_XDP]			= "xdp",
  129	[BPF_LINK_TYPE_PERF_EVENT]		= "perf_event",
  130	[BPF_LINK_TYPE_KPROBE_MULTI]		= "kprobe_multi",
  131	[BPF_LINK_TYPE_STRUCT_OPS]		= "struct_ops",
  132};
  133
  134static const char * const map_type_name[] = {
  135	[BPF_MAP_TYPE_UNSPEC]			= "unspec",
  136	[BPF_MAP_TYPE_HASH]			= "hash",
  137	[BPF_MAP_TYPE_ARRAY]			= "array",
  138	[BPF_MAP_TYPE_PROG_ARRAY]		= "prog_array",
  139	[BPF_MAP_TYPE_PERF_EVENT_ARRAY]		= "perf_event_array",
  140	[BPF_MAP_TYPE_PERCPU_HASH]		= "percpu_hash",
  141	[BPF_MAP_TYPE_PERCPU_ARRAY]		= "percpu_array",
  142	[BPF_MAP_TYPE_STACK_TRACE]		= "stack_trace",
  143	[BPF_MAP_TYPE_CGROUP_ARRAY]		= "cgroup_array",
  144	[BPF_MAP_TYPE_LRU_HASH]			= "lru_hash",
  145	[BPF_MAP_TYPE_LRU_PERCPU_HASH]		= "lru_percpu_hash",
  146	[BPF_MAP_TYPE_LPM_TRIE]			= "lpm_trie",
  147	[BPF_MAP_TYPE_ARRAY_OF_MAPS]		= "array_of_maps",
  148	[BPF_MAP_TYPE_HASH_OF_MAPS]		= "hash_of_maps",
  149	[BPF_MAP_TYPE_DEVMAP]			= "devmap",
  150	[BPF_MAP_TYPE_DEVMAP_HASH]		= "devmap_hash",
  151	[BPF_MAP_TYPE_SOCKMAP]			= "sockmap",
  152	[BPF_MAP_TYPE_CPUMAP]			= "cpumap",
  153	[BPF_MAP_TYPE_XSKMAP]			= "xskmap",
  154	[BPF_MAP_TYPE_SOCKHASH]			= "sockhash",
  155	[BPF_MAP_TYPE_CGROUP_STORAGE]		= "cgroup_storage",
  156	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]	= "reuseport_sockarray",
  157	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]	= "percpu_cgroup_storage",
  158	[BPF_MAP_TYPE_QUEUE]			= "queue",
  159	[BPF_MAP_TYPE_STACK]			= "stack",
  160	[BPF_MAP_TYPE_SK_STORAGE]		= "sk_storage",
  161	[BPF_MAP_TYPE_STRUCT_OPS]		= "struct_ops",
  162	[BPF_MAP_TYPE_RINGBUF]			= "ringbuf",
  163	[BPF_MAP_TYPE_INODE_STORAGE]		= "inode_storage",
  164	[BPF_MAP_TYPE_TASK_STORAGE]		= "task_storage",
  165	[BPF_MAP_TYPE_BLOOM_FILTER]		= "bloom_filter",
  166	[BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
  167	[BPF_MAP_TYPE_CGRP_STORAGE]		= "cgrp_storage",
  168};
  169
  170static const char * const prog_type_name[] = {
  171	[BPF_PROG_TYPE_UNSPEC]			= "unspec",
  172	[BPF_PROG_TYPE_SOCKET_FILTER]		= "socket_filter",
  173	[BPF_PROG_TYPE_KPROBE]			= "kprobe",
  174	[BPF_PROG_TYPE_SCHED_CLS]		= "sched_cls",
  175	[BPF_PROG_TYPE_SCHED_ACT]		= "sched_act",
  176	[BPF_PROG_TYPE_TRACEPOINT]		= "tracepoint",
  177	[BPF_PROG_TYPE_XDP]			= "xdp",
  178	[BPF_PROG_TYPE_PERF_EVENT]		= "perf_event",
  179	[BPF_PROG_TYPE_CGROUP_SKB]		= "cgroup_skb",
  180	[BPF_PROG_TYPE_CGROUP_SOCK]		= "cgroup_sock",
  181	[BPF_PROG_TYPE_LWT_IN]			= "lwt_in",
  182	[BPF_PROG_TYPE_LWT_OUT]			= "lwt_out",
  183	[BPF_PROG_TYPE_LWT_XMIT]		= "lwt_xmit",
  184	[BPF_PROG_TYPE_SOCK_OPS]		= "sock_ops",
  185	[BPF_PROG_TYPE_SK_SKB]			= "sk_skb",
  186	[BPF_PROG_TYPE_CGROUP_DEVICE]		= "cgroup_device",
  187	[BPF_PROG_TYPE_SK_MSG]			= "sk_msg",
  188	[BPF_PROG_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",
  189	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR]	= "cgroup_sock_addr",
  190	[BPF_PROG_TYPE_LWT_SEG6LOCAL]		= "lwt_seg6local",
  191	[BPF_PROG_TYPE_LIRC_MODE2]		= "lirc_mode2",
  192	[BPF_PROG_TYPE_SK_REUSEPORT]		= "sk_reuseport",
  193	[BPF_PROG_TYPE_FLOW_DISSECTOR]		= "flow_dissector",
  194	[BPF_PROG_TYPE_CGROUP_SYSCTL]		= "cgroup_sysctl",
  195	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE]	= "raw_tracepoint_writable",
  196	[BPF_PROG_TYPE_CGROUP_SOCKOPT]		= "cgroup_sockopt",
  197	[BPF_PROG_TYPE_TRACING]			= "tracing",
  198	[BPF_PROG_TYPE_STRUCT_OPS]		= "struct_ops",
  199	[BPF_PROG_TYPE_EXT]			= "ext",
  200	[BPF_PROG_TYPE_LSM]			= "lsm",
  201	[BPF_PROG_TYPE_SK_LOOKUP]		= "sk_lookup",
  202	[BPF_PROG_TYPE_SYSCALL]			= "syscall",
  203};
  204
  205static int __base_pr(enum libbpf_print_level level, const char *format,
  206		     va_list args)
  207{
  208	if (level == LIBBPF_DEBUG)
  209		return 0;
  210
  211	return vfprintf(stderr, format, args);
  212}
  213
  214static libbpf_print_fn_t __libbpf_pr = __base_pr;
  215
  216libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
  217{
  218	libbpf_print_fn_t old_print_fn = __libbpf_pr;
  219
  220	__libbpf_pr = fn;
  221	return old_print_fn;
  222}
  223
  224__printf(2, 3)
  225void libbpf_print(enum libbpf_print_level level, const char *format, ...)
  226{
  227	va_list args;
  228	int old_errno;
  229
  230	if (!__libbpf_pr)
  231		return;
  232
  233	old_errno = errno;
  234
  235	va_start(args, format);
  236	__libbpf_pr(level, format, args);
  237	va_end(args);
 
 
  238
  239	errno = old_errno;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  240}
  241
  242static void pr_perm_msg(int err)
  243{
  244	struct rlimit limit;
  245	char buf[100];
  246
  247	if (err != -EPERM || geteuid() != 0)
  248		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  249
  250	err = getrlimit(RLIMIT_MEMLOCK, &limit);
  251	if (err)
  252		return;
  253
  254	if (limit.rlim_cur == RLIM_INFINITY)
  255		return;
  256
  257	if (limit.rlim_cur < 1024)
  258		snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
  259	else if (limit.rlim_cur < 1024*1024)
  260		snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
  261	else
  262		snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 
 
 
 
 
 
 
  263
  264	pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
  265		buf);
 
  266}
  267
  268#define STRERR_BUFSIZE  128
 
 
 
 
 
  269
  270/* Copied from tools/perf/util/util.h */
  271#ifndef zfree
  272# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
  273#endif
  274
  275#ifndef zclose
  276# define zclose(fd) ({			\
  277	int ___err = 0;			\
  278	if ((fd) >= 0)			\
  279		___err = close((fd));	\
  280	fd = -1;			\
  281	___err; })
  282#endif
  283
  284static inline __u64 ptr_to_u64(const void *ptr)
  285{
  286	return (__u64) (unsigned long) ptr;
  287}
  288
  289int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
  290{
  291	/* as of v1.0 libbpf_set_strict_mode() is a no-op */
  292	return 0;
  293}
  294
  295__u32 libbpf_major_version(void)
  296{
  297	return LIBBPF_MAJOR_VERSION;
  298}
  299
  300__u32 libbpf_minor_version(void)
  301{
  302	return LIBBPF_MINOR_VERSION;
  303}
  304
  305const char *libbpf_version_string(void)
  306{
  307#define __S(X) #X
  308#define _S(X) __S(X)
  309	return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
  310#undef _S
  311#undef __S
  312}
  313
  314enum reloc_type {
  315	RELO_LD64,
  316	RELO_CALL,
  317	RELO_DATA,
  318	RELO_EXTERN_VAR,
  319	RELO_EXTERN_FUNC,
  320	RELO_SUBPROG_ADDR,
  321	RELO_CORE,
  322};
  323
  324struct reloc_desc {
  325	enum reloc_type type;
  326	int insn_idx;
  327	union {
  328		const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
  329		struct {
  330			int map_idx;
  331			int sym_off;
  332		};
  333	};
  334};
  335
  336/* stored as sec_def->cookie for all libbpf-supported SEC()s */
  337enum sec_def_flags {
  338	SEC_NONE = 0,
  339	/* expected_attach_type is optional, if kernel doesn't support that */
  340	SEC_EXP_ATTACH_OPT = 1,
  341	/* legacy, only used by libbpf_get_type_names() and
  342	 * libbpf_attach_type_by_name(), not used by libbpf itself at all.
  343	 * This used to be associated with cgroup (and few other) BPF programs
  344	 * that were attachable through BPF_PROG_ATTACH command. Pretty
  345	 * meaningless nowadays, though.
  346	 */
  347	SEC_ATTACHABLE = 2,
  348	SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
  349	/* attachment target is specified through BTF ID in either kernel or
  350	 * other BPF program's BTF object
  351	 */
  352	SEC_ATTACH_BTF = 4,
  353	/* BPF program type allows sleeping/blocking in kernel */
  354	SEC_SLEEPABLE = 8,
  355	/* BPF program support non-linear XDP buffer */
  356	SEC_XDP_FRAGS = 16,
  357};
  358
  359struct bpf_sec_def {
  360	char *sec;
  361	enum bpf_prog_type prog_type;
  362	enum bpf_attach_type expected_attach_type;
  363	long cookie;
  364	int handler_id;
  365
  366	libbpf_prog_setup_fn_t prog_setup_fn;
  367	libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
  368	libbpf_prog_attach_fn_t prog_attach_fn;
  369};
  370
  371/*
  372 * bpf_prog should be a better name but it has been used in
  373 * linux/filter.h.
  374 */
  375struct bpf_program {
  376	char *name;
  377	char *sec_name;
  378	size_t sec_idx;
  379	const struct bpf_sec_def *sec_def;
  380	/* this program's instruction offset (in number of instructions)
  381	 * within its containing ELF section
  382	 */
  383	size_t sec_insn_off;
  384	/* number of original instructions in ELF section belonging to this
  385	 * program, not taking into account subprogram instructions possible
  386	 * appended later during relocation
  387	 */
  388	size_t sec_insn_cnt;
  389	/* Offset (in number of instructions) of the start of instruction
  390	 * belonging to this BPF program  within its containing main BPF
  391	 * program. For the entry-point (main) BPF program, this is always
  392	 * zero. For a sub-program, this gets reset before each of main BPF
  393	 * programs are processed and relocated and is used to determined
  394	 * whether sub-program was already appended to the main program, and
  395	 * if yes, at which instruction offset.
  396	 */
  397	size_t sub_insn_off;
  398
  399	/* instructions that belong to BPF program; insns[0] is located at
  400	 * sec_insn_off instruction within its ELF section in ELF file, so
  401	 * when mapping ELF file instruction index to the local instruction,
  402	 * one needs to subtract sec_insn_off; and vice versa.
  403	 */
  404	struct bpf_insn *insns;
  405	/* actual number of instruction in this BPF program's image; for
  406	 * entry-point BPF programs this includes the size of main program
  407	 * itself plus all the used sub-programs, appended at the end
  408	 */
  409	size_t insns_cnt;
  410
  411	struct reloc_desc *reloc_desc;
 
 
 
  412	int nr_reloc;
  413
  414	/* BPF verifier log settings */
  415	char *log_buf;
  416	size_t log_size;
  417	__u32 log_level;
 
  418
  419	struct bpf_object *obj;
  420
  421	int fd;
  422	bool autoload;
  423	bool autoattach;
  424	bool mark_btf_static;
  425	enum bpf_prog_type type;
  426	enum bpf_attach_type expected_attach_type;
  427
  428	int prog_ifindex;
  429	__u32 attach_btf_obj_fd;
  430	__u32 attach_btf_id;
  431	__u32 attach_prog_fd;
  432
  433	void *func_info;
  434	__u32 func_info_rec_size;
  435	__u32 func_info_cnt;
  436
  437	void *line_info;
  438	__u32 line_info_rec_size;
  439	__u32 line_info_cnt;
  440	__u32 prog_flags;
  441};
  442
  443struct bpf_struct_ops {
  444	const char *tname;
  445	const struct btf_type *type;
  446	struct bpf_program **progs;
  447	__u32 *kern_func_off;
  448	/* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
  449	void *data;
  450	/* e.g. struct bpf_struct_ops_tcp_congestion_ops in
  451	 *      btf_vmlinux's format.
  452	 * struct bpf_struct_ops_tcp_congestion_ops {
  453	 *	[... some other kernel fields ...]
  454	 *	struct tcp_congestion_ops data;
  455	 * }
  456	 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
  457	 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
  458	 * from "data".
  459	 */
  460	void *kern_vdata;
  461	__u32 type_id;
  462};
  463
  464#define DATA_SEC ".data"
  465#define BSS_SEC ".bss"
  466#define RODATA_SEC ".rodata"
  467#define KCONFIG_SEC ".kconfig"
  468#define KSYMS_SEC ".ksyms"
  469#define STRUCT_OPS_SEC ".struct_ops"
  470
  471enum libbpf_map_type {
  472	LIBBPF_MAP_UNSPEC,
  473	LIBBPF_MAP_DATA,
  474	LIBBPF_MAP_BSS,
  475	LIBBPF_MAP_RODATA,
  476	LIBBPF_MAP_KCONFIG,
  477};
  478
  479struct bpf_map_def {
  480	unsigned int type;
  481	unsigned int key_size;
  482	unsigned int value_size;
  483	unsigned int max_entries;
  484	unsigned int map_flags;
  485};
  486
  487struct bpf_map {
  488	struct bpf_object *obj;
  489	char *name;
  490	/* real_name is defined for special internal maps (.rodata*,
  491	 * .data*, .bss, .kconfig) and preserves their original ELF section
  492	 * name. This is important to be able to find corresponding BTF
  493	 * DATASEC information.
  494	 */
  495	char *real_name;
  496	int fd;
  497	int sec_idx;
  498	size_t sec_offset;
  499	int map_ifindex;
  500	int inner_map_fd;
  501	struct bpf_map_def def;
  502	__u32 numa_node;
  503	__u32 btf_var_idx;
  504	__u32 btf_key_type_id;
  505	__u32 btf_value_type_id;
  506	__u32 btf_vmlinux_value_type_id;
  507	enum libbpf_map_type libbpf_type;
  508	void *mmaped;
  509	struct bpf_struct_ops *st_ops;
  510	struct bpf_map *inner_map;
  511	void **init_slots;
  512	int init_slots_sz;
  513	char *pin_path;
  514	bool pinned;
  515	bool reused;
  516	bool autocreate;
  517	__u64 map_extra;
  518};
  519
  520enum extern_type {
  521	EXT_UNKNOWN,
  522	EXT_KCFG,
  523	EXT_KSYM,
  524};
  525
  526enum kcfg_type {
  527	KCFG_UNKNOWN,
  528	KCFG_CHAR,
  529	KCFG_BOOL,
  530	KCFG_INT,
  531	KCFG_TRISTATE,
  532	KCFG_CHAR_ARR,
  533};
  534
  535struct extern_desc {
  536	enum extern_type type;
  537	int sym_idx;
  538	int btf_id;
  539	int sec_btf_id;
  540	const char *name;
  541	bool is_set;
  542	bool is_weak;
  543	union {
  544		struct {
  545			enum kcfg_type type;
  546			int sz;
  547			int align;
  548			int data_off;
  549			bool is_signed;
  550		} kcfg;
  551		struct {
  552			unsigned long long addr;
  553
  554			/* target btf_id of the corresponding kernel var. */
  555			int kernel_btf_obj_fd;
  556			int kernel_btf_id;
  557
  558			/* local btf_id of the ksym extern's type. */
  559			__u32 type_id;
  560			/* BTF fd index to be patched in for insn->off, this is
  561			 * 0 for vmlinux BTF, index in obj->fd_array for module
  562			 * BTF
  563			 */
  564			__s16 btf_fd_idx;
  565		} ksym;
  566	};
  567};
  568
  569struct module_btf {
  570	struct btf *btf;
  571	char *name;
  572	__u32 id;
  573	int fd;
  574	int fd_array_idx;
  575};
  576
  577enum sec_type {
  578	SEC_UNUSED = 0,
  579	SEC_RELO,
  580	SEC_BSS,
  581	SEC_DATA,
  582	SEC_RODATA,
  583};
  584
  585struct elf_sec_desc {
  586	enum sec_type sec_type;
  587	Elf64_Shdr *shdr;
  588	Elf_Data *data;
  589};
  590
  591struct elf_state {
  592	int fd;
  593	const void *obj_buf;
  594	size_t obj_buf_sz;
  595	Elf *elf;
  596	Elf64_Ehdr *ehdr;
  597	Elf_Data *symbols;
  598	Elf_Data *st_ops_data;
  599	size_t shstrndx; /* section index for section name strings */
  600	size_t strtabidx;
  601	struct elf_sec_desc *secs;
  602	size_t sec_cnt;
  603	int btf_maps_shndx;
  604	__u32 btf_maps_sec_btf_id;
  605	int text_shndx;
  606	int symbols_shndx;
  607	int st_ops_shndx;
  608};
  609
  610struct usdt_manager;
  611
  612struct bpf_object {
  613	char name[BPF_OBJ_NAME_LEN];
  614	char license[64];
  615	__u32 kern_version;
  616
  617	struct bpf_program *programs;
  618	size_t nr_programs;
  619	struct bpf_map *maps;
  620	size_t nr_maps;
  621	size_t maps_cap;
  622
  623	char *kconfig;
  624	struct extern_desc *externs;
  625	int nr_extern;
  626	int kconfig_map_idx;
  627
  628	bool loaded;
  629	bool has_subcalls;
  630	bool has_rodata;
  631
  632	struct bpf_gen *gen_loader;
  633
  634	/* Information when doing ELF related work. Only valid if efile.elf is not NULL */
  635	struct elf_state efile;
  636
  637	struct btf *btf;
  638	struct btf_ext *btf_ext;
  639
  640	/* Parse and load BTF vmlinux if any of the programs in the object need
  641	 * it at load time.
  642	 */
  643	struct btf *btf_vmlinux;
  644	/* Path to the custom BTF to be used for BPF CO-RE relocations as an
  645	 * override for vmlinux BTF.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  646	 */
  647	char *btf_custom_path;
  648	/* vmlinux BTF override for CO-RE relocations */
  649	struct btf *btf_vmlinux_override;
  650	/* Lazily initialized kernel module BTFs */
  651	struct module_btf *btf_modules;
  652	bool btf_modules_loaded;
  653	size_t btf_module_cnt;
  654	size_t btf_module_cap;
  655
  656	/* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
  657	char *log_buf;
  658	size_t log_size;
  659	__u32 log_level;
  660
  661	int *fd_array;
  662	size_t fd_array_cap;
  663	size_t fd_array_cnt;
  664
  665	struct usdt_manager *usdt_man;
  666
  667	char path[];
  668};
 
  669
  670static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
  671static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
  672static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
  673static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
  674static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
  675static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
  676static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
  677static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
  678static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
  679
  680void bpf_program__unload(struct bpf_program *prog)
  681{
 
 
  682	if (!prog)
  683		return;
  684
  685	zclose(prog->fd);
 
 
 
 
 
 
 
 
 
 
  686
  687	zfree(&prog->func_info);
  688	zfree(&prog->line_info);
  689}
  690
  691static void bpf_program__exit(struct bpf_program *prog)
  692{
  693	if (!prog)
  694		return;
  695
 
 
 
 
 
 
  696	bpf_program__unload(prog);
  697	zfree(&prog->name);
  698	zfree(&prog->sec_name);
  699	zfree(&prog->insns);
  700	zfree(&prog->reloc_desc);
  701
  702	prog->nr_reloc = 0;
  703	prog->insns_cnt = 0;
  704	prog->sec_idx = -1;
  705}
  706
  707static bool insn_is_subprog_call(const struct bpf_insn *insn)
  708{
  709	return BPF_CLASS(insn->code) == BPF_JMP &&
  710	       BPF_OP(insn->code) == BPF_CALL &&
  711	       BPF_SRC(insn->code) == BPF_K &&
  712	       insn->src_reg == BPF_PSEUDO_CALL &&
  713	       insn->dst_reg == 0 &&
  714	       insn->off == 0;
  715}
  716
  717static bool is_call_insn(const struct bpf_insn *insn)
  718{
  719	return insn->code == (BPF_JMP | BPF_CALL);
  720}
  721
  722static bool insn_is_pseudo_func(struct bpf_insn *insn)
  723{
  724	return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
  725}
  726
  727static int
  728bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
  729		      const char *name, size_t sec_idx, const char *sec_name,
  730		      size_t sec_off, void *insn_data, size_t insn_data_sz)
  731{
  732	if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
  733		pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
  734			sec_name, name, sec_off, insn_data_sz);
  735		return -EINVAL;
  736	}
  737
  738	memset(prog, 0, sizeof(*prog));
  739	prog->obj = obj;
  740
  741	prog->sec_idx = sec_idx;
  742	prog->sec_insn_off = sec_off / BPF_INSN_SZ;
  743	prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
  744	/* insns_cnt can later be increased by appending used subprograms */
  745	prog->insns_cnt = prog->sec_insn_cnt;
  746
  747	prog->type = BPF_PROG_TYPE_UNSPEC;
  748	prog->fd = -1;
  749
  750	/* libbpf's convention for SEC("?abc...") is that it's just like
  751	 * SEC("abc...") but the corresponding bpf_program starts out with
  752	 * autoload set to false.
  753	 */
  754	if (sec_name[0] == '?') {
  755		prog->autoload = false;
  756		/* from now on forget there was ? in section name */
  757		sec_name++;
  758	} else {
  759		prog->autoload = true;
  760	}
  761
  762	prog->autoattach = true;
  763
  764	/* inherit object's log_level */
  765	prog->log_level = obj->log_level;
  766
  767	prog->sec_name = strdup(sec_name);
  768	if (!prog->sec_name)
  769		goto errout;
  770
  771	prog->name = strdup(name);
  772	if (!prog->name)
  773		goto errout;
 
  774
  775	prog->insns = malloc(insn_data_sz);
  776	if (!prog->insns)
 
  777		goto errout;
  778	memcpy(prog->insns, insn_data, insn_data_sz);
 
 
 
 
 
 
  779
  780	return 0;
  781errout:
  782	pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
  783	bpf_program__exit(prog);
  784	return -ENOMEM;
  785}
  786
  787static int
  788bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
  789			 const char *sec_name, int sec_idx)
  790{
  791	Elf_Data *symbols = obj->efile.symbols;
  792	struct bpf_program *prog, *progs;
  793	void *data = sec_data->d_buf;
  794	size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
  795	int nr_progs, err, i;
  796	const char *name;
  797	Elf64_Sym *sym;
  798
  799	progs = obj->programs;
  800	nr_progs = obj->nr_programs;
  801	nr_syms = symbols->d_size / sizeof(Elf64_Sym);
  802	sec_off = 0;
  803
  804	for (i = 0; i < nr_syms; i++) {
  805		sym = elf_sym_by_idx(obj, i);
  806
  807		if (sym->st_shndx != sec_idx)
  808			continue;
  809		if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
  810			continue;
  811
  812		prog_sz = sym->st_size;
  813		sec_off = sym->st_value;
  814
  815		name = elf_sym_str(obj, sym->st_name);
  816		if (!name) {
  817			pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
  818				sec_name, sec_off);
  819			return -LIBBPF_ERRNO__FORMAT;
  820		}
  821
  822		if (sec_off + prog_sz > sec_sz) {
  823			pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
  824				sec_name, sec_off);
  825			return -LIBBPF_ERRNO__FORMAT;
  826		}
  827
  828		if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
  829			pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
  830			return -ENOTSUP;
  831		}
  832
  833		pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
  834			 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
  835
  836		progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
  837		if (!progs) {
  838			/*
  839			 * In this case the original obj->programs
  840			 * is still valid, so don't need special treat for
  841			 * bpf_close_object().
  842			 */
  843			pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
  844				sec_name, name);
  845			return -ENOMEM;
  846		}
  847		obj->programs = progs;
  848
  849		prog = &progs[nr_progs];
  850
  851		err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
  852					    sec_off, data + sec_off, prog_sz);
  853		if (err)
  854			return err;
  855
  856		/* if function is a global/weak symbol, but has restricted
  857		 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
  858		 * as static to enable more permissive BPF verification mode
  859		 * with more outside context available to BPF verifier
  860		 */
  861		if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL
  862		    && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
  863			|| ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
  864			prog->mark_btf_static = true;
  865
  866		nr_progs++;
  867		obj->nr_programs = nr_progs;
  868	}
  869
  870	return 0;
  871}
  872
  873__u32 get_kernel_version(void)
  874{
  875	/* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
  876	 * but Ubuntu provides /proc/version_signature file, as described at
  877	 * https://ubuntu.com/kernel, with an example contents below, which we
  878	 * can use to get a proper LINUX_VERSION_CODE.
  879	 *
  880	 *   Ubuntu 5.4.0-12.15-generic 5.4.8
  881	 *
  882	 * In the above, 5.4.8 is what kernel is actually expecting, while
  883	 * uname() call will return 5.4.0 in info.release.
  884	 */
  885	const char *ubuntu_kver_file = "/proc/version_signature";
  886	__u32 major, minor, patch;
  887	struct utsname info;
  888
  889	if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) {
  890		FILE *f;
  891
  892		f = fopen(ubuntu_kver_file, "r");
  893		if (f) {
  894			if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) {
  895				fclose(f);
  896				return KERNEL_VERSION(major, minor, patch);
  897			}
  898			fclose(f);
  899		}
  900		/* something went wrong, fall back to uname() approach */
  901	}
  902
  903	uname(&info);
  904	if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
  905		return 0;
  906	return KERNEL_VERSION(major, minor, patch);
  907}
  908
  909static const struct btf_member *
  910find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
  911{
  912	struct btf_member *m;
  913	int i;
  914
  915	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
  916		if (btf_member_bit_offset(t, i) == bit_offset)
  917			return m;
  918	}
  919
  920	return NULL;
  921}
  922
  923static const struct btf_member *
  924find_member_by_name(const struct btf *btf, const struct btf_type *t,
  925		    const char *name)
  926{
  927	struct btf_member *m;
  928	int i;
  929
  930	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
  931		if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
  932			return m;
  933	}
  934
  935	return NULL;
  936}
  937
  938#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
  939static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
  940				   const char *name, __u32 kind);
  941
  942static int
  943find_struct_ops_kern_types(const struct btf *btf, const char *tname,
  944			   const struct btf_type **type, __u32 *type_id,
  945			   const struct btf_type **vtype, __u32 *vtype_id,
  946			   const struct btf_member **data_member)
  947{
  948	const struct btf_type *kern_type, *kern_vtype;
  949	const struct btf_member *kern_data_member;
  950	__s32 kern_vtype_id, kern_type_id;
  951	__u32 i;
  952
  953	kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
  954	if (kern_type_id < 0) {
  955		pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
  956			tname);
  957		return kern_type_id;
  958	}
  959	kern_type = btf__type_by_id(btf, kern_type_id);
  960
  961	/* Find the corresponding "map_value" type that will be used
  962	 * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
  963	 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
  964	 * btf_vmlinux.
  965	 */
  966	kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
  967						tname, BTF_KIND_STRUCT);
  968	if (kern_vtype_id < 0) {
  969		pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
  970			STRUCT_OPS_VALUE_PREFIX, tname);
  971		return kern_vtype_id;
  972	}
  973	kern_vtype = btf__type_by_id(btf, kern_vtype_id);
  974
  975	/* Find "struct tcp_congestion_ops" from
  976	 * struct bpf_struct_ops_tcp_congestion_ops {
  977	 *	[ ... ]
  978	 *	struct tcp_congestion_ops data;
  979	 * }
  980	 */
  981	kern_data_member = btf_members(kern_vtype);
  982	for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
  983		if (kern_data_member->type == kern_type_id)
  984			break;
  985	}
  986	if (i == btf_vlen(kern_vtype)) {
  987		pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
  988			tname, STRUCT_OPS_VALUE_PREFIX, tname);
  989		return -EINVAL;
  990	}
  991
  992	*type = kern_type;
  993	*type_id = kern_type_id;
  994	*vtype = kern_vtype;
  995	*vtype_id = kern_vtype_id;
  996	*data_member = kern_data_member;
  997
  998	return 0;
  999}
 1000
 1001static bool bpf_map__is_struct_ops(const struct bpf_map *map)
 1002{
 1003	return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
 1004}
 1005
 1006/* Init the map's fields that depend on kern_btf */
 1007static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
 1008					 const struct btf *btf,
 1009					 const struct btf *kern_btf)
 1010{
 1011	const struct btf_member *member, *kern_member, *kern_data_member;
 1012	const struct btf_type *type, *kern_type, *kern_vtype;
 1013	__u32 i, kern_type_id, kern_vtype_id, kern_data_off;
 1014	struct bpf_struct_ops *st_ops;
 1015	void *data, *kern_data;
 1016	const char *tname;
 1017	int err;
 1018
 1019	st_ops = map->st_ops;
 1020	type = st_ops->type;
 1021	tname = st_ops->tname;
 1022	err = find_struct_ops_kern_types(kern_btf, tname,
 1023					 &kern_type, &kern_type_id,
 1024					 &kern_vtype, &kern_vtype_id,
 1025					 &kern_data_member);
 1026	if (err)
 1027		return err;
 1028
 1029	pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
 1030		 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
 1031
 1032	map->def.value_size = kern_vtype->size;
 1033	map->btf_vmlinux_value_type_id = kern_vtype_id;
 1034
 1035	st_ops->kern_vdata = calloc(1, kern_vtype->size);
 1036	if (!st_ops->kern_vdata)
 
 
 
 
 
 
 
 
 1037		return -ENOMEM;
 1038
 1039	data = st_ops->data;
 1040	kern_data_off = kern_data_member->offset / 8;
 1041	kern_data = st_ops->kern_vdata + kern_data_off;
 1042
 1043	member = btf_members(type);
 1044	for (i = 0; i < btf_vlen(type); i++, member++) {
 1045		const struct btf_type *mtype, *kern_mtype;
 1046		__u32 mtype_id, kern_mtype_id;
 1047		void *mdata, *kern_mdata;
 1048		__s64 msize, kern_msize;
 1049		__u32 moff, kern_moff;
 1050		__u32 kern_member_idx;
 1051		const char *mname;
 1052
 1053		mname = btf__name_by_offset(btf, member->name_off);
 1054		kern_member = find_member_by_name(kern_btf, kern_type, mname);
 1055		if (!kern_member) {
 1056			pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
 1057				map->name, mname);
 1058			return -ENOTSUP;
 1059		}
 1060
 1061		kern_member_idx = kern_member - btf_members(kern_type);
 1062		if (btf_member_bitfield_size(type, i) ||
 1063		    btf_member_bitfield_size(kern_type, kern_member_idx)) {
 1064			pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
 1065				map->name, mname);
 1066			return -ENOTSUP;
 1067		}
 1068
 1069		moff = member->offset / 8;
 1070		kern_moff = kern_member->offset / 8;
 1071
 1072		mdata = data + moff;
 1073		kern_mdata = kern_data + kern_moff;
 1074
 1075		mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
 1076		kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
 1077						    &kern_mtype_id);
 1078		if (BTF_INFO_KIND(mtype->info) !=
 1079		    BTF_INFO_KIND(kern_mtype->info)) {
 1080			pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
 1081				map->name, mname, BTF_INFO_KIND(mtype->info),
 1082				BTF_INFO_KIND(kern_mtype->info));
 1083			return -ENOTSUP;
 1084		}
 1085
 1086		if (btf_is_ptr(mtype)) {
 1087			struct bpf_program *prog;
 1088
 1089			prog = st_ops->progs[i];
 1090			if (!prog)
 1091				continue;
 1092
 1093			kern_mtype = skip_mods_and_typedefs(kern_btf,
 1094							    kern_mtype->type,
 1095							    &kern_mtype_id);
 1096
 1097			/* mtype->type must be a func_proto which was
 1098			 * guaranteed in bpf_object__collect_st_ops_relos(),
 1099			 * so only check kern_mtype for func_proto here.
 1100			 */
 1101			if (!btf_is_func_proto(kern_mtype)) {
 1102				pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
 1103					map->name, mname);
 1104				return -ENOTSUP;
 1105			}
 1106
 1107			prog->attach_btf_id = kern_type_id;
 1108			prog->expected_attach_type = kern_member_idx;
 1109
 1110			st_ops->kern_func_off[i] = kern_data_off + kern_moff;
 1111
 1112			pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
 1113				 map->name, mname, prog->name, moff,
 1114				 kern_moff);
 1115
 1116			continue;
 1117		}
 1118
 1119		msize = btf__resolve_size(btf, mtype_id);
 1120		kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
 1121		if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
 1122			pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
 1123				map->name, mname, (ssize_t)msize,
 1124				(ssize_t)kern_msize);
 1125			return -ENOTSUP;
 1126		}
 1127
 1128		pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
 1129			 map->name, mname, (unsigned int)msize,
 1130			 moff, kern_moff);
 1131		memcpy(kern_mdata, mdata, msize);
 1132	}
 1133
 1134	return 0;
 1135}
 1136
 1137static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
 1138{
 1139	struct bpf_map *map;
 1140	size_t i;
 1141	int err;
 1142
 1143	for (i = 0; i < obj->nr_maps; i++) {
 1144		map = &obj->maps[i];
 1145
 1146		if (!bpf_map__is_struct_ops(map))
 1147			continue;
 1148
 1149		err = bpf_map__init_kern_struct_ops(map, obj->btf,
 1150						    obj->btf_vmlinux);
 1151		if (err)
 1152			return err;
 1153	}
 1154
 1155	return 0;
 1156}
 1157
 1158static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
 1159{
 1160	const struct btf_type *type, *datasec;
 1161	const struct btf_var_secinfo *vsi;
 1162	struct bpf_struct_ops *st_ops;
 1163	const char *tname, *var_name;
 1164	__s32 type_id, datasec_id;
 1165	const struct btf *btf;
 1166	struct bpf_map *map;
 1167	__u32 i;
 1168
 1169	if (obj->efile.st_ops_shndx == -1)
 1170		return 0;
 1171
 1172	btf = obj->btf;
 1173	datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
 1174					    BTF_KIND_DATASEC);
 1175	if (datasec_id < 0) {
 1176		pr_warn("struct_ops init: DATASEC %s not found\n",
 1177			STRUCT_OPS_SEC);
 1178		return -EINVAL;
 1179	}
 1180
 1181	datasec = btf__type_by_id(btf, datasec_id);
 1182	vsi = btf_var_secinfos(datasec);
 1183	for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
 1184		type = btf__type_by_id(obj->btf, vsi->type);
 1185		var_name = btf__name_by_offset(obj->btf, type->name_off);
 1186
 1187		type_id = btf__resolve_type(obj->btf, vsi->type);
 1188		if (type_id < 0) {
 1189			pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
 1190				vsi->type, STRUCT_OPS_SEC);
 1191			return -EINVAL;
 1192		}
 1193
 1194		type = btf__type_by_id(obj->btf, type_id);
 1195		tname = btf__name_by_offset(obj->btf, type->name_off);
 1196		if (!tname[0]) {
 1197			pr_warn("struct_ops init: anonymous type is not supported\n");
 1198			return -ENOTSUP;
 1199		}
 1200		if (!btf_is_struct(type)) {
 1201			pr_warn("struct_ops init: %s is not a struct\n", tname);
 1202			return -EINVAL;
 1203		}
 1204
 1205		map = bpf_object__add_map(obj);
 1206		if (IS_ERR(map))
 1207			return PTR_ERR(map);
 1208
 1209		map->sec_idx = obj->efile.st_ops_shndx;
 1210		map->sec_offset = vsi->offset;
 1211		map->name = strdup(var_name);
 1212		if (!map->name)
 1213			return -ENOMEM;
 1214
 1215		map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
 1216		map->def.key_size = sizeof(int);
 1217		map->def.value_size = type->size;
 1218		map->def.max_entries = 1;
 1219
 1220		map->st_ops = calloc(1, sizeof(*map->st_ops));
 1221		if (!map->st_ops)
 1222			return -ENOMEM;
 1223		st_ops = map->st_ops;
 1224		st_ops->data = malloc(type->size);
 1225		st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
 1226		st_ops->kern_func_off = malloc(btf_vlen(type) *
 1227					       sizeof(*st_ops->kern_func_off));
 1228		if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
 1229			return -ENOMEM;
 1230
 1231		if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
 1232			pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
 1233				var_name, STRUCT_OPS_SEC);
 1234			return -EINVAL;
 1235		}
 1236
 1237		memcpy(st_ops->data,
 1238		       obj->efile.st_ops_data->d_buf + vsi->offset,
 1239		       type->size);
 1240		st_ops->tname = tname;
 1241		st_ops->type = type;
 1242		st_ops->type_id = type_id;
 1243
 1244		pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
 1245			 tname, type_id, var_name, vsi->offset);
 1246	}
 1247
 
 
 
 
 
 1248	return 0;
 1249}
 1250
 1251static struct bpf_object *bpf_object__new(const char *path,
 1252					  const void *obj_buf,
 1253					  size_t obj_buf_sz,
 1254					  const char *obj_name)
 1255{
 1256	struct bpf_object *obj;
 1257	char *end;
 1258
 1259	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 1260	if (!obj) {
 1261		pr_warn("alloc memory failed for %s\n", path);
 1262		return ERR_PTR(-ENOMEM);
 1263	}
 1264
 1265	strcpy(obj->path, path);
 1266	if (obj_name) {
 1267		libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
 1268	} else {
 1269		/* Using basename() GNU version which doesn't modify arg. */
 1270		libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
 1271		end = strchr(obj->name, '.');
 1272		if (end)
 1273			*end = 0;
 1274	}
 1275
 1276	obj->efile.fd = -1;
 
 1277	/*
 1278	 * Caller of this function should also call
 1279	 * bpf_object__elf_finish() after data collection to return
 1280	 * obj_buf to user. If not, we should duplicate the buffer to
 1281	 * avoid user freeing them before elf finish.
 1282	 */
 1283	obj->efile.obj_buf = obj_buf;
 1284	obj->efile.obj_buf_sz = obj_buf_sz;
 1285	obj->efile.btf_maps_shndx = -1;
 1286	obj->efile.st_ops_shndx = -1;
 1287	obj->kconfig_map_idx = -1;
 1288
 1289	obj->kern_version = get_kernel_version();
 1290	obj->loaded = false;
 1291
 
 
 1292	return obj;
 1293}
 1294
 1295static void bpf_object__elf_finish(struct bpf_object *obj)
 1296{
 1297	if (!obj->efile.elf)
 1298		return;
 1299
 1300	elf_end(obj->efile.elf);
 1301	obj->efile.elf = NULL;
 
 
 1302	obj->efile.symbols = NULL;
 1303	obj->efile.st_ops_data = NULL;
 1304
 1305	zfree(&obj->efile.secs);
 1306	obj->efile.sec_cnt = 0;
 1307	zclose(obj->efile.fd);
 1308	obj->efile.obj_buf = NULL;
 1309	obj->efile.obj_buf_sz = 0;
 1310}
 1311
 1312static int bpf_object__elf_init(struct bpf_object *obj)
 1313{
 1314	Elf64_Ehdr *ehdr;
 1315	int err = 0;
 1316	Elf *elf;
 1317
 1318	if (obj->efile.elf) {
 1319		pr_warn("elf: init internal error\n");
 1320		return -LIBBPF_ERRNO__LIBELF;
 1321	}
 1322
 1323	if (obj->efile.obj_buf_sz > 0) {
 1324		/* obj_buf should have been validated by bpf_object__open_mem(). */
 1325		elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
 
 
 
 
 1326	} else {
 1327		obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
 1328		if (obj->efile.fd < 0) {
 1329			char errmsg[STRERR_BUFSIZE], *cp;
 1330
 1331			err = -errno;
 1332			cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 1333			pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
 1334			return err;
 1335		}
 1336
 1337		elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
 
 
 1338	}
 1339
 1340	if (!elf) {
 1341		pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
 
 1342		err = -LIBBPF_ERRNO__LIBELF;
 1343		goto errout;
 1344	}
 1345
 1346	obj->efile.elf = elf;
 1347
 1348	if (elf_kind(elf) != ELF_K_ELF) {
 1349		err = -LIBBPF_ERRNO__FORMAT;
 1350		pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
 1351		goto errout;
 1352	}
 1353
 1354	if (gelf_getclass(elf) != ELFCLASS64) {
 1355		err = -LIBBPF_ERRNO__FORMAT;
 1356		pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
 1357		goto errout;
 1358	}
 1359
 1360	obj->efile.ehdr = ehdr = elf64_getehdr(elf);
 1361	if (!obj->efile.ehdr) {
 1362		pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
 1363		err = -LIBBPF_ERRNO__FORMAT;
 1364		goto errout;
 1365	}
 
 1366
 1367	if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
 1368		pr_warn("elf: failed to get section names section index for %s: %s\n",
 1369			obj->path, elf_errmsg(-1));
 1370		err = -LIBBPF_ERRNO__FORMAT;
 1371		goto errout;
 1372	}
 1373
 1374	/* Elf is corrupted/truncated, avoid calling elf_strptr. */
 1375	if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
 1376		pr_warn("elf: failed to get section names strings from %s: %s\n",
 1377			obj->path, elf_errmsg(-1));
 1378		err = -LIBBPF_ERRNO__FORMAT;
 1379		goto errout;
 1380	}
 1381
 1382	/* Old LLVM set e_machine to EM_NONE */
 1383	if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
 1384		pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
 1385		err = -LIBBPF_ERRNO__FORMAT;
 1386		goto errout;
 1387	}
 1388
 1389	return 0;
 1390errout:
 1391	bpf_object__elf_finish(obj);
 1392	return err;
 1393}
 1394
 1395static int bpf_object__check_endianness(struct bpf_object *obj)
 1396{
 1397#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 1398	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
 1399		return 0;
 1400#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 1401	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
 1402		return 0;
 1403#else
 1404# error "Unrecognized __BYTE_ORDER__"
 1405#endif
 1406	pr_warn("elf: endianness mismatch in %s.\n", obj->path);
 1407	return -LIBBPF_ERRNO__ENDIAN;
 1408}
 1409
 1410static int
 1411bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
 1412{
 1413	if (!data) {
 1414		pr_warn("invalid license section in %s\n", obj->path);
 1415		return -LIBBPF_ERRNO__FORMAT;
 1416	}
 1417	/* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
 1418	 * go over allowed ELF data section buffer
 1419	 */
 1420	libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
 1421	pr_debug("license of %s is %s\n", obj->path, obj->license);
 1422	return 0;
 1423}
 1424
 1425static int
 1426bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
 1427{
 1428	__u32 kver;
 1429
 1430	if (!data || size != sizeof(kver)) {
 1431		pr_warn("invalid kver section in %s\n", obj->path);
 1432		return -LIBBPF_ERRNO__FORMAT;
 1433	}
 1434	memcpy(&kver, data, sizeof(kver));
 1435	obj->kern_version = kver;
 1436	pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
 1437	return 0;
 1438}
 1439
 1440static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
 1441{
 1442	if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
 1443	    type == BPF_MAP_TYPE_HASH_OF_MAPS)
 1444		return true;
 1445	return false;
 1446}
 1447
 1448static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
 1449{
 1450	Elf_Data *data;
 1451	Elf_Scn *scn;
 1452
 1453	if (!name)
 1454		return -EINVAL;
 1455
 1456	scn = elf_sec_by_name(obj, name);
 1457	data = elf_sec_data(obj, scn);
 1458	if (data) {
 1459		*size = data->d_size;
 1460		return 0; /* found it */
 1461	}
 1462
 1463	return -ENOENT;
 1464}
 1465
 1466static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
 1467{
 1468	Elf_Data *symbols = obj->efile.symbols;
 1469	const char *sname;
 1470	size_t si;
 1471
 1472	for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
 1473		Elf64_Sym *sym = elf_sym_by_idx(obj, si);
 1474
 1475		if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
 1476			continue;
 1477
 1478		if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
 1479		    ELF64_ST_BIND(sym->st_info) != STB_WEAK)
 1480			continue;
 1481
 1482		sname = elf_sym_str(obj, sym->st_name);
 1483		if (!sname) {
 1484			pr_warn("failed to get sym name string for var %s\n", name);
 1485			return ERR_PTR(-EIO);
 1486		}
 1487		if (strcmp(name, sname) == 0)
 1488			return sym;
 1489	}
 1490
 1491	return ERR_PTR(-ENOENT);
 1492}
 1493
 1494static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
 1495{
 1496	struct bpf_map *map;
 1497	int err;
 1498
 1499	err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
 1500				sizeof(*obj->maps), obj->nr_maps + 1);
 1501	if (err)
 1502		return ERR_PTR(err);
 1503
 1504	map = &obj->maps[obj->nr_maps++];
 1505	map->obj = obj;
 1506	map->fd = -1;
 1507	map->inner_map_fd = -1;
 1508	map->autocreate = true;
 1509
 1510	return map;
 1511}
 1512
 1513static size_t bpf_map_mmap_sz(const struct bpf_map *map)
 1514{
 1515	long page_sz = sysconf(_SC_PAGE_SIZE);
 1516	size_t map_sz;
 1517
 1518	map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
 1519	map_sz = roundup(map_sz, page_sz);
 1520	return map_sz;
 1521}
 1522
 1523static char *internal_map_name(struct bpf_object *obj, const char *real_name)
 1524{
 1525	char map_name[BPF_OBJ_NAME_LEN], *p;
 1526	int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
 1527
 1528	/* This is one of the more confusing parts of libbpf for various
 1529	 * reasons, some of which are historical. The original idea for naming
 1530	 * internal names was to include as much of BPF object name prefix as
 1531	 * possible, so that it can be distinguished from similar internal
 1532	 * maps of a different BPF object.
 1533	 * As an example, let's say we have bpf_object named 'my_object_name'
 1534	 * and internal map corresponding to '.rodata' ELF section. The final
 1535	 * map name advertised to user and to the kernel will be
 1536	 * 'my_objec.rodata', taking first 8 characters of object name and
 1537	 * entire 7 characters of '.rodata'.
 1538	 * Somewhat confusingly, if internal map ELF section name is shorter
 1539	 * than 7 characters, e.g., '.bss', we still reserve 7 characters
 1540	 * for the suffix, even though we only have 4 actual characters, and
 1541	 * resulting map will be called 'my_objec.bss', not even using all 15
 1542	 * characters allowed by the kernel. Oh well, at least the truncated
 1543	 * object name is somewhat consistent in this case. But if the map
 1544	 * name is '.kconfig', we'll still have entirety of '.kconfig' added
 1545	 * (8 chars) and thus will be left with only first 7 characters of the
 1546	 * object name ('my_obje'). Happy guessing, user, that the final map
 1547	 * name will be "my_obje.kconfig".
 1548	 * Now, with libbpf starting to support arbitrarily named .rodata.*
 1549	 * and .data.* data sections, it's possible that ELF section name is
 1550	 * longer than allowed 15 chars, so we now need to be careful to take
 1551	 * only up to 15 first characters of ELF name, taking no BPF object
 1552	 * name characters at all. So '.rodata.abracadabra' will result in
 1553	 * '.rodata.abracad' kernel and user-visible name.
 1554	 * We need to keep this convoluted logic intact for .data, .bss and
 1555	 * .rodata maps, but for new custom .data.custom and .rodata.custom
 1556	 * maps we use their ELF names as is, not prepending bpf_object name
 1557	 * in front. We still need to truncate them to 15 characters for the
 1558	 * kernel. Full name can be recovered for such maps by using DATASEC
 1559	 * BTF type associated with such map's value type, though.
 1560	 */
 1561	if (sfx_len >= BPF_OBJ_NAME_LEN)
 1562		sfx_len = BPF_OBJ_NAME_LEN - 1;
 1563
 1564	/* if there are two or more dots in map name, it's a custom dot map */
 1565	if (strchr(real_name + 1, '.') != NULL)
 1566		pfx_len = 0;
 1567	else
 1568		pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
 1569
 1570	snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
 1571		 sfx_len, real_name);
 1572
 1573	/* sanitise map name to characters allowed by kernel */
 1574	for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
 1575		if (!isalnum(*p) && *p != '_' && *p != '.')
 1576			*p = '_';
 1577
 1578	return strdup(map_name);
 1579}
 1580
 1581static int
 1582map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
 1583
 1584/* Internal BPF map is mmap()'able only if at least one of corresponding
 1585 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
 1586 * variable and it's not marked as __hidden (which turns it into, effectively,
 1587 * a STATIC variable).
 1588 */
 1589static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
 1590{
 1591	const struct btf_type *t, *vt;
 1592	struct btf_var_secinfo *vsi;
 1593	int i, n;
 1594
 1595	if (!map->btf_value_type_id)
 1596		return false;
 1597
 1598	t = btf__type_by_id(obj->btf, map->btf_value_type_id);
 1599	if (!btf_is_datasec(t))
 1600		return false;
 1601
 1602	vsi = btf_var_secinfos(t);
 1603	for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
 1604		vt = btf__type_by_id(obj->btf, vsi->type);
 1605		if (!btf_is_var(vt))
 1606			continue;
 1607
 1608		if (btf_var(vt)->linkage != BTF_VAR_STATIC)
 1609			return true;
 1610	}
 1611
 1612	return false;
 1613}
 1614
 1615static int
 1616bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 1617			      const char *real_name, int sec_idx, void *data, size_t data_sz)
 1618{
 1619	struct bpf_map_def *def;
 1620	struct bpf_map *map;
 1621	int err;
 1622
 1623	map = bpf_object__add_map(obj);
 1624	if (IS_ERR(map))
 1625		return PTR_ERR(map);
 1626
 1627	map->libbpf_type = type;
 1628	map->sec_idx = sec_idx;
 1629	map->sec_offset = 0;
 1630	map->real_name = strdup(real_name);
 1631	map->name = internal_map_name(obj, real_name);
 1632	if (!map->real_name || !map->name) {
 1633		zfree(&map->real_name);
 1634		zfree(&map->name);
 1635		return -ENOMEM;
 1636	}
 1637
 1638	def = &map->def;
 1639	def->type = BPF_MAP_TYPE_ARRAY;
 1640	def->key_size = sizeof(int);
 1641	def->value_size = data_sz;
 1642	def->max_entries = 1;
 1643	def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
 1644			 ? BPF_F_RDONLY_PROG : 0;
 1645
 1646	/* failures are fine because of maps like .rodata.str1.1 */
 1647	(void) map_fill_btf_type_info(obj, map);
 1648
 1649	if (map_is_mmapable(obj, map))
 1650		def->map_flags |= BPF_F_MMAPABLE;
 1651
 1652	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
 1653		 map->name, map->sec_idx, map->sec_offset, def->map_flags);
 1654
 1655	map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
 1656			   MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 1657	if (map->mmaped == MAP_FAILED) {
 1658		err = -errno;
 1659		map->mmaped = NULL;
 1660		pr_warn("failed to alloc map '%s' content buffer: %d\n",
 1661			map->name, err);
 1662		zfree(&map->real_name);
 1663		zfree(&map->name);
 1664		return err;
 1665	}
 1666
 1667	if (data)
 1668		memcpy(map->mmaped, data, data_sz);
 1669
 1670	pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
 1671	return 0;
 1672}
 1673
 1674static int bpf_object__init_global_data_maps(struct bpf_object *obj)
 1675{
 1676	struct elf_sec_desc *sec_desc;
 1677	const char *sec_name;
 1678	int err = 0, sec_idx;
 1679
 1680	/*
 1681	 * Populate obj->maps with libbpf internal maps.
 1682	 */
 1683	for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
 1684		sec_desc = &obj->efile.secs[sec_idx];
 1685
 1686		/* Skip recognized sections with size 0. */
 1687		if (!sec_desc->data || sec_desc->data->d_size == 0)
 1688			continue;
 1689
 1690		switch (sec_desc->sec_type) {
 1691		case SEC_DATA:
 1692			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
 1693			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
 1694							    sec_name, sec_idx,
 1695							    sec_desc->data->d_buf,
 1696							    sec_desc->data->d_size);
 1697			break;
 1698		case SEC_RODATA:
 1699			obj->has_rodata = true;
 1700			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
 1701			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
 1702							    sec_name, sec_idx,
 1703							    sec_desc->data->d_buf,
 1704							    sec_desc->data->d_size);
 1705			break;
 1706		case SEC_BSS:
 1707			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
 1708			err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
 1709							    sec_name, sec_idx,
 1710							    NULL,
 1711							    sec_desc->data->d_size);
 1712			break;
 1713		default:
 1714			/* skip */
 1715			break;
 1716		}
 1717		if (err)
 1718			return err;
 1719	}
 1720	return 0;
 1721}
 1722
 1723
 1724static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
 1725					       const void *name)
 1726{
 1727	int i;
 1728
 1729	for (i = 0; i < obj->nr_extern; i++) {
 1730		if (strcmp(obj->externs[i].name, name) == 0)
 1731			return &obj->externs[i];
 1732	}
 1733	return NULL;
 1734}
 1735
 1736static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
 1737			      char value)
 1738{
 1739	switch (ext->kcfg.type) {
 1740	case KCFG_BOOL:
 1741		if (value == 'm') {
 1742			pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
 1743				ext->name, value);
 1744			return -EINVAL;
 1745		}
 1746		*(bool *)ext_val = value == 'y' ? true : false;
 1747		break;
 1748	case KCFG_TRISTATE:
 1749		if (value == 'y')
 1750			*(enum libbpf_tristate *)ext_val = TRI_YES;
 1751		else if (value == 'm')
 1752			*(enum libbpf_tristate *)ext_val = TRI_MODULE;
 1753		else /* value == 'n' */
 1754			*(enum libbpf_tristate *)ext_val = TRI_NO;
 1755		break;
 1756	case KCFG_CHAR:
 1757		*(char *)ext_val = value;
 1758		break;
 1759	case KCFG_UNKNOWN:
 1760	case KCFG_INT:
 1761	case KCFG_CHAR_ARR:
 1762	default:
 1763		pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
 1764			ext->name, value);
 1765		return -EINVAL;
 1766	}
 1767	ext->is_set = true;
 1768	return 0;
 1769}
 1770
 1771static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
 1772			      const char *value)
 1773{
 1774	size_t len;
 1775
 1776	if (ext->kcfg.type != KCFG_CHAR_ARR) {
 1777		pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
 1778			ext->name, value);
 1779		return -EINVAL;
 1780	}
 1781
 1782	len = strlen(value);
 1783	if (value[len - 1] != '"') {
 1784		pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
 1785			ext->name, value);
 1786		return -EINVAL;
 1787	}
 1788
 1789	/* strip quotes */
 1790	len -= 2;
 1791	if (len >= ext->kcfg.sz) {
 1792		pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
 1793			ext->name, value, len, ext->kcfg.sz - 1);
 1794		len = ext->kcfg.sz - 1;
 1795	}
 1796	memcpy(ext_val, value + 1, len);
 1797	ext_val[len] = '\0';
 1798	ext->is_set = true;
 1799	return 0;
 1800}
 1801
 1802static int parse_u64(const char *value, __u64 *res)
 1803{
 1804	char *value_end;
 1805	int err;
 1806
 1807	errno = 0;
 1808	*res = strtoull(value, &value_end, 0);
 1809	if (errno) {
 1810		err = -errno;
 1811		pr_warn("failed to parse '%s' as integer: %d\n", value, err);
 1812		return err;
 1813	}
 1814	if (*value_end) {
 1815		pr_warn("failed to parse '%s' as integer completely\n", value);
 1816		return -EINVAL;
 1817	}
 1818	return 0;
 1819}
 1820
 1821static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
 1822{
 1823	int bit_sz = ext->kcfg.sz * 8;
 1824
 1825	if (ext->kcfg.sz == 8)
 1826		return true;
 1827
 1828	/* Validate that value stored in u64 fits in integer of `ext->sz`
 1829	 * bytes size without any loss of information. If the target integer
 1830	 * is signed, we rely on the following limits of integer type of
 1831	 * Y bits and subsequent transformation:
 1832	 *
 1833	 *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
 1834	 *            0 <= X + 2^(Y-1) <= 2^Y - 1
 1835	 *            0 <= X + 2^(Y-1) <  2^Y
 1836	 *
 1837	 *  For unsigned target integer, check that all the (64 - Y) bits are
 1838	 *  zero.
 1839	 */
 1840	if (ext->kcfg.is_signed)
 1841		return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
 1842	else
 1843		return (v >> bit_sz) == 0;
 1844}
 1845
 1846static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
 1847			      __u64 value)
 1848{
 1849	if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
 1850	    ext->kcfg.type != KCFG_BOOL) {
 1851		pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
 1852			ext->name, (unsigned long long)value);
 1853		return -EINVAL;
 1854	}
 1855	if (ext->kcfg.type == KCFG_BOOL && value > 1) {
 1856		pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
 1857			ext->name, (unsigned long long)value);
 1858		return -EINVAL;
 1859
 1860	}
 1861	if (!is_kcfg_value_in_range(ext, value)) {
 1862		pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
 1863			ext->name, (unsigned long long)value, ext->kcfg.sz);
 1864		return -ERANGE;
 1865	}
 1866	switch (ext->kcfg.sz) {
 1867	case 1:
 1868		*(__u8 *)ext_val = value;
 1869		break;
 1870	case 2:
 1871		*(__u16 *)ext_val = value;
 1872		break;
 1873	case 4:
 1874		*(__u32 *)ext_val = value;
 1875		break;
 1876	case 8:
 1877		*(__u64 *)ext_val = value;
 1878		break;
 1879	default:
 1880		return -EINVAL;
 1881	}
 1882	ext->is_set = true;
 1883	return 0;
 1884}
 1885
 1886static int bpf_object__process_kconfig_line(struct bpf_object *obj,
 1887					    char *buf, void *data)
 1888{
 1889	struct extern_desc *ext;
 1890	char *sep, *value;
 1891	int len, err = 0;
 1892	void *ext_val;
 1893	__u64 num;
 1894
 1895	if (!str_has_pfx(buf, "CONFIG_"))
 1896		return 0;
 1897
 1898	sep = strchr(buf, '=');
 1899	if (!sep) {
 1900		pr_warn("failed to parse '%s': no separator\n", buf);
 1901		return -EINVAL;
 1902	}
 1903
 1904	/* Trim ending '\n' */
 1905	len = strlen(buf);
 1906	if (buf[len - 1] == '\n')
 1907		buf[len - 1] = '\0';
 1908	/* Split on '=' and ensure that a value is present. */
 1909	*sep = '\0';
 1910	if (!sep[1]) {
 1911		*sep = '=';
 1912		pr_warn("failed to parse '%s': no value\n", buf);
 1913		return -EINVAL;
 1914	}
 1915
 1916	ext = find_extern_by_name(obj, buf);
 1917	if (!ext || ext->is_set)
 1918		return 0;
 1919
 1920	ext_val = data + ext->kcfg.data_off;
 1921	value = sep + 1;
 1922
 1923	switch (*value) {
 1924	case 'y': case 'n': case 'm':
 1925		err = set_kcfg_value_tri(ext, ext_val, *value);
 1926		break;
 1927	case '"':
 1928		err = set_kcfg_value_str(ext, ext_val, value);
 1929		break;
 1930	default:
 1931		/* assume integer */
 1932		err = parse_u64(value, &num);
 1933		if (err) {
 1934			pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
 1935			return err;
 1936		}
 1937		if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
 1938			pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
 1939			return -EINVAL;
 1940		}
 1941		err = set_kcfg_value_num(ext, ext_val, num);
 1942		break;
 1943	}
 1944	if (err)
 1945		return err;
 1946	pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
 1947	return 0;
 1948}
 1949
 1950static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
 1951{
 1952	char buf[PATH_MAX];
 1953	struct utsname uts;
 1954	int len, err = 0;
 1955	gzFile file;
 1956
 1957	uname(&uts);
 1958	len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
 1959	if (len < 0)
 1960		return -EINVAL;
 1961	else if (len >= PATH_MAX)
 1962		return -ENAMETOOLONG;
 1963
 1964	/* gzopen also accepts uncompressed files. */
 1965	file = gzopen(buf, "r");
 1966	if (!file)
 1967		file = gzopen("/proc/config.gz", "r");
 1968
 1969	if (!file) {
 1970		pr_warn("failed to open system Kconfig\n");
 1971		return -ENOENT;
 1972	}
 1973
 1974	while (gzgets(file, buf, sizeof(buf))) {
 1975		err = bpf_object__process_kconfig_line(obj, buf, data);
 1976		if (err) {
 1977			pr_warn("error parsing system Kconfig line '%s': %d\n",
 1978				buf, err);
 1979			goto out;
 1980		}
 1981	}
 1982
 1983out:
 1984	gzclose(file);
 1985	return err;
 1986}
 1987
 1988static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
 1989					const char *config, void *data)
 1990{
 1991	char buf[PATH_MAX];
 1992	int err = 0;
 1993	FILE *file;
 1994
 1995	file = fmemopen((void *)config, strlen(config), "r");
 1996	if (!file) {
 1997		err = -errno;
 1998		pr_warn("failed to open in-memory Kconfig: %d\n", err);
 1999		return err;
 2000	}
 2001
 2002	while (fgets(buf, sizeof(buf), file)) {
 2003		err = bpf_object__process_kconfig_line(obj, buf, data);
 2004		if (err) {
 2005			pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
 2006				buf, err);
 2007			break;
 2008		}
 2009	}
 2010
 2011	fclose(file);
 2012	return err;
 2013}
 2014
 2015static int bpf_object__init_kconfig_map(struct bpf_object *obj)
 2016{
 2017	struct extern_desc *last_ext = NULL, *ext;
 2018	size_t map_sz;
 2019	int i, err;
 2020
 2021	for (i = 0; i < obj->nr_extern; i++) {
 2022		ext = &obj->externs[i];
 2023		if (ext->type == EXT_KCFG)
 2024			last_ext = ext;
 2025	}
 2026
 2027	if (!last_ext)
 2028		return 0;
 2029
 2030	map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
 2031	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
 2032					    ".kconfig", obj->efile.symbols_shndx,
 2033					    NULL, map_sz);
 2034	if (err)
 2035		return err;
 2036
 2037	obj->kconfig_map_idx = obj->nr_maps - 1;
 2038
 2039	return 0;
 2040}
 2041
 2042const struct btf_type *
 2043skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
 2044{
 2045	const struct btf_type *t = btf__type_by_id(btf, id);
 2046
 2047	if (res_id)
 2048		*res_id = id;
 2049
 2050	while (btf_is_mod(t) || btf_is_typedef(t)) {
 2051		if (res_id)
 2052			*res_id = t->type;
 2053		t = btf__type_by_id(btf, t->type);
 2054	}
 2055
 2056	return t;
 2057}
 2058
 2059static const struct btf_type *
 2060resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
 2061{
 2062	const struct btf_type *t;
 2063
 2064	t = skip_mods_and_typedefs(btf, id, NULL);
 2065	if (!btf_is_ptr(t))
 2066		return NULL;
 2067
 2068	t = skip_mods_and_typedefs(btf, t->type, res_id);
 2069
 2070	return btf_is_func_proto(t) ? t : NULL;
 2071}
 2072
 2073static const char *__btf_kind_str(__u16 kind)
 2074{
 2075	switch (kind) {
 2076	case BTF_KIND_UNKN: return "void";
 2077	case BTF_KIND_INT: return "int";
 2078	case BTF_KIND_PTR: return "ptr";
 2079	case BTF_KIND_ARRAY: return "array";
 2080	case BTF_KIND_STRUCT: return "struct";
 2081	case BTF_KIND_UNION: return "union";
 2082	case BTF_KIND_ENUM: return "enum";
 2083	case BTF_KIND_FWD: return "fwd";
 2084	case BTF_KIND_TYPEDEF: return "typedef";
 2085	case BTF_KIND_VOLATILE: return "volatile";
 2086	case BTF_KIND_CONST: return "const";
 2087	case BTF_KIND_RESTRICT: return "restrict";
 2088	case BTF_KIND_FUNC: return "func";
 2089	case BTF_KIND_FUNC_PROTO: return "func_proto";
 2090	case BTF_KIND_VAR: return "var";
 2091	case BTF_KIND_DATASEC: return "datasec";
 2092	case BTF_KIND_FLOAT: return "float";
 2093	case BTF_KIND_DECL_TAG: return "decl_tag";
 2094	case BTF_KIND_TYPE_TAG: return "type_tag";
 2095	case BTF_KIND_ENUM64: return "enum64";
 2096	default: return "unknown";
 2097	}
 2098}
 2099
 2100const char *btf_kind_str(const struct btf_type *t)
 2101{
 2102	return __btf_kind_str(btf_kind(t));
 2103}
 2104
 2105/*
 2106 * Fetch integer attribute of BTF map definition. Such attributes are
 2107 * represented using a pointer to an array, in which dimensionality of array
 2108 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
 2109 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
 2110 * type definition, while using only sizeof(void *) space in ELF data section.
 2111 */
 2112static bool get_map_field_int(const char *map_name, const struct btf *btf,
 2113			      const struct btf_member *m, __u32 *res)
 2114{
 2115	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
 2116	const char *name = btf__name_by_offset(btf, m->name_off);
 2117	const struct btf_array *arr_info;
 2118	const struct btf_type *arr_t;
 2119
 2120	if (!btf_is_ptr(t)) {
 2121		pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
 2122			map_name, name, btf_kind_str(t));
 2123		return false;
 2124	}
 2125
 2126	arr_t = btf__type_by_id(btf, t->type);
 2127	if (!arr_t) {
 2128		pr_warn("map '%s': attr '%s': type [%u] not found.\n",
 2129			map_name, name, t->type);
 2130		return false;
 2131	}
 2132	if (!btf_is_array(arr_t)) {
 2133		pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
 2134			map_name, name, btf_kind_str(arr_t));
 2135		return false;
 2136	}
 2137	arr_info = btf_array(arr_t);
 2138	*res = arr_info->nelems;
 2139	return true;
 2140}
 2141
 2142static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
 
 
 2143{
 2144	int len;
 2145
 2146	len = snprintf(buf, buf_sz, "%s/%s", path, name);
 2147	if (len < 0)
 2148		return -EINVAL;
 2149	if (len >= buf_sz)
 2150		return -ENAMETOOLONG;
 2151
 2152	return 0;
 2153}
 2154
 2155static int build_map_pin_path(struct bpf_map *map, const char *path)
 
 
 2156{
 2157	char buf[PATH_MAX];
 2158	int err;
 2159
 2160	if (!path)
 2161		path = "/sys/fs/bpf";
 2162
 2163	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
 2164	if (err)
 2165		return err;
 2166
 2167	return bpf_map__set_pin_path(map, buf);
 2168}
 2169
 2170/* should match definition in bpf_helpers.h */
 2171enum libbpf_pin_type {
 2172	LIBBPF_PIN_NONE,
 2173	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
 2174	LIBBPF_PIN_BY_NAME,
 2175};
 2176
 2177int parse_btf_map_def(const char *map_name, struct btf *btf,
 2178		      const struct btf_type *def_t, bool strict,
 2179		      struct btf_map_def *map_def, struct btf_map_def *inner_def)
 2180{
 2181	const struct btf_type *t;
 2182	const struct btf_member *m;
 2183	bool is_inner = inner_def == NULL;
 2184	int vlen, i;
 2185
 2186	vlen = btf_vlen(def_t);
 2187	m = btf_members(def_t);
 2188	for (i = 0; i < vlen; i++, m++) {
 2189		const char *name = btf__name_by_offset(btf, m->name_off);
 2190
 2191		if (!name) {
 2192			pr_warn("map '%s': invalid field #%d.\n", map_name, i);
 2193			return -EINVAL;
 2194		}
 2195		if (strcmp(name, "type") == 0) {
 2196			if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
 2197				return -EINVAL;
 2198			map_def->parts |= MAP_DEF_MAP_TYPE;
 2199		} else if (strcmp(name, "max_entries") == 0) {
 2200			if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
 2201				return -EINVAL;
 2202			map_def->parts |= MAP_DEF_MAX_ENTRIES;
 2203		} else if (strcmp(name, "map_flags") == 0) {
 2204			if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
 2205				return -EINVAL;
 2206			map_def->parts |= MAP_DEF_MAP_FLAGS;
 2207		} else if (strcmp(name, "numa_node") == 0) {
 2208			if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
 2209				return -EINVAL;
 2210			map_def->parts |= MAP_DEF_NUMA_NODE;
 2211		} else if (strcmp(name, "key_size") == 0) {
 2212			__u32 sz;
 2213
 2214			if (!get_map_field_int(map_name, btf, m, &sz))
 2215				return -EINVAL;
 2216			if (map_def->key_size && map_def->key_size != sz) {
 2217				pr_warn("map '%s': conflicting key size %u != %u.\n",
 2218					map_name, map_def->key_size, sz);
 2219				return -EINVAL;
 2220			}
 2221			map_def->key_size = sz;
 2222			map_def->parts |= MAP_DEF_KEY_SIZE;
 2223		} else if (strcmp(name, "key") == 0) {
 2224			__s64 sz;
 2225
 2226			t = btf__type_by_id(btf, m->type);
 2227			if (!t) {
 2228				pr_warn("map '%s': key type [%d] not found.\n",
 2229					map_name, m->type);
 2230				return -EINVAL;
 2231			}
 2232			if (!btf_is_ptr(t)) {
 2233				pr_warn("map '%s': key spec is not PTR: %s.\n",
 2234					map_name, btf_kind_str(t));
 2235				return -EINVAL;
 2236			}
 2237			sz = btf__resolve_size(btf, t->type);
 2238			if (sz < 0) {
 2239				pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
 2240					map_name, t->type, (ssize_t)sz);
 2241				return sz;
 2242			}
 2243			if (map_def->key_size && map_def->key_size != sz) {
 2244				pr_warn("map '%s': conflicting key size %u != %zd.\n",
 2245					map_name, map_def->key_size, (ssize_t)sz);
 2246				return -EINVAL;
 2247			}
 2248			map_def->key_size = sz;
 2249			map_def->key_type_id = t->type;
 2250			map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
 2251		} else if (strcmp(name, "value_size") == 0) {
 2252			__u32 sz;
 2253
 2254			if (!get_map_field_int(map_name, btf, m, &sz))
 2255				return -EINVAL;
 2256			if (map_def->value_size && map_def->value_size != sz) {
 2257				pr_warn("map '%s': conflicting value size %u != %u.\n",
 2258					map_name, map_def->value_size, sz);
 2259				return -EINVAL;
 2260			}
 2261			map_def->value_size = sz;
 2262			map_def->parts |= MAP_DEF_VALUE_SIZE;
 2263		} else if (strcmp(name, "value") == 0) {
 2264			__s64 sz;
 2265
 2266			t = btf__type_by_id(btf, m->type);
 2267			if (!t) {
 2268				pr_warn("map '%s': value type [%d] not found.\n",
 2269					map_name, m->type);
 2270				return -EINVAL;
 2271			}
 2272			if (!btf_is_ptr(t)) {
 2273				pr_warn("map '%s': value spec is not PTR: %s.\n",
 2274					map_name, btf_kind_str(t));
 2275				return -EINVAL;
 2276			}
 2277			sz = btf__resolve_size(btf, t->type);
 2278			if (sz < 0) {
 2279				pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
 2280					map_name, t->type, (ssize_t)sz);
 2281				return sz;
 2282			}
 2283			if (map_def->value_size && map_def->value_size != sz) {
 2284				pr_warn("map '%s': conflicting value size %u != %zd.\n",
 2285					map_name, map_def->value_size, (ssize_t)sz);
 2286				return -EINVAL;
 2287			}
 2288			map_def->value_size = sz;
 2289			map_def->value_type_id = t->type;
 2290			map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
 2291		}
 2292		else if (strcmp(name, "values") == 0) {
 2293			bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
 2294			bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
 2295			const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
 2296			char inner_map_name[128];
 2297			int err;
 2298
 2299			if (is_inner) {
 2300				pr_warn("map '%s': multi-level inner maps not supported.\n",
 2301					map_name);
 2302				return -ENOTSUP;
 2303			}
 2304			if (i != vlen - 1) {
 2305				pr_warn("map '%s': '%s' member should be last.\n",
 2306					map_name, name);
 2307				return -EINVAL;
 2308			}
 2309			if (!is_map_in_map && !is_prog_array) {
 2310				pr_warn("map '%s': should be map-in-map or prog-array.\n",
 2311					map_name);
 2312				return -ENOTSUP;
 2313			}
 2314			if (map_def->value_size && map_def->value_size != 4) {
 2315				pr_warn("map '%s': conflicting value size %u != 4.\n",
 2316					map_name, map_def->value_size);
 2317				return -EINVAL;
 2318			}
 2319			map_def->value_size = 4;
 2320			t = btf__type_by_id(btf, m->type);
 2321			if (!t) {
 2322				pr_warn("map '%s': %s type [%d] not found.\n",
 2323					map_name, desc, m->type);
 2324				return -EINVAL;
 2325			}
 2326			if (!btf_is_array(t) || btf_array(t)->nelems) {
 2327				pr_warn("map '%s': %s spec is not a zero-sized array.\n",
 2328					map_name, desc);
 2329				return -EINVAL;
 2330			}
 2331			t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
 2332			if (!btf_is_ptr(t)) {
 2333				pr_warn("map '%s': %s def is of unexpected kind %s.\n",
 2334					map_name, desc, btf_kind_str(t));
 2335				return -EINVAL;
 2336			}
 2337			t = skip_mods_and_typedefs(btf, t->type, NULL);
 2338			if (is_prog_array) {
 2339				if (!btf_is_func_proto(t)) {
 2340					pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
 2341						map_name, btf_kind_str(t));
 2342					return -EINVAL;
 2343				}
 2344				continue;
 2345			}
 2346			if (!btf_is_struct(t)) {
 2347				pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
 2348					map_name, btf_kind_str(t));
 2349				return -EINVAL;
 2350			}
 2351
 2352			snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
 2353			err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
 2354			if (err)
 2355				return err;
 2356
 2357			map_def->parts |= MAP_DEF_INNER_MAP;
 2358		} else if (strcmp(name, "pinning") == 0) {
 2359			__u32 val;
 2360
 2361			if (is_inner) {
 2362				pr_warn("map '%s': inner def can't be pinned.\n", map_name);
 2363				return -EINVAL;
 2364			}
 2365			if (!get_map_field_int(map_name, btf, m, &val))
 2366				return -EINVAL;
 2367			if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
 2368				pr_warn("map '%s': invalid pinning value %u.\n",
 2369					map_name, val);
 2370				return -EINVAL;
 2371			}
 2372			map_def->pinning = val;
 2373			map_def->parts |= MAP_DEF_PINNING;
 2374		} else if (strcmp(name, "map_extra") == 0) {
 2375			__u32 map_extra;
 2376
 2377			if (!get_map_field_int(map_name, btf, m, &map_extra))
 2378				return -EINVAL;
 2379			map_def->map_extra = map_extra;
 2380			map_def->parts |= MAP_DEF_MAP_EXTRA;
 2381		} else {
 2382			if (strict) {
 2383				pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
 2384				return -ENOTSUP;
 2385			}
 2386			pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
 2387		}
 2388	}
 2389
 2390	if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
 2391		pr_warn("map '%s': map type isn't specified.\n", map_name);
 2392		return -EINVAL;
 2393	}
 2394
 
 
 
 2395	return 0;
 2396}
 2397
 2398static size_t adjust_ringbuf_sz(size_t sz)
 
 
 2399{
 2400	__u32 page_sz = sysconf(_SC_PAGE_SIZE);
 2401	__u32 mul;
 2402
 2403	/* if user forgot to set any size, make sure they see error */
 2404	if (sz == 0)
 
 
 2405		return 0;
 2406	/* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
 2407	 * a power-of-2 multiple of kernel's page size. If user diligently
 2408	 * satisified these conditions, pass the size through.
 2409	 */
 2410	if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
 2411		return sz;
 2412
 2413	/* Otherwise find closest (page_sz * power_of_2) product bigger than
 2414	 * user-set size to satisfy both user size request and kernel
 2415	 * requirements and substitute correct max_entries for map creation.
 2416	 */
 2417	for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
 2418		if (mul * page_sz > sz)
 2419			return mul * page_sz;
 2420	}
 2421
 2422	/* if it's impossible to satisfy the conditions (i.e., user size is
 2423	 * very close to UINT_MAX but is not a power-of-2 multiple of
 2424	 * page_size) then just return original size and let kernel reject it
 2425	 */
 2426	return sz;
 2427}
 2428
 2429static bool map_is_ringbuf(const struct bpf_map *map)
 2430{
 2431	return map->def.type == BPF_MAP_TYPE_RINGBUF ||
 2432	       map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
 2433}
 2434
 2435static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
 2436{
 2437	map->def.type = def->map_type;
 2438	map->def.key_size = def->key_size;
 2439	map->def.value_size = def->value_size;
 2440	map->def.max_entries = def->max_entries;
 2441	map->def.map_flags = def->map_flags;
 2442	map->map_extra = def->map_extra;
 2443
 2444	map->numa_node = def->numa_node;
 2445	map->btf_key_type_id = def->key_type_id;
 2446	map->btf_value_type_id = def->value_type_id;
 2447
 2448	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
 2449	if (map_is_ringbuf(map))
 2450		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
 2451
 2452	if (def->parts & MAP_DEF_MAP_TYPE)
 2453		pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
 2454
 2455	if (def->parts & MAP_DEF_KEY_TYPE)
 2456		pr_debug("map '%s': found key [%u], sz = %u.\n",
 2457			 map->name, def->key_type_id, def->key_size);
 2458	else if (def->parts & MAP_DEF_KEY_SIZE)
 2459		pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
 2460
 2461	if (def->parts & MAP_DEF_VALUE_TYPE)
 2462		pr_debug("map '%s': found value [%u], sz = %u.\n",
 2463			 map->name, def->value_type_id, def->value_size);
 2464	else if (def->parts & MAP_DEF_VALUE_SIZE)
 2465		pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
 2466
 2467	if (def->parts & MAP_DEF_MAX_ENTRIES)
 2468		pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
 2469	if (def->parts & MAP_DEF_MAP_FLAGS)
 2470		pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
 2471	if (def->parts & MAP_DEF_MAP_EXTRA)
 2472		pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
 2473			 (unsigned long long)def->map_extra);
 2474	if (def->parts & MAP_DEF_PINNING)
 2475		pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
 2476	if (def->parts & MAP_DEF_NUMA_NODE)
 2477		pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
 2478
 2479	if (def->parts & MAP_DEF_INNER_MAP)
 2480		pr_debug("map '%s': found inner map definition.\n", map->name);
 2481}
 2482
 2483static const char *btf_var_linkage_str(__u32 linkage)
 2484{
 2485	switch (linkage) {
 2486	case BTF_VAR_STATIC: return "static";
 2487	case BTF_VAR_GLOBAL_ALLOCATED: return "global";
 2488	case BTF_VAR_GLOBAL_EXTERN: return "extern";
 2489	default: return "unknown";
 2490	}
 2491}
 2492
 2493static int bpf_object__init_user_btf_map(struct bpf_object *obj,
 2494					 const struct btf_type *sec,
 2495					 int var_idx, int sec_idx,
 2496					 const Elf_Data *data, bool strict,
 2497					 const char *pin_root_path)
 2498{
 2499	struct btf_map_def map_def = {}, inner_def = {};
 2500	const struct btf_type *var, *def;
 2501	const struct btf_var_secinfo *vi;
 2502	const struct btf_var *var_extra;
 2503	const char *map_name;
 2504	struct bpf_map *map;
 2505	int err;
 2506
 2507	vi = btf_var_secinfos(sec) + var_idx;
 2508	var = btf__type_by_id(obj->btf, vi->type);
 2509	var_extra = btf_var(var);
 2510	map_name = btf__name_by_offset(obj->btf, var->name_off);
 2511
 2512	if (map_name == NULL || map_name[0] == '\0') {
 2513		pr_warn("map #%d: empty name.\n", var_idx);
 2514		return -EINVAL;
 2515	}
 2516	if ((__u64)vi->offset + vi->size > data->d_size) {
 2517		pr_warn("map '%s' BTF data is corrupted.\n", map_name);
 2518		return -EINVAL;
 2519	}
 2520	if (!btf_is_var(var)) {
 2521		pr_warn("map '%s': unexpected var kind %s.\n",
 2522			map_name, btf_kind_str(var));
 2523		return -EINVAL;
 2524	}
 2525	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
 2526		pr_warn("map '%s': unsupported map linkage %s.\n",
 2527			map_name, btf_var_linkage_str(var_extra->linkage));
 2528		return -EOPNOTSUPP;
 2529	}
 2530
 2531	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
 2532	if (!btf_is_struct(def)) {
 2533		pr_warn("map '%s': unexpected def kind %s.\n",
 2534			map_name, btf_kind_str(var));
 2535		return -EINVAL;
 2536	}
 2537	if (def->size > vi->size) {
 2538		pr_warn("map '%s': invalid def size.\n", map_name);
 2539		return -EINVAL;
 2540	}
 2541
 2542	map = bpf_object__add_map(obj);
 2543	if (IS_ERR(map))
 2544		return PTR_ERR(map);
 2545	map->name = strdup(map_name);
 2546	if (!map->name) {
 2547		pr_warn("map '%s': failed to alloc map name.\n", map_name);
 2548		return -ENOMEM;
 2549	}
 2550	map->libbpf_type = LIBBPF_MAP_UNSPEC;
 2551	map->def.type = BPF_MAP_TYPE_UNSPEC;
 2552	map->sec_idx = sec_idx;
 2553	map->sec_offset = vi->offset;
 2554	map->btf_var_idx = var_idx;
 2555	pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
 2556		 map_name, map->sec_idx, map->sec_offset);
 2557
 2558	err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
 2559	if (err)
 2560		return err;
 2561
 2562	fill_map_from_def(map, &map_def);
 
 2563
 2564	if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
 2565		err = build_map_pin_path(map, pin_root_path);
 2566		if (err) {
 2567			pr_warn("map '%s': couldn't build pin path.\n", map->name);
 2568			return err;
 2569		}
 2570	}
 2571
 2572	if (map_def.parts & MAP_DEF_INNER_MAP) {
 2573		map->inner_map = calloc(1, sizeof(*map->inner_map));
 2574		if (!map->inner_map)
 2575			return -ENOMEM;
 2576		map->inner_map->fd = -1;
 2577		map->inner_map->sec_idx = sec_idx;
 2578		map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
 2579		if (!map->inner_map->name)
 2580			return -ENOMEM;
 2581		sprintf(map->inner_map->name, "%s.inner", map_name);
 2582
 2583		fill_map_from_def(map->inner_map, &inner_def);
 2584	}
 2585
 2586	err = map_fill_btf_type_info(obj, map);
 2587	if (err)
 2588		return err;
 2589
 2590	return 0;
 2591}
 2592
 2593static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
 2594					  const char *pin_root_path)
 2595{
 2596	const struct btf_type *sec = NULL;
 2597	int nr_types, i, vlen, err;
 2598	const struct btf_type *t;
 2599	const char *name;
 2600	Elf_Data *data;
 2601	Elf_Scn *scn;
 2602
 2603	if (obj->efile.btf_maps_shndx < 0)
 2604		return 0;
 2605
 2606	scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
 2607	data = elf_sec_data(obj, scn);
 2608	if (!scn || !data) {
 2609		pr_warn("elf: failed to get %s map definitions for %s\n",
 2610			MAPS_ELF_SEC, obj->path);
 2611		return -EINVAL;
 2612	}
 2613
 2614	nr_types = btf__type_cnt(obj->btf);
 2615	for (i = 1; i < nr_types; i++) {
 2616		t = btf__type_by_id(obj->btf, i);
 2617		if (!btf_is_datasec(t))
 2618			continue;
 2619		name = btf__name_by_offset(obj->btf, t->name_off);
 2620		if (strcmp(name, MAPS_ELF_SEC) == 0) {
 2621			sec = t;
 2622			obj->efile.btf_maps_sec_btf_id = i;
 2623			break;
 2624		}
 2625	}
 2626
 2627	if (!sec) {
 2628		pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
 2629		return -ENOENT;
 2630	}
 2631
 2632	vlen = btf_vlen(sec);
 2633	for (i = 0; i < vlen; i++) {
 2634		err = bpf_object__init_user_btf_map(obj, sec, i,
 2635						    obj->efile.btf_maps_shndx,
 2636						    data, strict,
 2637						    pin_root_path);
 2638		if (err)
 2639			return err;
 2640	}
 2641
 2642	return 0;
 2643}
 2644
 2645static int bpf_object__init_maps(struct bpf_object *obj,
 2646				 const struct bpf_object_open_opts *opts)
 2647{
 2648	const char *pin_root_path;
 2649	bool strict;
 2650	int err = 0;
 2651
 2652	strict = !OPTS_GET(opts, relaxed_maps, false);
 2653	pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
 2654
 2655	err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
 2656	err = err ?: bpf_object__init_global_data_maps(obj);
 2657	err = err ?: bpf_object__init_kconfig_map(obj);
 2658	err = err ?: bpf_object__init_struct_ops_maps(obj);
 2659
 2660	return err;
 2661}
 2662
 2663static bool section_have_execinstr(struct bpf_object *obj, int idx)
 2664{
 2665	Elf64_Shdr *sh;
 2666
 2667	sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
 2668	if (!sh)
 2669		return false;
 2670
 2671	return sh->sh_flags & SHF_EXECINSTR;
 2672}
 2673
 2674static bool btf_needs_sanitization(struct bpf_object *obj)
 2675{
 2676	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
 2677	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
 2678	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
 2679	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
 2680	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
 2681	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
 2682	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
 2683
 2684	return !has_func || !has_datasec || !has_func_global || !has_float ||
 2685	       !has_decl_tag || !has_type_tag || !has_enum64;
 2686}
 2687
 2688static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
 2689{
 2690	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
 2691	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
 2692	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
 2693	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
 2694	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
 2695	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
 2696	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
 2697	int enum64_placeholder_id = 0;
 2698	struct btf_type *t;
 2699	int i, j, vlen;
 2700
 2701	for (i = 1; i < btf__type_cnt(btf); i++) {
 2702		t = (struct btf_type *)btf__type_by_id(btf, i);
 2703
 2704		if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
 2705			/* replace VAR/DECL_TAG with INT */
 2706			t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
 2707			/*
 2708			 * using size = 1 is the safest choice, 4 will be too
 2709			 * big and cause kernel BTF validation failure if
 2710			 * original variable took less than 4 bytes
 2711			 */
 2712			t->size = 1;
 2713			*(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
 2714		} else if (!has_datasec && btf_is_datasec(t)) {
 2715			/* replace DATASEC with STRUCT */
 2716			const struct btf_var_secinfo *v = btf_var_secinfos(t);
 2717			struct btf_member *m = btf_members(t);
 2718			struct btf_type *vt;
 2719			char *name;
 2720
 2721			name = (char *)btf__name_by_offset(btf, t->name_off);
 2722			while (*name) {
 2723				if (*name == '.')
 2724					*name = '_';
 2725				name++;
 2726			}
 2727
 2728			vlen = btf_vlen(t);
 2729			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
 2730			for (j = 0; j < vlen; j++, v++, m++) {
 2731				/* order of field assignments is important */
 2732				m->offset = v->offset * 8;
 2733				m->type = v->type;
 2734				/* preserve variable name as member name */
 2735				vt = (void *)btf__type_by_id(btf, v->type);
 2736				m->name_off = vt->name_off;
 2737			}
 2738		} else if (!has_func && btf_is_func_proto(t)) {
 2739			/* replace FUNC_PROTO with ENUM */
 2740			vlen = btf_vlen(t);
 2741			t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
 2742			t->size = sizeof(__u32); /* kernel enforced */
 2743		} else if (!has_func && btf_is_func(t)) {
 2744			/* replace FUNC with TYPEDEF */
 2745			t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
 2746		} else if (!has_func_global && btf_is_func(t)) {
 2747			/* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
 2748			t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
 2749		} else if (!has_float && btf_is_float(t)) {
 2750			/* replace FLOAT with an equally-sized empty STRUCT;
 2751			 * since C compilers do not accept e.g. "float" as a
 2752			 * valid struct name, make it anonymous
 2753			 */
 2754			t->name_off = 0;
 2755			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
 2756		} else if (!has_type_tag && btf_is_type_tag(t)) {
 2757			/* replace TYPE_TAG with a CONST */
 2758			t->name_off = 0;
 2759			t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
 2760		} else if (!has_enum64 && btf_is_enum(t)) {
 2761			/* clear the kflag */
 2762			t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
 2763		} else if (!has_enum64 && btf_is_enum64(t)) {
 2764			/* replace ENUM64 with a union */
 2765			struct btf_member *m;
 2766
 2767			if (enum64_placeholder_id == 0) {
 2768				enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
 2769				if (enum64_placeholder_id < 0)
 2770					return enum64_placeholder_id;
 2771
 2772				t = (struct btf_type *)btf__type_by_id(btf, i);
 2773			}
 2774
 2775			m = btf_members(t);
 2776			vlen = btf_vlen(t);
 2777			t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
 2778			for (j = 0; j < vlen; j++, m++) {
 2779				m->type = enum64_placeholder_id;
 2780				m->offset = 0;
 2781			}
 2782		}
 2783	}
 2784
 2785	return 0;
 2786}
 2787
 2788static bool libbpf_needs_btf(const struct bpf_object *obj)
 2789{
 2790	return obj->efile.btf_maps_shndx >= 0 ||
 2791	       obj->efile.st_ops_shndx >= 0 ||
 2792	       obj->nr_extern > 0;
 2793}
 2794
 2795static bool kernel_needs_btf(const struct bpf_object *obj)
 2796{
 2797	return obj->efile.st_ops_shndx >= 0;
 2798}
 2799
 2800static int bpf_object__init_btf(struct bpf_object *obj,
 2801				Elf_Data *btf_data,
 2802				Elf_Data *btf_ext_data)
 2803{
 2804	int err = -ENOENT;
 2805
 2806	if (btf_data) {
 2807		obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
 2808		err = libbpf_get_error(obj->btf);
 2809		if (err) {
 2810			obj->btf = NULL;
 2811			pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
 2812			goto out;
 2813		}
 2814		/* enforce 8-byte pointers for BPF-targeted BTFs */
 2815		btf__set_pointer_size(obj->btf, 8);
 2816	}
 2817	if (btf_ext_data) {
 2818		struct btf_ext_info *ext_segs[3];
 2819		int seg_num, sec_num;
 2820
 2821		if (!obj->btf) {
 2822			pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
 2823				 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
 2824			goto out;
 2825		}
 2826		obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
 2827		err = libbpf_get_error(obj->btf_ext);
 2828		if (err) {
 2829			pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
 2830				BTF_EXT_ELF_SEC, err);
 2831			obj->btf_ext = NULL;
 2832			goto out;
 2833		}
 2834
 2835		/* setup .BTF.ext to ELF section mapping */
 2836		ext_segs[0] = &obj->btf_ext->func_info;
 2837		ext_segs[1] = &obj->btf_ext->line_info;
 2838		ext_segs[2] = &obj->btf_ext->core_relo_info;
 2839		for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
 2840			struct btf_ext_info *seg = ext_segs[seg_num];
 2841			const struct btf_ext_info_sec *sec;
 2842			const char *sec_name;
 2843			Elf_Scn *scn;
 2844
 2845			if (seg->sec_cnt == 0)
 2846				continue;
 2847
 2848			seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
 2849			if (!seg->sec_idxs) {
 2850				err = -ENOMEM;
 2851				goto out;
 2852			}
 2853
 2854			sec_num = 0;
 2855			for_each_btf_ext_sec(seg, sec) {
 2856				/* preventively increment index to avoid doing
 2857				 * this before every continue below
 2858				 */
 2859				sec_num++;
 2860
 2861				sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
 2862				if (str_is_empty(sec_name))
 2863					continue;
 2864				scn = elf_sec_by_name(obj, sec_name);
 2865				if (!scn)
 2866					continue;
 2867
 2868				seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
 2869			}
 2870		}
 2871	}
 2872out:
 2873	if (err && libbpf_needs_btf(obj)) {
 2874		pr_warn("BTF is required, but is missing or corrupted.\n");
 2875		return err;
 2876	}
 2877	return 0;
 2878}
 2879
 2880static int compare_vsi_off(const void *_a, const void *_b)
 2881{
 2882	const struct btf_var_secinfo *a = _a;
 2883	const struct btf_var_secinfo *b = _b;
 2884
 2885	return a->offset - b->offset;
 2886}
 2887
 2888static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
 2889			     struct btf_type *t)
 2890{
 2891	__u32 size = 0, i, vars = btf_vlen(t);
 2892	const char *sec_name = btf__name_by_offset(btf, t->name_off);
 2893	struct btf_var_secinfo *vsi;
 2894	bool fixup_offsets = false;
 2895	int err;
 2896
 2897	if (!sec_name) {
 2898		pr_debug("No name found in string section for DATASEC kind.\n");
 2899		return -ENOENT;
 2900	}
 2901
 2902	/* Extern-backing datasecs (.ksyms, .kconfig) have their size and
 2903	 * variable offsets set at the previous step. Further, not every
 2904	 * extern BTF VAR has corresponding ELF symbol preserved, so we skip
 2905	 * all fixups altogether for such sections and go straight to sorting
 2906	 * VARs within their DATASEC.
 2907	 */
 2908	if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)
 2909		goto sort_vars;
 2910
 2911	/* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
 2912	 * fix this up. But BPF static linker already fixes this up and fills
 2913	 * all the sizes and offsets during static linking. So this step has
 2914	 * to be optional. But the STV_HIDDEN handling is non-optional for any
 2915	 * non-extern DATASEC, so the variable fixup loop below handles both
 2916	 * functions at the same time, paying the cost of BTF VAR <-> ELF
 2917	 * symbol matching just once.
 2918	 */
 2919	if (t->size == 0) {
 2920		err = find_elf_sec_sz(obj, sec_name, &size);
 2921		if (err || !size) {
 2922			pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
 2923				 sec_name, size, err);
 2924			return -ENOENT;
 2925		}
 2926
 2927		t->size = size;
 2928		fixup_offsets = true;
 2929	}
 2930
 2931	for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
 2932		const struct btf_type *t_var;
 2933		struct btf_var *var;
 2934		const char *var_name;
 2935		Elf64_Sym *sym;
 2936
 2937		t_var = btf__type_by_id(btf, vsi->type);
 2938		if (!t_var || !btf_is_var(t_var)) {
 2939			pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
 2940			return -EINVAL;
 2941		}
 2942
 2943		var = btf_var(t_var);
 2944		if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
 2945			continue;
 2946
 2947		var_name = btf__name_by_offset(btf, t_var->name_off);
 2948		if (!var_name) {
 2949			pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
 2950				 sec_name, i);
 2951			return -ENOENT;
 2952		}
 2953
 2954		sym = find_elf_var_sym(obj, var_name);
 2955		if (IS_ERR(sym)) {
 2956			pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
 2957				 sec_name, var_name);
 2958			return -ENOENT;
 2959		}
 2960
 2961		if (fixup_offsets)
 2962			vsi->offset = sym->st_value;
 2963
 2964		/* if variable is a global/weak symbol, but has restricted
 2965		 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
 2966		 * as static. This follows similar logic for functions (BPF
 2967		 * subprogs) and influences libbpf's further decisions about
 2968		 * whether to make global data BPF array maps as
 2969		 * BPF_F_MMAPABLE.
 2970		 */
 2971		if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
 2972		    || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
 2973			var->linkage = BTF_VAR_STATIC;
 2974	}
 2975
 2976sort_vars:
 2977	qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
 2978	return 0;
 2979}
 2980
 2981static int bpf_object_fixup_btf(struct bpf_object *obj)
 2982{
 2983	int i, n, err = 0;
 2984
 2985	if (!obj->btf)
 2986		return 0;
 2987
 2988	n = btf__type_cnt(obj->btf);
 2989	for (i = 1; i < n; i++) {
 2990		struct btf_type *t = btf_type_by_id(obj->btf, i);
 2991
 2992		/* Loader needs to fix up some of the things compiler
 2993		 * couldn't get its hands on while emitting BTF. This
 2994		 * is section size and global variable offset. We use
 2995		 * the info from the ELF itself for this purpose.
 2996		 */
 2997		if (btf_is_datasec(t)) {
 2998			err = btf_fixup_datasec(obj, obj->btf, t);
 2999			if (err)
 3000				return err;
 3001		}
 3002	}
 3003
 3004	return 0;
 3005}
 3006
 3007static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
 3008{
 3009	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
 3010	    prog->type == BPF_PROG_TYPE_LSM)
 3011		return true;
 3012
 3013	/* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
 3014	 * also need vmlinux BTF
 3015	 */
 3016	if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
 3017		return true;
 3018
 3019	return false;
 3020}
 3021
 3022static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
 3023{
 3024	struct bpf_program *prog;
 3025	int i;
 
 3026
 3027	/* CO-RE relocations need kernel BTF, only when btf_custom_path
 3028	 * is not specified
 3029	 */
 3030	if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
 3031		return true;
 3032
 3033	/* Support for typed ksyms needs kernel BTF */
 3034	for (i = 0; i < obj->nr_extern; i++) {
 3035		const struct extern_desc *ext;
 3036
 3037		ext = &obj->externs[i];
 3038		if (ext->type == EXT_KSYM && ext->ksym.type_id)
 3039			return true;
 3040	}
 3041
 3042	bpf_object__for_each_program(prog, obj) {
 3043		if (!prog->autoload)
 3044			continue;
 3045		if (prog_needs_vmlinux_btf(prog))
 3046			return true;
 3047	}
 3048
 3049	return false;
 3050}
 3051
 3052static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
 3053{
 3054	int err;
 3055
 3056	/* btf_vmlinux could be loaded earlier */
 3057	if (obj->btf_vmlinux || obj->gen_loader)
 3058		return 0;
 3059
 3060	if (!force && !obj_needs_vmlinux_btf(obj))
 3061		return 0;
 3062
 3063	obj->btf_vmlinux = btf__load_vmlinux_btf();
 3064	err = libbpf_get_error(obj->btf_vmlinux);
 3065	if (err) {
 3066		pr_warn("Error loading vmlinux BTF: %d\n", err);
 3067		obj->btf_vmlinux = NULL;
 3068		return err;
 3069	}
 3070	return 0;
 3071}
 3072
 3073static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
 3074{
 3075	struct btf *kern_btf = obj->btf;
 3076	bool btf_mandatory, sanitize;
 3077	int i, err = 0;
 3078
 3079	if (!obj->btf)
 3080		return 0;
 3081
 3082	if (!kernel_supports(obj, FEAT_BTF)) {
 3083		if (kernel_needs_btf(obj)) {
 3084			err = -EOPNOTSUPP;
 3085			goto report;
 3086		}
 3087		pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
 3088		return 0;
 3089	}
 3090
 3091	/* Even though some subprogs are global/weak, user might prefer more
 3092	 * permissive BPF verification process that BPF verifier performs for
 3093	 * static functions, taking into account more context from the caller
 3094	 * functions. In such case, they need to mark such subprogs with
 3095	 * __attribute__((visibility("hidden"))) and libbpf will adjust
 3096	 * corresponding FUNC BTF type to be marked as static and trigger more
 3097	 * involved BPF verification process.
 3098	 */
 3099	for (i = 0; i < obj->nr_programs; i++) {
 3100		struct bpf_program *prog = &obj->programs[i];
 3101		struct btf_type *t;
 3102		const char *name;
 3103		int j, n;
 3104
 3105		if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
 3106			continue;
 3107
 3108		n = btf__type_cnt(obj->btf);
 3109		for (j = 1; j < n; j++) {
 3110			t = btf_type_by_id(obj->btf, j);
 3111			if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
 3112				continue;
 3113
 3114			name = btf__str_by_offset(obj->btf, t->name_off);
 3115			if (strcmp(name, prog->name) != 0)
 3116				continue;
 3117
 3118			t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
 3119			break;
 3120		}
 3121	}
 3122
 3123	sanitize = btf_needs_sanitization(obj);
 3124	if (sanitize) {
 3125		const void *raw_data;
 3126		__u32 sz;
 3127
 3128		/* clone BTF to sanitize a copy and leave the original intact */
 3129		raw_data = btf__raw_data(obj->btf, &sz);
 3130		kern_btf = btf__new(raw_data, sz);
 3131		err = libbpf_get_error(kern_btf);
 3132		if (err)
 3133			return err;
 3134
 3135		/* enforce 8-byte pointers for BPF-targeted BTFs */
 3136		btf__set_pointer_size(obj->btf, 8);
 3137		err = bpf_object__sanitize_btf(obj, kern_btf);
 3138		if (err)
 3139			return err;
 3140	}
 3141
 3142	if (obj->gen_loader) {
 3143		__u32 raw_size = 0;
 3144		const void *raw_data = btf__raw_data(kern_btf, &raw_size);
 3145
 3146		if (!raw_data)
 3147			return -ENOMEM;
 3148		bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
 3149		/* Pretend to have valid FD to pass various fd >= 0 checks.
 3150		 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
 3151		 */
 3152		btf__set_fd(kern_btf, 0);
 3153	} else {
 3154		/* currently BPF_BTF_LOAD only supports log_level 1 */
 3155		err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
 3156					   obj->log_level ? 1 : 0);
 3157	}
 3158	if (sanitize) {
 3159		if (!err) {
 3160			/* move fd to libbpf's BTF */
 3161			btf__set_fd(obj->btf, btf__fd(kern_btf));
 3162			btf__set_fd(kern_btf, -1);
 3163		}
 3164		btf__free(kern_btf);
 3165	}
 3166report:
 3167	if (err) {
 3168		btf_mandatory = kernel_needs_btf(obj);
 3169		pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
 3170			btf_mandatory ? "BTF is mandatory, can't proceed."
 3171				      : "BTF is optional, ignoring.");
 3172		if (!btf_mandatory)
 3173			err = 0;
 3174	}
 3175	return err;
 3176}
 3177
 3178static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
 3179{
 3180	const char *name;
 3181
 3182	name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
 3183	if (!name) {
 3184		pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
 3185			off, obj->path, elf_errmsg(-1));
 3186		return NULL;
 3187	}
 3188
 3189	return name;
 3190}
 3191
 3192static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
 3193{
 3194	const char *name;
 3195
 3196	name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
 3197	if (!name) {
 3198		pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
 3199			off, obj->path, elf_errmsg(-1));
 3200		return NULL;
 3201	}
 3202
 3203	return name;
 3204}
 3205
 3206static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
 3207{
 3208	Elf_Scn *scn;
 3209
 3210	scn = elf_getscn(obj->efile.elf, idx);
 3211	if (!scn) {
 3212		pr_warn("elf: failed to get section(%zu) from %s: %s\n",
 3213			idx, obj->path, elf_errmsg(-1));
 3214		return NULL;
 3215	}
 3216	return scn;
 3217}
 3218
 3219static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
 3220{
 3221	Elf_Scn *scn = NULL;
 3222	Elf *elf = obj->efile.elf;
 3223	const char *sec_name;
 3224
 3225	while ((scn = elf_nextscn(elf, scn)) != NULL) {
 3226		sec_name = elf_sec_name(obj, scn);
 3227		if (!sec_name)
 3228			return NULL;
 3229
 3230		if (strcmp(sec_name, name) != 0)
 3231			continue;
 3232
 3233		return scn;
 3234	}
 3235	return NULL;
 3236}
 3237
 3238static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
 3239{
 3240	Elf64_Shdr *shdr;
 3241
 3242	if (!scn)
 3243		return NULL;
 3244
 3245	shdr = elf64_getshdr(scn);
 3246	if (!shdr) {
 3247		pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
 3248			elf_ndxscn(scn), obj->path, elf_errmsg(-1));
 3249		return NULL;
 3250	}
 3251
 3252	return shdr;
 3253}
 3254
 3255static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
 3256{
 3257	const char *name;
 3258	Elf64_Shdr *sh;
 3259
 3260	if (!scn)
 3261		return NULL;
 3262
 3263	sh = elf_sec_hdr(obj, scn);
 3264	if (!sh)
 3265		return NULL;
 3266
 3267	name = elf_sec_str(obj, sh->sh_name);
 3268	if (!name) {
 3269		pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
 3270			elf_ndxscn(scn), obj->path, elf_errmsg(-1));
 3271		return NULL;
 3272	}
 3273
 3274	return name;
 3275}
 3276
 3277static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
 3278{
 3279	Elf_Data *data;
 3280
 3281	if (!scn)
 3282		return NULL;
 3283
 3284	data = elf_getdata(scn, 0);
 3285	if (!data) {
 3286		pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
 3287			elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
 3288			obj->path, elf_errmsg(-1));
 3289		return NULL;
 3290	}
 3291
 3292	return data;
 3293}
 3294
 3295static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
 3296{
 3297	if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
 3298		return NULL;
 3299
 3300	return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
 3301}
 3302
 3303static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
 3304{
 3305	if (idx >= data->d_size / sizeof(Elf64_Rel))
 3306		return NULL;
 3307
 3308	return (Elf64_Rel *)data->d_buf + idx;
 3309}
 3310
 3311static bool is_sec_name_dwarf(const char *name)
 3312{
 3313	/* approximation, but the actual list is too long */
 3314	return str_has_pfx(name, ".debug_");
 3315}
 3316
 3317static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
 3318{
 3319	/* no special handling of .strtab */
 3320	if (hdr->sh_type == SHT_STRTAB)
 3321		return true;
 3322
 3323	/* ignore .llvm_addrsig section as well */
 3324	if (hdr->sh_type == SHT_LLVM_ADDRSIG)
 3325		return true;
 3326
 3327	/* no subprograms will lead to an empty .text section, ignore it */
 3328	if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
 3329	    strcmp(name, ".text") == 0)
 3330		return true;
 3331
 3332	/* DWARF sections */
 3333	if (is_sec_name_dwarf(name))
 3334		return true;
 3335
 3336	if (str_has_pfx(name, ".rel")) {
 3337		name += sizeof(".rel") - 1;
 3338		/* DWARF section relocations */
 3339		if (is_sec_name_dwarf(name))
 3340			return true;
 3341
 3342		/* .BTF and .BTF.ext don't need relocations */
 3343		if (strcmp(name, BTF_ELF_SEC) == 0 ||
 3344		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
 3345			return true;
 3346	}
 3347
 3348	return false;
 3349}
 3350
 3351static int cmp_progs(const void *_a, const void *_b)
 3352{
 3353	const struct bpf_program *a = _a;
 3354	const struct bpf_program *b = _b;
 3355
 3356	if (a->sec_idx != b->sec_idx)
 3357		return a->sec_idx < b->sec_idx ? -1 : 1;
 3358
 3359	/* sec_insn_off can't be the same within the section */
 3360	return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
 3361}
 3362
 3363static int bpf_object__elf_collect(struct bpf_object *obj)
 3364{
 3365	struct elf_sec_desc *sec_desc;
 3366	Elf *elf = obj->efile.elf;
 3367	Elf_Data *btf_ext_data = NULL;
 3368	Elf_Data *btf_data = NULL;
 3369	int idx = 0, err = 0;
 3370	const char *name;
 3371	Elf_Data *data;
 3372	Elf_Scn *scn;
 3373	Elf64_Shdr *sh;
 3374
 3375	/* ELF section indices are 0-based, but sec #0 is special "invalid"
 3376	 * section. Since section count retrieved by elf_getshdrnum() does
 3377	 * include sec #0, it is already the necessary size of an array to keep
 3378	 * all the sections.
 3379	 */
 3380	if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
 3381		pr_warn("elf: failed to get the number of sections for %s: %s\n",
 3382			obj->path, elf_errmsg(-1));
 3383		return -LIBBPF_ERRNO__FORMAT;
 3384	}
 3385	obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
 3386	if (!obj->efile.secs)
 3387		return -ENOMEM;
 3388
 3389	/* a bunch of ELF parsing functionality depends on processing symbols,
 3390	 * so do the first pass and find the symbol table
 3391	 */
 3392	scn = NULL;
 3393	while ((scn = elf_nextscn(elf, scn)) != NULL) {
 3394		sh = elf_sec_hdr(obj, scn);
 3395		if (!sh)
 3396			return -LIBBPF_ERRNO__FORMAT;
 3397
 3398		if (sh->sh_type == SHT_SYMTAB) {
 3399			if (obj->efile.symbols) {
 3400				pr_warn("elf: multiple symbol tables in %s\n", obj->path);
 3401				return -LIBBPF_ERRNO__FORMAT;
 3402			}
 3403
 3404			data = elf_sec_data(obj, scn);
 3405			if (!data)
 3406				return -LIBBPF_ERRNO__FORMAT;
 3407
 3408			idx = elf_ndxscn(scn);
 3409
 3410			obj->efile.symbols = data;
 3411			obj->efile.symbols_shndx = idx;
 3412			obj->efile.strtabidx = sh->sh_link;
 3413		}
 3414	}
 3415
 3416	if (!obj->efile.symbols) {
 3417		pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
 3418			obj->path);
 3419		return -ENOENT;
 3420	}
 3421
 3422	scn = NULL;
 3423	while ((scn = elf_nextscn(elf, scn)) != NULL) {
 3424		idx = elf_ndxscn(scn);
 3425		sec_desc = &obj->efile.secs[idx];
 3426
 3427		sh = elf_sec_hdr(obj, scn);
 3428		if (!sh)
 3429			return -LIBBPF_ERRNO__FORMAT;
 3430
 3431		name = elf_sec_str(obj, sh->sh_name);
 3432		if (!name)
 3433			return -LIBBPF_ERRNO__FORMAT;
 3434
 3435		if (ignore_elf_section(sh, name))
 3436			continue;
 3437
 3438		data = elf_sec_data(obj, scn);
 3439		if (!data)
 3440			return -LIBBPF_ERRNO__FORMAT;
 3441
 3442		pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 3443			 idx, name, (unsigned long)data->d_size,
 3444			 (int)sh->sh_link, (unsigned long)sh->sh_flags,
 3445			 (int)sh->sh_type);
 3446
 3447		if (strcmp(name, "license") == 0) {
 3448			err = bpf_object__init_license(obj, data->d_buf, data->d_size);
 3449			if (err)
 3450				return err;
 3451		} else if (strcmp(name, "version") == 0) {
 3452			err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
 3453			if (err)
 3454				return err;
 3455		} else if (strcmp(name, "maps") == 0) {
 3456			pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
 3457			return -ENOTSUP;
 3458		} else if (strcmp(name, MAPS_ELF_SEC) == 0) {
 3459			obj->efile.btf_maps_shndx = idx;
 3460		} else if (strcmp(name, BTF_ELF_SEC) == 0) {
 3461			if (sh->sh_type != SHT_PROGBITS)
 3462				return -LIBBPF_ERRNO__FORMAT;
 3463			btf_data = data;
 3464		} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
 3465			if (sh->sh_type != SHT_PROGBITS)
 3466				return -LIBBPF_ERRNO__FORMAT;
 3467			btf_ext_data = data;
 3468		} else if (sh->sh_type == SHT_SYMTAB) {
 3469			/* already processed during the first pass above */
 3470		} else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
 3471			if (sh->sh_flags & SHF_EXECINSTR) {
 3472				if (strcmp(name, ".text") == 0)
 3473					obj->efile.text_shndx = idx;
 3474				err = bpf_object__add_programs(obj, data, name, idx);
 3475				if (err)
 3476					return err;
 3477			} else if (strcmp(name, DATA_SEC) == 0 ||
 3478				   str_has_pfx(name, DATA_SEC ".")) {
 3479				sec_desc->sec_type = SEC_DATA;
 3480				sec_desc->shdr = sh;
 3481				sec_desc->data = data;
 3482			} else if (strcmp(name, RODATA_SEC) == 0 ||
 3483				   str_has_pfx(name, RODATA_SEC ".")) {
 3484				sec_desc->sec_type = SEC_RODATA;
 3485				sec_desc->shdr = sh;
 3486				sec_desc->data = data;
 3487			} else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
 3488				obj->efile.st_ops_data = data;
 3489				obj->efile.st_ops_shndx = idx;
 3490			} else {
 3491				pr_info("elf: skipping unrecognized data section(%d) %s\n",
 3492					idx, name);
 3493			}
 3494		} else if (sh->sh_type == SHT_REL) {
 3495			int targ_sec_idx = sh->sh_info; /* points to other section */
 3496
 3497			if (sh->sh_entsize != sizeof(Elf64_Rel) ||
 3498			    targ_sec_idx >= obj->efile.sec_cnt)
 3499				return -LIBBPF_ERRNO__FORMAT;
 3500
 3501			/* Only do relo for section with exec instructions */
 3502			if (!section_have_execinstr(obj, targ_sec_idx) &&
 3503			    strcmp(name, ".rel" STRUCT_OPS_SEC) &&
 3504			    strcmp(name, ".rel" MAPS_ELF_SEC)) {
 3505				pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
 3506					idx, name, targ_sec_idx,
 3507					elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
 3508				continue;
 3509			}
 3510
 3511			sec_desc->sec_type = SEC_RELO;
 3512			sec_desc->shdr = sh;
 3513			sec_desc->data = data;
 3514		} else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
 3515							 str_has_pfx(name, BSS_SEC "."))) {
 3516			sec_desc->sec_type = SEC_BSS;
 3517			sec_desc->shdr = sh;
 3518			sec_desc->data = data;
 3519		} else {
 3520			pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
 3521				(size_t)sh->sh_size);
 3522		}
 3523	}
 3524
 3525	if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
 3526		pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
 3527		return -LIBBPF_ERRNO__FORMAT;
 3528	}
 3529
 3530	/* sort BPF programs by section name and in-section instruction offset
 3531	 * for faster search
 3532	 */
 3533	if (obj->nr_programs)
 3534		qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
 3535
 3536	return bpf_object__init_btf(obj, btf_data, btf_ext_data);
 3537}
 3538
 3539static bool sym_is_extern(const Elf64_Sym *sym)
 3540{
 3541	int bind = ELF64_ST_BIND(sym->st_info);
 3542	/* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
 3543	return sym->st_shndx == SHN_UNDEF &&
 3544	       (bind == STB_GLOBAL || bind == STB_WEAK) &&
 3545	       ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
 3546}
 3547
 3548static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
 3549{
 3550	int bind = ELF64_ST_BIND(sym->st_info);
 3551	int type = ELF64_ST_TYPE(sym->st_info);
 3552
 3553	/* in .text section */
 3554	if (sym->st_shndx != text_shndx)
 3555		return false;
 3556
 3557	/* local function */
 3558	if (bind == STB_LOCAL && type == STT_SECTION)
 3559		return true;
 3560
 3561	/* global function */
 3562	return bind == STB_GLOBAL && type == STT_FUNC;
 3563}
 3564
 3565static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
 3566{
 3567	const struct btf_type *t;
 3568	const char *tname;
 3569	int i, n;
 3570
 3571	if (!btf)
 3572		return -ESRCH;
 3573
 3574	n = btf__type_cnt(btf);
 3575	for (i = 1; i < n; i++) {
 3576		t = btf__type_by_id(btf, i);
 3577
 3578		if (!btf_is_var(t) && !btf_is_func(t))
 3579			continue;
 3580
 3581		tname = btf__name_by_offset(btf, t->name_off);
 3582		if (strcmp(tname, ext_name))
 3583			continue;
 3584
 3585		if (btf_is_var(t) &&
 3586		    btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
 3587			return -EINVAL;
 3588
 3589		if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
 3590			return -EINVAL;
 3591
 3592		return i;
 3593	}
 3594
 3595	return -ENOENT;
 3596}
 3597
 3598static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
 3599	const struct btf_var_secinfo *vs;
 3600	const struct btf_type *t;
 3601	int i, j, n;
 3602
 3603	if (!btf)
 3604		return -ESRCH;
 3605
 3606	n = btf__type_cnt(btf);
 3607	for (i = 1; i < n; i++) {
 3608		t = btf__type_by_id(btf, i);
 3609
 3610		if (!btf_is_datasec(t))
 3611			continue;
 3612
 3613		vs = btf_var_secinfos(t);
 3614		for (j = 0; j < btf_vlen(t); j++, vs++) {
 3615			if (vs->type == ext_btf_id)
 3616				return i;
 3617		}
 3618	}
 3619
 3620	return -ENOENT;
 3621}
 3622
 3623static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
 3624				     bool *is_signed)
 3625{
 3626	const struct btf_type *t;
 3627	const char *name;
 3628
 3629	t = skip_mods_and_typedefs(btf, id, NULL);
 3630	name = btf__name_by_offset(btf, t->name_off);
 3631
 3632	if (is_signed)
 3633		*is_signed = false;
 3634	switch (btf_kind(t)) {
 3635	case BTF_KIND_INT: {
 3636		int enc = btf_int_encoding(t);
 3637
 3638		if (enc & BTF_INT_BOOL)
 3639			return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
 3640		if (is_signed)
 3641			*is_signed = enc & BTF_INT_SIGNED;
 3642		if (t->size == 1)
 3643			return KCFG_CHAR;
 3644		if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
 3645			return KCFG_UNKNOWN;
 3646		return KCFG_INT;
 3647	}
 3648	case BTF_KIND_ENUM:
 3649		if (t->size != 4)
 3650			return KCFG_UNKNOWN;
 3651		if (strcmp(name, "libbpf_tristate"))
 3652			return KCFG_UNKNOWN;
 3653		return KCFG_TRISTATE;
 3654	case BTF_KIND_ENUM64:
 3655		if (strcmp(name, "libbpf_tristate"))
 3656			return KCFG_UNKNOWN;
 3657		return KCFG_TRISTATE;
 3658	case BTF_KIND_ARRAY:
 3659		if (btf_array(t)->nelems == 0)
 3660			return KCFG_UNKNOWN;
 3661		if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
 3662			return KCFG_UNKNOWN;
 3663		return KCFG_CHAR_ARR;
 3664	default:
 3665		return KCFG_UNKNOWN;
 3666	}
 3667}
 3668
 3669static int cmp_externs(const void *_a, const void *_b)
 3670{
 3671	const struct extern_desc *a = _a;
 3672	const struct extern_desc *b = _b;
 3673
 3674	if (a->type != b->type)
 3675		return a->type < b->type ? -1 : 1;
 3676
 3677	if (a->type == EXT_KCFG) {
 3678		/* descending order by alignment requirements */
 3679		if (a->kcfg.align != b->kcfg.align)
 3680			return a->kcfg.align > b->kcfg.align ? -1 : 1;
 3681		/* ascending order by size, within same alignment class */
 3682		if (a->kcfg.sz != b->kcfg.sz)
 3683			return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
 3684	}
 3685
 3686	/* resolve ties by name */
 3687	return strcmp(a->name, b->name);
 3688}
 3689
 3690static int find_int_btf_id(const struct btf *btf)
 3691{
 3692	const struct btf_type *t;
 3693	int i, n;
 3694
 3695	n = btf__type_cnt(btf);
 3696	for (i = 1; i < n; i++) {
 3697		t = btf__type_by_id(btf, i);
 3698
 3699		if (btf_is_int(t) && btf_int_bits(t) == 32)
 3700			return i;
 3701	}
 3702
 3703	return 0;
 3704}
 3705
 3706static int add_dummy_ksym_var(struct btf *btf)
 3707{
 3708	int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
 3709	const struct btf_var_secinfo *vs;
 3710	const struct btf_type *sec;
 3711
 3712	if (!btf)
 3713		return 0;
 3714
 3715	sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
 3716					    BTF_KIND_DATASEC);
 3717	if (sec_btf_id < 0)
 3718		return 0;
 3719
 3720	sec = btf__type_by_id(btf, sec_btf_id);
 3721	vs = btf_var_secinfos(sec);
 3722	for (i = 0; i < btf_vlen(sec); i++, vs++) {
 3723		const struct btf_type *vt;
 3724
 3725		vt = btf__type_by_id(btf, vs->type);
 3726		if (btf_is_func(vt))
 3727			break;
 3728	}
 3729
 3730	/* No func in ksyms sec.  No need to add dummy var. */
 3731	if (i == btf_vlen(sec))
 3732		return 0;
 3733
 3734	int_btf_id = find_int_btf_id(btf);
 3735	dummy_var_btf_id = btf__add_var(btf,
 3736					"dummy_ksym",
 3737					BTF_VAR_GLOBAL_ALLOCATED,
 3738					int_btf_id);
 3739	if (dummy_var_btf_id < 0)
 3740		pr_warn("cannot create a dummy_ksym var\n");
 3741
 3742	return dummy_var_btf_id;
 3743}
 3744
 3745static int bpf_object__collect_externs(struct bpf_object *obj)
 3746{
 3747	struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
 3748	const struct btf_type *t;
 3749	struct extern_desc *ext;
 3750	int i, n, off, dummy_var_btf_id;
 3751	const char *ext_name, *sec_name;
 3752	Elf_Scn *scn;
 3753	Elf64_Shdr *sh;
 3754
 3755	if (!obj->efile.symbols)
 3756		return 0;
 3757
 3758	scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
 3759	sh = elf_sec_hdr(obj, scn);
 3760	if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
 3761		return -LIBBPF_ERRNO__FORMAT;
 3762
 3763	dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
 3764	if (dummy_var_btf_id < 0)
 3765		return dummy_var_btf_id;
 3766
 3767	n = sh->sh_size / sh->sh_entsize;
 3768	pr_debug("looking for externs among %d symbols...\n", n);
 3769
 3770	for (i = 0; i < n; i++) {
 3771		Elf64_Sym *sym = elf_sym_by_idx(obj, i);
 3772
 3773		if (!sym)
 3774			return -LIBBPF_ERRNO__FORMAT;
 3775		if (!sym_is_extern(sym))
 3776			continue;
 3777		ext_name = elf_sym_str(obj, sym->st_name);
 3778		if (!ext_name || !ext_name[0])
 3779			continue;
 3780
 3781		ext = obj->externs;
 3782		ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
 3783		if (!ext)
 3784			return -ENOMEM;
 3785		obj->externs = ext;
 3786		ext = &ext[obj->nr_extern];
 3787		memset(ext, 0, sizeof(*ext));
 3788		obj->nr_extern++;
 3789
 3790		ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
 3791		if (ext->btf_id <= 0) {
 3792			pr_warn("failed to find BTF for extern '%s': %d\n",
 3793				ext_name, ext->btf_id);
 3794			return ext->btf_id;
 3795		}
 3796		t = btf__type_by_id(obj->btf, ext->btf_id);
 3797		ext->name = btf__name_by_offset(obj->btf, t->name_off);
 3798		ext->sym_idx = i;
 3799		ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
 3800
 3801		ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
 3802		if (ext->sec_btf_id <= 0) {
 3803			pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
 3804				ext_name, ext->btf_id, ext->sec_btf_id);
 3805			return ext->sec_btf_id;
 3806		}
 3807		sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
 3808		sec_name = btf__name_by_offset(obj->btf, sec->name_off);
 3809
 3810		if (strcmp(sec_name, KCONFIG_SEC) == 0) {
 3811			if (btf_is_func(t)) {
 3812				pr_warn("extern function %s is unsupported under %s section\n",
 3813					ext->name, KCONFIG_SEC);
 3814				return -ENOTSUP;
 3815			}
 3816			kcfg_sec = sec;
 3817			ext->type = EXT_KCFG;
 3818			ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
 3819			if (ext->kcfg.sz <= 0) {
 3820				pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
 3821					ext_name, ext->kcfg.sz);
 3822				return ext->kcfg.sz;
 3823			}
 3824			ext->kcfg.align = btf__align_of(obj->btf, t->type);
 3825			if (ext->kcfg.align <= 0) {
 3826				pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
 3827					ext_name, ext->kcfg.align);
 3828				return -EINVAL;
 3829			}
 3830			ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
 3831							&ext->kcfg.is_signed);
 3832			if (ext->kcfg.type == KCFG_UNKNOWN) {
 3833				pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
 3834				return -ENOTSUP;
 3835			}
 3836		} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
 3837			ksym_sec = sec;
 3838			ext->type = EXT_KSYM;
 3839			skip_mods_and_typedefs(obj->btf, t->type,
 3840					       &ext->ksym.type_id);
 3841		} else {
 3842			pr_warn("unrecognized extern section '%s'\n", sec_name);
 3843			return -ENOTSUP;
 3844		}
 3845	}
 3846	pr_debug("collected %d externs total\n", obj->nr_extern);
 3847
 3848	if (!obj->nr_extern)
 3849		return 0;
 3850
 3851	/* sort externs by type, for kcfg ones also by (align, size, name) */
 3852	qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
 3853
 3854	/* for .ksyms section, we need to turn all externs into allocated
 3855	 * variables in BTF to pass kernel verification; we do this by
 3856	 * pretending that each extern is a 8-byte variable
 3857	 */
 3858	if (ksym_sec) {
 3859		/* find existing 4-byte integer type in BTF to use for fake
 3860		 * extern variables in DATASEC
 3861		 */
 3862		int int_btf_id = find_int_btf_id(obj->btf);
 3863		/* For extern function, a dummy_var added earlier
 3864		 * will be used to replace the vs->type and
 3865		 * its name string will be used to refill
 3866		 * the missing param's name.
 3867		 */
 3868		const struct btf_type *dummy_var;
 3869
 3870		dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
 3871		for (i = 0; i < obj->nr_extern; i++) {
 3872			ext = &obj->externs[i];
 3873			if (ext->type != EXT_KSYM)
 3874				continue;
 3875			pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
 3876				 i, ext->sym_idx, ext->name);
 3877		}
 3878
 3879		sec = ksym_sec;
 3880		n = btf_vlen(sec);
 3881		for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
 3882			struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
 3883			struct btf_type *vt;
 3884
 3885			vt = (void *)btf__type_by_id(obj->btf, vs->type);
 3886			ext_name = btf__name_by_offset(obj->btf, vt->name_off);
 3887			ext = find_extern_by_name(obj, ext_name);
 3888			if (!ext) {
 3889				pr_warn("failed to find extern definition for BTF %s '%s'\n",
 3890					btf_kind_str(vt), ext_name);
 3891				return -ESRCH;
 3892			}
 3893			if (btf_is_func(vt)) {
 3894				const struct btf_type *func_proto;
 3895				struct btf_param *param;
 3896				int j;
 3897
 3898				func_proto = btf__type_by_id(obj->btf,
 3899							     vt->type);
 3900				param = btf_params(func_proto);
 3901				/* Reuse the dummy_var string if the
 3902				 * func proto does not have param name.
 3903				 */
 3904				for (j = 0; j < btf_vlen(func_proto); j++)
 3905					if (param[j].type && !param[j].name_off)
 3906						param[j].name_off =
 3907							dummy_var->name_off;
 3908				vs->type = dummy_var_btf_id;
 3909				vt->info &= ~0xffff;
 3910				vt->info |= BTF_FUNC_GLOBAL;
 3911			} else {
 3912				btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
 3913				vt->type = int_btf_id;
 3914			}
 3915			vs->offset = off;
 3916			vs->size = sizeof(int);
 3917		}
 3918		sec->size = off;
 
 3919	}
 3920
 3921	if (kcfg_sec) {
 3922		sec = kcfg_sec;
 3923		/* for kcfg externs calculate their offsets within a .kconfig map */
 3924		off = 0;
 3925		for (i = 0; i < obj->nr_extern; i++) {
 3926			ext = &obj->externs[i];
 3927			if (ext->type != EXT_KCFG)
 3928				continue;
 3929
 3930			ext->kcfg.data_off = roundup(off, ext->kcfg.align);
 3931			off = ext->kcfg.data_off + ext->kcfg.sz;
 3932			pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
 3933				 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
 3934		}
 3935		sec->size = off;
 3936		n = btf_vlen(sec);
 3937		for (i = 0; i < n; i++) {
 3938			struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
 3939
 3940			t = btf__type_by_id(obj->btf, vs->type);
 3941			ext_name = btf__name_by_offset(obj->btf, t->name_off);
 3942			ext = find_extern_by_name(obj, ext_name);
 3943			if (!ext) {
 3944				pr_warn("failed to find extern definition for BTF var '%s'\n",
 3945					ext_name);
 3946				return -ESRCH;
 3947			}
 3948			btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
 3949			vs->offset = ext->kcfg.data_off;
 3950		}
 3951	}
 3952	return 0;
 3953}
 3954
 3955static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
 3956{
 3957	return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
 3958}
 3959
 3960struct bpf_program *
 3961bpf_object__find_program_by_name(const struct bpf_object *obj,
 3962				 const char *name)
 3963{
 3964	struct bpf_program *prog;
 
 3965
 3966	bpf_object__for_each_program(prog, obj) {
 3967		if (prog_is_subprog(obj, prog))
 3968			continue;
 3969		if (!strcmp(prog->name, name))
 3970			return prog;
 3971	}
 3972	return errno = ENOENT, NULL;
 3973}
 3974
 3975static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
 3976				      int shndx)
 3977{
 3978	switch (obj->efile.secs[shndx].sec_type) {
 3979	case SEC_BSS:
 3980	case SEC_DATA:
 3981	case SEC_RODATA:
 3982		return true;
 3983	default:
 3984		return false;
 3985	}
 3986}
 3987
 3988static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
 3989				      int shndx)
 3990{
 3991	return shndx == obj->efile.btf_maps_shndx;
 3992}
 3993
 3994static enum libbpf_map_type
 3995bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
 3996{
 3997	if (shndx == obj->efile.symbols_shndx)
 3998		return LIBBPF_MAP_KCONFIG;
 3999
 4000	switch (obj->efile.secs[shndx].sec_type) {
 4001	case SEC_BSS:
 4002		return LIBBPF_MAP_BSS;
 4003	case SEC_DATA:
 4004		return LIBBPF_MAP_DATA;
 4005	case SEC_RODATA:
 4006		return LIBBPF_MAP_RODATA;
 4007	default:
 4008		return LIBBPF_MAP_UNSPEC;
 4009	}
 4010}
 4011
 4012static int bpf_program__record_reloc(struct bpf_program *prog,
 4013				     struct reloc_desc *reloc_desc,
 4014				     __u32 insn_idx, const char *sym_name,
 4015				     const Elf64_Sym *sym, const Elf64_Rel *rel)
 4016{
 4017	struct bpf_insn *insn = &prog->insns[insn_idx];
 4018	size_t map_idx, nr_maps = prog->obj->nr_maps;
 4019	struct bpf_object *obj = prog->obj;
 4020	__u32 shdr_idx = sym->st_shndx;
 4021	enum libbpf_map_type type;
 4022	const char *sym_sec_name;
 4023	struct bpf_map *map;
 4024
 4025	if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
 4026		pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
 4027			prog->name, sym_name, insn_idx, insn->code);
 4028		return -LIBBPF_ERRNO__RELOC;
 4029	}
 4030
 4031	if (sym_is_extern(sym)) {
 4032		int sym_idx = ELF64_R_SYM(rel->r_info);
 4033		int i, n = obj->nr_extern;
 4034		struct extern_desc *ext;
 4035
 4036		for (i = 0; i < n; i++) {
 4037			ext = &obj->externs[i];
 4038			if (ext->sym_idx == sym_idx)
 4039				break;
 4040		}
 4041		if (i >= n) {
 4042			pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
 4043				prog->name, sym_name, sym_idx);
 4044			return -LIBBPF_ERRNO__RELOC;
 4045		}
 4046		pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
 4047			 prog->name, i, ext->name, ext->sym_idx, insn_idx);
 4048		if (insn->code == (BPF_JMP | BPF_CALL))
 4049			reloc_desc->type = RELO_EXTERN_FUNC;
 4050		else
 4051			reloc_desc->type = RELO_EXTERN_VAR;
 4052		reloc_desc->insn_idx = insn_idx;
 4053		reloc_desc->sym_off = i; /* sym_off stores extern index */
 4054		return 0;
 4055	}
 4056
 4057	/* sub-program call relocation */
 4058	if (is_call_insn(insn)) {
 4059		if (insn->src_reg != BPF_PSEUDO_CALL) {
 4060			pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
 4061			return -LIBBPF_ERRNO__RELOC;
 4062		}
 4063		/* text_shndx can be 0, if no default "main" program exists */
 4064		if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
 4065			sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
 4066			pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
 4067				prog->name, sym_name, sym_sec_name);
 4068			return -LIBBPF_ERRNO__RELOC;
 4069		}
 4070		if (sym->st_value % BPF_INSN_SZ) {
 4071			pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
 4072				prog->name, sym_name, (size_t)sym->st_value);
 4073			return -LIBBPF_ERRNO__RELOC;
 4074		}
 4075		reloc_desc->type = RELO_CALL;
 4076		reloc_desc->insn_idx = insn_idx;
 4077		reloc_desc->sym_off = sym->st_value;
 4078		return 0;
 4079	}
 4080
 4081	if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
 4082		pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
 4083			prog->name, sym_name, shdr_idx);
 4084		return -LIBBPF_ERRNO__RELOC;
 4085	}
 4086
 4087	/* loading subprog addresses */
 4088	if (sym_is_subprog(sym, obj->efile.text_shndx)) {
 4089		/* global_func: sym->st_value = offset in the section, insn->imm = 0.
 4090		 * local_func: sym->st_value = 0, insn->imm = offset in the section.
 4091		 */
 4092		if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
 4093			pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
 4094				prog->name, sym_name, (size_t)sym->st_value, insn->imm);
 4095			return -LIBBPF_ERRNO__RELOC;
 4096		}
 4097
 4098		reloc_desc->type = RELO_SUBPROG_ADDR;
 4099		reloc_desc->insn_idx = insn_idx;
 4100		reloc_desc->sym_off = sym->st_value;
 4101		return 0;
 4102	}
 4103
 4104	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
 4105	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
 4106
 4107	/* generic map reference relocation */
 4108	if (type == LIBBPF_MAP_UNSPEC) {
 4109		if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
 4110			pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
 4111				prog->name, sym_name, sym_sec_name);
 4112			return -LIBBPF_ERRNO__RELOC;
 4113		}
 4114		for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 4115			map = &obj->maps[map_idx];
 4116			if (map->libbpf_type != type ||
 4117			    map->sec_idx != sym->st_shndx ||
 4118			    map->sec_offset != sym->st_value)
 4119				continue;
 4120			pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
 4121				 prog->name, map_idx, map->name, map->sec_idx,
 4122				 map->sec_offset, insn_idx);
 4123			break;
 4124		}
 4125		if (map_idx >= nr_maps) {
 4126			pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
 4127				prog->name, sym_sec_name, (size_t)sym->st_value);
 4128			return -LIBBPF_ERRNO__RELOC;
 4129		}
 4130		reloc_desc->type = RELO_LD64;
 4131		reloc_desc->insn_idx = insn_idx;
 4132		reloc_desc->map_idx = map_idx;
 4133		reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
 4134		return 0;
 4135	}
 4136
 4137	/* global data map relocation */
 4138	if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
 4139		pr_warn("prog '%s': bad data relo against section '%s'\n",
 4140			prog->name, sym_sec_name);
 4141		return -LIBBPF_ERRNO__RELOC;
 4142	}
 4143	for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 4144		map = &obj->maps[map_idx];
 4145		if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
 4146			continue;
 4147		pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
 4148			 prog->name, map_idx, map->name, map->sec_idx,
 4149			 map->sec_offset, insn_idx);
 4150		break;
 4151	}
 4152	if (map_idx >= nr_maps) {
 4153		pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
 4154			prog->name, sym_sec_name);
 4155		return -LIBBPF_ERRNO__RELOC;
 4156	}
 4157
 4158	reloc_desc->type = RELO_DATA;
 4159	reloc_desc->insn_idx = insn_idx;
 4160	reloc_desc->map_idx = map_idx;
 4161	reloc_desc->sym_off = sym->st_value;
 4162	return 0;
 4163}
 4164
 4165static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
 4166{
 4167	return insn_idx >= prog->sec_insn_off &&
 4168	       insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
 4169}
 4170
 4171static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
 4172						 size_t sec_idx, size_t insn_idx)
 4173{
 4174	int l = 0, r = obj->nr_programs - 1, m;
 4175	struct bpf_program *prog;
 4176
 4177	if (!obj->nr_programs)
 4178		return NULL;
 4179
 4180	while (l < r) {
 4181		m = l + (r - l + 1) / 2;
 4182		prog = &obj->programs[m];
 4183
 4184		if (prog->sec_idx < sec_idx ||
 4185		    (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
 4186			l = m;
 4187		else
 4188			r = m - 1;
 4189	}
 4190	/* matching program could be at index l, but it still might be the
 4191	 * wrong one, so we need to double check conditions for the last time
 4192	 */
 4193	prog = &obj->programs[l];
 4194	if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
 4195		return prog;
 4196	return NULL;
 4197}
 4198
 4199static int
 4200bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
 
 
 
 4201{
 4202	const char *relo_sec_name, *sec_name;
 4203	size_t sec_idx = shdr->sh_info, sym_idx;
 4204	struct bpf_program *prog;
 4205	struct reloc_desc *relos;
 4206	int err, i, nrels;
 4207	const char *sym_name;
 4208	__u32 insn_idx;
 4209	Elf_Scn *scn;
 4210	Elf_Data *scn_data;
 4211	Elf64_Sym *sym;
 4212	Elf64_Rel *rel;
 4213
 4214	if (sec_idx >= obj->efile.sec_cnt)
 4215		return -EINVAL;
 4216
 4217	scn = elf_sec_by_idx(obj, sec_idx);
 4218	scn_data = elf_sec_data(obj, scn);
 4219
 4220	relo_sec_name = elf_sec_str(obj, shdr->sh_name);
 4221	sec_name = elf_sec_name(obj, scn);
 4222	if (!relo_sec_name || !sec_name)
 4223		return -EINVAL;
 4224
 4225	pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
 4226		 relo_sec_name, sec_idx, sec_name);
 4227	nrels = shdr->sh_size / shdr->sh_entsize;
 4228
 4229	for (i = 0; i < nrels; i++) {
 4230		rel = elf_rel_by_idx(data, i);
 4231		if (!rel) {
 4232			pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
 4233			return -LIBBPF_ERRNO__FORMAT;
 4234		}
 4235
 4236		sym_idx = ELF64_R_SYM(rel->r_info);
 4237		sym = elf_sym_by_idx(obj, sym_idx);
 4238		if (!sym) {
 4239			pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
 4240				relo_sec_name, sym_idx, i);
 4241			return -LIBBPF_ERRNO__FORMAT;
 4242		}
 4243
 4244		if (sym->st_shndx >= obj->efile.sec_cnt) {
 4245			pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
 4246				relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
 4247			return -LIBBPF_ERRNO__FORMAT;
 4248		}
 4249
 4250		if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
 4251			pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
 4252				relo_sec_name, (size_t)rel->r_offset, i);
 
 
 4253			return -LIBBPF_ERRNO__FORMAT;
 4254		}
 4255
 4256		insn_idx = rel->r_offset / BPF_INSN_SZ;
 4257		/* relocations against static functions are recorded as
 4258		 * relocations against the section that contains a function;
 4259		 * in such case, symbol will be STT_SECTION and sym.st_name
 4260		 * will point to empty string (0), so fetch section name
 4261		 * instead
 4262		 */
 4263		if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
 4264			sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
 4265		else
 4266			sym_name = elf_sym_str(obj, sym->st_name);
 4267		sym_name = sym_name ?: "<?";
 4268
 4269		pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
 4270			 relo_sec_name, i, insn_idx, sym_name);
 4271
 4272		prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
 4273		if (!prog) {
 4274			pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
 4275				relo_sec_name, i, sec_name, insn_idx);
 4276			continue;
 4277		}
 4278
 4279		relos = libbpf_reallocarray(prog->reloc_desc,
 4280					    prog->nr_reloc + 1, sizeof(*relos));
 4281		if (!relos)
 4282			return -ENOMEM;
 4283		prog->reloc_desc = relos;
 4284
 4285		/* adjust insn_idx to local BPF program frame of reference */
 4286		insn_idx -= prog->sec_insn_off;
 4287		err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
 4288						insn_idx, sym_name, sym, rel);
 4289		if (err)
 4290			return err;
 4291
 4292		prog->nr_reloc++;
 4293	}
 4294	return 0;
 4295}
 4296
 4297static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
 4298{
 4299	int id;
 4300
 4301	if (!obj->btf)
 4302		return -ENOENT;
 4303
 4304	/* if it's BTF-defined map, we don't need to search for type IDs.
 4305	 * For struct_ops map, it does not need btf_key_type_id and
 4306	 * btf_value_type_id.
 4307	 */
 4308	if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
 4309		return 0;
 4310
 4311	/*
 4312	 * LLVM annotates global data differently in BTF, that is,
 4313	 * only as '.data', '.bss' or '.rodata'.
 4314	 */
 4315	if (!bpf_map__is_internal(map))
 4316		return -ENOENT;
 4317
 4318	id = btf__find_by_name(obj->btf, map->real_name);
 4319	if (id < 0)
 4320		return id;
 4321
 4322	map->btf_key_type_id = 0;
 4323	map->btf_value_type_id = id;
 4324	return 0;
 4325}
 4326
 4327static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
 4328{
 4329	char file[PATH_MAX], buff[4096];
 4330	FILE *fp;
 4331	__u32 val;
 4332	int err;
 4333
 4334	snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
 4335	memset(info, 0, sizeof(*info));
 4336
 4337	fp = fopen(file, "r");
 4338	if (!fp) {
 4339		err = -errno;
 4340		pr_warn("failed to open %s: %d. No procfs support?\n", file,
 4341			err);
 4342		return err;
 4343	}
 4344
 4345	while (fgets(buff, sizeof(buff), fp)) {
 4346		if (sscanf(buff, "map_type:\t%u", &val) == 1)
 4347			info->type = val;
 4348		else if (sscanf(buff, "key_size:\t%u", &val) == 1)
 4349			info->key_size = val;
 4350		else if (sscanf(buff, "value_size:\t%u", &val) == 1)
 4351			info->value_size = val;
 4352		else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
 4353			info->max_entries = val;
 4354		else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
 4355			info->map_flags = val;
 4356	}
 4357
 4358	fclose(fp);
 4359
 4360	return 0;
 4361}
 4362
 4363bool bpf_map__autocreate(const struct bpf_map *map)
 4364{
 4365	return map->autocreate;
 4366}
 4367
 4368int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
 4369{
 4370	if (map->obj->loaded)
 4371		return libbpf_err(-EBUSY);
 4372
 4373	map->autocreate = autocreate;
 4374	return 0;
 4375}
 4376
 4377int bpf_map__reuse_fd(struct bpf_map *map, int fd)
 4378{
 4379	struct bpf_map_info info;
 4380	__u32 len = sizeof(info), name_len;
 4381	int new_fd, err;
 4382	char *new_name;
 4383
 4384	memset(&info, 0, len);
 4385	err = bpf_obj_get_info_by_fd(fd, &info, &len);
 4386	if (err && errno == EINVAL)
 4387		err = bpf_get_map_info_from_fdinfo(fd, &info);
 4388	if (err)
 4389		return libbpf_err(err);
 4390
 4391	name_len = strlen(info.name);
 4392	if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
 4393		new_name = strdup(map->name);
 4394	else
 4395		new_name = strdup(info.name);
 4396
 4397	if (!new_name)
 4398		return libbpf_err(-errno);
 4399
 4400	new_fd = open("/", O_RDONLY | O_CLOEXEC);
 4401	if (new_fd < 0) {
 4402		err = -errno;
 4403		goto err_free_new_name;
 4404	}
 4405
 4406	new_fd = dup3(fd, new_fd, O_CLOEXEC);
 4407	if (new_fd < 0) {
 4408		err = -errno;
 4409		goto err_close_new_fd;
 4410	}
 4411
 4412	err = zclose(map->fd);
 4413	if (err) {
 4414		err = -errno;
 4415		goto err_close_new_fd;
 4416	}
 4417	free(map->name);
 4418
 4419	map->fd = new_fd;
 4420	map->name = new_name;
 4421	map->def.type = info.type;
 4422	map->def.key_size = info.key_size;
 4423	map->def.value_size = info.value_size;
 4424	map->def.max_entries = info.max_entries;
 4425	map->def.map_flags = info.map_flags;
 4426	map->btf_key_type_id = info.btf_key_type_id;
 4427	map->btf_value_type_id = info.btf_value_type_id;
 4428	map->reused = true;
 4429	map->map_extra = info.map_extra;
 4430
 4431	return 0;
 4432
 4433err_close_new_fd:
 4434	close(new_fd);
 4435err_free_new_name:
 4436	free(new_name);
 4437	return libbpf_err(err);
 4438}
 4439
 4440__u32 bpf_map__max_entries(const struct bpf_map *map)
 4441{
 4442	return map->def.max_entries;
 4443}
 4444
 4445struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
 4446{
 4447	if (!bpf_map_type__is_map_in_map(map->def.type))
 4448		return errno = EINVAL, NULL;
 4449
 4450	return map->inner_map;
 4451}
 4452
 4453int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
 4454{
 4455	if (map->obj->loaded)
 4456		return libbpf_err(-EBUSY);
 4457
 4458	map->def.max_entries = max_entries;
 4459
 4460	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
 4461	if (map_is_ringbuf(map))
 4462		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
 4463
 4464	return 0;
 4465}
 4466
 4467static int
 4468bpf_object__probe_loading(struct bpf_object *obj)
 4469{
 4470	char *cp, errmsg[STRERR_BUFSIZE];
 4471	struct bpf_insn insns[] = {
 4472		BPF_MOV64_IMM(BPF_REG_0, 0),
 4473		BPF_EXIT_INSN(),
 4474	};
 4475	int ret, insn_cnt = ARRAY_SIZE(insns);
 4476
 4477	if (obj->gen_loader)
 4478		return 0;
 4479
 4480	ret = bump_rlimit_memlock();
 4481	if (ret)
 4482		pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
 4483
 4484	/* make sure basic loading works */
 4485	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
 4486	if (ret < 0)
 4487		ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
 4488	if (ret < 0) {
 4489		ret = errno;
 4490		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
 4491		pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
 4492			"program. Make sure your kernel supports BPF "
 4493			"(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
 4494			"set to big enough value.\n", __func__, cp, ret);
 4495		return -ret;
 4496	}
 4497	close(ret);
 4498
 4499	return 0;
 4500}
 4501
 4502static int probe_fd(int fd)
 4503{
 4504	if (fd >= 0)
 4505		close(fd);
 4506	return fd >= 0;
 4507}
 4508
 4509static int probe_kern_prog_name(void)
 4510{
 4511	const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
 4512	struct bpf_insn insns[] = {
 4513		BPF_MOV64_IMM(BPF_REG_0, 0),
 4514		BPF_EXIT_INSN(),
 4515	};
 4516	union bpf_attr attr;
 4517	int ret;
 4518
 4519	memset(&attr, 0, attr_sz);
 4520	attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
 4521	attr.license = ptr_to_u64("GPL");
 4522	attr.insns = ptr_to_u64(insns);
 4523	attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
 4524	libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
 4525
 4526	/* make sure loading with name works */
 4527	ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
 4528	return probe_fd(ret);
 4529}
 4530
 4531static int probe_kern_global_data(void)
 4532{
 4533	char *cp, errmsg[STRERR_BUFSIZE];
 4534	struct bpf_insn insns[] = {
 4535		BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
 4536		BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
 4537		BPF_MOV64_IMM(BPF_REG_0, 0),
 4538		BPF_EXIT_INSN(),
 4539	};
 4540	int ret, map, insn_cnt = ARRAY_SIZE(insns);
 4541
 4542	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
 4543	if (map < 0) {
 4544		ret = -errno;
 4545		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
 4546		pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
 4547			__func__, cp, -ret);
 4548		return ret;
 4549	}
 4550
 4551	insns[0].imm = map;
 4552
 4553	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
 4554	close(map);
 4555	return probe_fd(ret);
 4556}
 4557
 4558static int probe_kern_btf(void)
 4559{
 4560	static const char strs[] = "\0int";
 4561	__u32 types[] = {
 4562		/* int */
 4563		BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
 4564	};
 4565
 4566	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4567					     strs, sizeof(strs)));
 4568}
 4569
 4570static int probe_kern_btf_func(void)
 4571{
 4572	static const char strs[] = "\0int\0x\0a";
 4573	/* void x(int a) {} */
 4574	__u32 types[] = {
 4575		/* int */
 4576		BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
 4577		/* FUNC_PROTO */                                /* [2] */
 4578		BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
 4579		BTF_PARAM_ENC(7, 1),
 4580		/* FUNC x */                                    /* [3] */
 4581		BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
 4582	};
 4583
 4584	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4585					     strs, sizeof(strs)));
 4586}
 4587
 4588static int probe_kern_btf_func_global(void)
 4589{
 4590	static const char strs[] = "\0int\0x\0a";
 4591	/* static void x(int a) {} */
 4592	__u32 types[] = {
 4593		/* int */
 4594		BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
 4595		/* FUNC_PROTO */                                /* [2] */
 4596		BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
 4597		BTF_PARAM_ENC(7, 1),
 4598		/* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
 4599		BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
 4600	};
 4601
 4602	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4603					     strs, sizeof(strs)));
 4604}
 4605
 4606static int probe_kern_btf_datasec(void)
 4607{
 4608	static const char strs[] = "\0x\0.data";
 4609	/* static int a; */
 4610	__u32 types[] = {
 4611		/* int */
 4612		BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
 4613		/* VAR x */                                     /* [2] */
 4614		BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
 4615		BTF_VAR_STATIC,
 4616		/* DATASEC val */                               /* [3] */
 4617		BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
 4618		BTF_VAR_SECINFO_ENC(2, 0, 4),
 4619	};
 4620
 4621	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4622					     strs, sizeof(strs)));
 4623}
 4624
 4625static int probe_kern_btf_float(void)
 4626{
 4627	static const char strs[] = "\0float";
 4628	__u32 types[] = {
 4629		/* float */
 4630		BTF_TYPE_FLOAT_ENC(1, 4),
 4631	};
 4632
 4633	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4634					     strs, sizeof(strs)));
 4635}
 4636
 4637static int probe_kern_btf_decl_tag(void)
 4638{
 4639	static const char strs[] = "\0tag";
 4640	__u32 types[] = {
 4641		/* int */
 4642		BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
 4643		/* VAR x */                                     /* [2] */
 4644		BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
 4645		BTF_VAR_STATIC,
 4646		/* attr */
 4647		BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
 4648	};
 4649
 4650	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4651					     strs, sizeof(strs)));
 4652}
 4653
 4654static int probe_kern_btf_type_tag(void)
 4655{
 4656	static const char strs[] = "\0tag";
 4657	__u32 types[] = {
 4658		/* int */
 4659		BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),		/* [1] */
 4660		/* attr */
 4661		BTF_TYPE_TYPE_TAG_ENC(1, 1),				/* [2] */
 4662		/* ptr */
 4663		BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),	/* [3] */
 4664	};
 4665
 4666	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4667					     strs, sizeof(strs)));
 4668}
 4669
 4670static int probe_kern_array_mmap(void)
 4671{
 4672	LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
 4673	int fd;
 4674
 4675	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
 4676	return probe_fd(fd);
 4677}
 4678
 4679static int probe_kern_exp_attach_type(void)
 4680{
 4681	LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
 4682	struct bpf_insn insns[] = {
 4683		BPF_MOV64_IMM(BPF_REG_0, 0),
 4684		BPF_EXIT_INSN(),
 4685	};
 4686	int fd, insn_cnt = ARRAY_SIZE(insns);
 4687
 4688	/* use any valid combination of program type and (optional)
 4689	 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
 4690	 * to see if kernel supports expected_attach_type field for
 4691	 * BPF_PROG_LOAD command
 4692	 */
 4693	fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
 4694	return probe_fd(fd);
 4695}
 4696
 4697static int probe_kern_probe_read_kernel(void)
 4698{
 4699	struct bpf_insn insns[] = {
 4700		BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),	/* r1 = r10 (fp) */
 4701		BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),	/* r1 += -8 */
 4702		BPF_MOV64_IMM(BPF_REG_2, 8),		/* r2 = 8 */
 4703		BPF_MOV64_IMM(BPF_REG_3, 0),		/* r3 = 0 */
 4704		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
 4705		BPF_EXIT_INSN(),
 4706	};
 4707	int fd, insn_cnt = ARRAY_SIZE(insns);
 4708
 4709	fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
 4710	return probe_fd(fd);
 4711}
 4712
 4713static int probe_prog_bind_map(void)
 4714{
 4715	char *cp, errmsg[STRERR_BUFSIZE];
 4716	struct bpf_insn insns[] = {
 4717		BPF_MOV64_IMM(BPF_REG_0, 0),
 4718		BPF_EXIT_INSN(),
 4719	};
 4720	int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
 4721
 4722	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
 4723	if (map < 0) {
 4724		ret = -errno;
 4725		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
 4726		pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
 4727			__func__, cp, -ret);
 4728		return ret;
 4729	}
 4730
 4731	prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
 4732	if (prog < 0) {
 4733		close(map);
 4734		return 0;
 4735	}
 4736
 4737	ret = bpf_prog_bind_map(prog, map, NULL);
 4738
 4739	close(map);
 4740	close(prog);
 4741
 4742	return ret >= 0;
 4743}
 4744
 4745static int probe_module_btf(void)
 4746{
 4747	static const char strs[] = "\0int";
 4748	__u32 types[] = {
 4749		/* int */
 4750		BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
 4751	};
 4752	struct bpf_btf_info info;
 4753	__u32 len = sizeof(info);
 4754	char name[16];
 4755	int fd, err;
 4756
 4757	fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
 4758	if (fd < 0)
 4759		return 0; /* BTF not supported at all */
 4760
 4761	memset(&info, 0, sizeof(info));
 4762	info.name = ptr_to_u64(name);
 4763	info.name_len = sizeof(name);
 4764
 4765	/* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
 4766	 * kernel's module BTF support coincides with support for
 4767	 * name/name_len fields in struct bpf_btf_info.
 4768	 */
 4769	err = bpf_obj_get_info_by_fd(fd, &info, &len);
 4770	close(fd);
 4771	return !err;
 4772}
 4773
 4774static int probe_perf_link(void)
 4775{
 4776	struct bpf_insn insns[] = {
 4777		BPF_MOV64_IMM(BPF_REG_0, 0),
 4778		BPF_EXIT_INSN(),
 4779	};
 4780	int prog_fd, link_fd, err;
 4781
 4782	prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
 4783				insns, ARRAY_SIZE(insns), NULL);
 4784	if (prog_fd < 0)
 4785		return -errno;
 4786
 4787	/* use invalid perf_event FD to get EBADF, if link is supported;
 4788	 * otherwise EINVAL should be returned
 4789	 */
 4790	link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
 4791	err = -errno; /* close() can clobber errno */
 4792
 4793	if (link_fd >= 0)
 4794		close(link_fd);
 4795	close(prog_fd);
 4796
 4797	return link_fd < 0 && err == -EBADF;
 4798}
 4799
 4800static int probe_kern_bpf_cookie(void)
 4801{
 4802	struct bpf_insn insns[] = {
 4803		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
 4804		BPF_EXIT_INSN(),
 4805	};
 4806	int ret, insn_cnt = ARRAY_SIZE(insns);
 4807
 4808	ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
 4809	return probe_fd(ret);
 4810}
 4811
 4812static int probe_kern_btf_enum64(void)
 4813{
 4814	static const char strs[] = "\0enum64";
 4815	__u32 types[] = {
 4816		BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
 4817	};
 4818
 4819	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
 4820					     strs, sizeof(strs)));
 4821}
 4822
 4823static int probe_kern_syscall_wrapper(void);
 4824
 4825enum kern_feature_result {
 4826	FEAT_UNKNOWN = 0,
 4827	FEAT_SUPPORTED = 1,
 4828	FEAT_MISSING = 2,
 4829};
 4830
 4831typedef int (*feature_probe_fn)(void);
 4832
 4833static struct kern_feature_desc {
 4834	const char *desc;
 4835	feature_probe_fn probe;
 4836	enum kern_feature_result res;
 4837} feature_probes[__FEAT_CNT] = {
 4838	[FEAT_PROG_NAME] = {
 4839		"BPF program name", probe_kern_prog_name,
 4840	},
 4841	[FEAT_GLOBAL_DATA] = {
 4842		"global variables", probe_kern_global_data,
 4843	},
 4844	[FEAT_BTF] = {
 4845		"minimal BTF", probe_kern_btf,
 4846	},
 4847	[FEAT_BTF_FUNC] = {
 4848		"BTF functions", probe_kern_btf_func,
 4849	},
 4850	[FEAT_BTF_GLOBAL_FUNC] = {
 4851		"BTF global function", probe_kern_btf_func_global,
 4852	},
 4853	[FEAT_BTF_DATASEC] = {
 4854		"BTF data section and variable", probe_kern_btf_datasec,
 4855	},
 4856	[FEAT_ARRAY_MMAP] = {
 4857		"ARRAY map mmap()", probe_kern_array_mmap,
 4858	},
 4859	[FEAT_EXP_ATTACH_TYPE] = {
 4860		"BPF_PROG_LOAD expected_attach_type attribute",
 4861		probe_kern_exp_attach_type,
 4862	},
 4863	[FEAT_PROBE_READ_KERN] = {
 4864		"bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
 4865	},
 4866	[FEAT_PROG_BIND_MAP] = {
 4867		"BPF_PROG_BIND_MAP support", probe_prog_bind_map,
 4868	},
 4869	[FEAT_MODULE_BTF] = {
 4870		"module BTF support", probe_module_btf,
 4871	},
 4872	[FEAT_BTF_FLOAT] = {
 4873		"BTF_KIND_FLOAT support", probe_kern_btf_float,
 4874	},
 4875	[FEAT_PERF_LINK] = {
 4876		"BPF perf link support", probe_perf_link,
 4877	},
 4878	[FEAT_BTF_DECL_TAG] = {
 4879		"BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
 4880	},
 4881	[FEAT_BTF_TYPE_TAG] = {
 4882		"BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
 4883	},
 4884	[FEAT_MEMCG_ACCOUNT] = {
 4885		"memcg-based memory accounting", probe_memcg_account,
 4886	},
 4887	[FEAT_BPF_COOKIE] = {
 4888		"BPF cookie support", probe_kern_bpf_cookie,
 4889	},
 4890	[FEAT_BTF_ENUM64] = {
 4891		"BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
 4892	},
 4893	[FEAT_SYSCALL_WRAPPER] = {
 4894		"Kernel using syscall wrapper", probe_kern_syscall_wrapper,
 4895	},
 4896};
 4897
 4898bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
 4899{
 4900	struct kern_feature_desc *feat = &feature_probes[feat_id];
 4901	int ret;
 4902
 4903	if (obj && obj->gen_loader)
 4904		/* To generate loader program assume the latest kernel
 4905		 * to avoid doing extra prog_load, map_create syscalls.
 4906		 */
 4907		return true;
 4908
 4909	if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
 4910		ret = feat->probe();
 4911		if (ret > 0) {
 4912			WRITE_ONCE(feat->res, FEAT_SUPPORTED);
 4913		} else if (ret == 0) {
 4914			WRITE_ONCE(feat->res, FEAT_MISSING);
 4915		} else {
 4916			pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
 4917			WRITE_ONCE(feat->res, FEAT_MISSING);
 4918		}
 4919	}
 4920
 4921	return READ_ONCE(feat->res) == FEAT_SUPPORTED;
 4922}
 4923
 4924static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
 4925{
 4926	struct bpf_map_info map_info;
 4927	char msg[STRERR_BUFSIZE];
 4928	__u32 map_info_len = sizeof(map_info);
 4929	int err;
 4930
 4931	memset(&map_info, 0, map_info_len);
 4932	err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
 4933	if (err && errno == EINVAL)
 4934		err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
 4935	if (err) {
 4936		pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
 4937			libbpf_strerror_r(errno, msg, sizeof(msg)));
 4938		return false;
 4939	}
 4940
 4941	return (map_info.type == map->def.type &&
 4942		map_info.key_size == map->def.key_size &&
 4943		map_info.value_size == map->def.value_size &&
 4944		map_info.max_entries == map->def.max_entries &&
 4945		map_info.map_flags == map->def.map_flags &&
 4946		map_info.map_extra == map->map_extra);
 4947}
 4948
 4949static int
 4950bpf_object__reuse_map(struct bpf_map *map)
 4951{
 4952	char *cp, errmsg[STRERR_BUFSIZE];
 4953	int err, pin_fd;
 4954
 4955	pin_fd = bpf_obj_get(map->pin_path);
 4956	if (pin_fd < 0) {
 4957		err = -errno;
 4958		if (err == -ENOENT) {
 4959			pr_debug("found no pinned map to reuse at '%s'\n",
 4960				 map->pin_path);
 4961			return 0;
 4962		}
 4963
 4964		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
 4965		pr_warn("couldn't retrieve pinned map '%s': %s\n",
 4966			map->pin_path, cp);
 4967		return err;
 4968	}
 4969
 4970	if (!map_is_reuse_compat(map, pin_fd)) {
 4971		pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
 4972			map->pin_path);
 4973		close(pin_fd);
 4974		return -EINVAL;
 4975	}
 4976
 4977	err = bpf_map__reuse_fd(map, pin_fd);
 4978	close(pin_fd);
 4979	if (err)
 4980		return err;
 4981
 4982	map->pinned = true;
 4983	pr_debug("reused pinned map at '%s'\n", map->pin_path);
 4984
 4985	return 0;
 4986}
 4987
 4988static int
 4989bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
 4990{
 4991	enum libbpf_map_type map_type = map->libbpf_type;
 4992	char *cp, errmsg[STRERR_BUFSIZE];
 4993	int err, zero = 0;
 4994
 4995	if (obj->gen_loader) {
 4996		bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
 4997					 map->mmaped, map->def.value_size);
 4998		if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
 4999			bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
 5000		return 0;
 5001	}
 5002	err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
 5003	if (err) {
 5004		err = -errno;
 5005		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 5006		pr_warn("Error setting initial map(%s) contents: %s\n",
 5007			map->name, cp);
 5008		return err;
 5009	}
 5010
 5011	/* Freeze .rodata and .kconfig map as read-only from syscall side. */
 5012	if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
 5013		err = bpf_map_freeze(map->fd);
 5014		if (err) {
 5015			err = -errno;
 5016			cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 5017			pr_warn("Error freezing map(%s) as read-only: %s\n",
 5018				map->name, cp);
 5019			return err;
 5020		}
 5021	}
 5022	return 0;
 5023}
 5024
 5025static void bpf_map__destroy(struct bpf_map *map);
 5026
 5027static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
 5028{
 5029	LIBBPF_OPTS(bpf_map_create_opts, create_attr);
 5030	struct bpf_map_def *def = &map->def;
 5031	const char *map_name = NULL;
 5032	int err = 0;
 5033
 5034	if (kernel_supports(obj, FEAT_PROG_NAME))
 5035		map_name = map->name;
 5036	create_attr.map_ifindex = map->map_ifindex;
 5037	create_attr.map_flags = def->map_flags;
 5038	create_attr.numa_node = map->numa_node;
 5039	create_attr.map_extra = map->map_extra;
 5040
 5041	if (bpf_map__is_struct_ops(map))
 5042		create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
 5043
 5044	if (obj->btf && btf__fd(obj->btf) >= 0) {
 5045		create_attr.btf_fd = btf__fd(obj->btf);
 5046		create_attr.btf_key_type_id = map->btf_key_type_id;
 5047		create_attr.btf_value_type_id = map->btf_value_type_id;
 5048	}
 5049
 5050	if (bpf_map_type__is_map_in_map(def->type)) {
 5051		if (map->inner_map) {
 5052			err = bpf_object__create_map(obj, map->inner_map, true);
 5053			if (err) {
 5054				pr_warn("map '%s': failed to create inner map: %d\n",
 5055					map->name, err);
 5056				return err;
 5057			}
 5058			map->inner_map_fd = bpf_map__fd(map->inner_map);
 5059		}
 5060		if (map->inner_map_fd >= 0)
 5061			create_attr.inner_map_fd = map->inner_map_fd;
 5062	}
 5063
 5064	switch (def->type) {
 5065	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
 5066	case BPF_MAP_TYPE_CGROUP_ARRAY:
 5067	case BPF_MAP_TYPE_STACK_TRACE:
 5068	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
 5069	case BPF_MAP_TYPE_HASH_OF_MAPS:
 5070	case BPF_MAP_TYPE_DEVMAP:
 5071	case BPF_MAP_TYPE_DEVMAP_HASH:
 5072	case BPF_MAP_TYPE_CPUMAP:
 5073	case BPF_MAP_TYPE_XSKMAP:
 5074	case BPF_MAP_TYPE_SOCKMAP:
 5075	case BPF_MAP_TYPE_SOCKHASH:
 5076	case BPF_MAP_TYPE_QUEUE:
 5077	case BPF_MAP_TYPE_STACK:
 5078		create_attr.btf_fd = 0;
 5079		create_attr.btf_key_type_id = 0;
 5080		create_attr.btf_value_type_id = 0;
 5081		map->btf_key_type_id = 0;
 5082		map->btf_value_type_id = 0;
 5083	default:
 5084		break;
 5085	}
 5086
 5087	if (obj->gen_loader) {
 5088		bpf_gen__map_create(obj->gen_loader, def->type, map_name,
 5089				    def->key_size, def->value_size, def->max_entries,
 5090				    &create_attr, is_inner ? -1 : map - obj->maps);
 5091		/* Pretend to have valid FD to pass various fd >= 0 checks.
 5092		 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
 5093		 */
 5094		map->fd = 0;
 5095	} else {
 5096		map->fd = bpf_map_create(def->type, map_name,
 5097					 def->key_size, def->value_size,
 5098					 def->max_entries, &create_attr);
 5099	}
 5100	if (map->fd < 0 && (create_attr.btf_key_type_id ||
 5101			    create_attr.btf_value_type_id)) {
 5102		char *cp, errmsg[STRERR_BUFSIZE];
 5103
 5104		err = -errno;
 5105		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 5106		pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
 5107			map->name, cp, err);
 5108		create_attr.btf_fd = 0;
 5109		create_attr.btf_key_type_id = 0;
 5110		create_attr.btf_value_type_id = 0;
 5111		map->btf_key_type_id = 0;
 5112		map->btf_value_type_id = 0;
 5113		map->fd = bpf_map_create(def->type, map_name,
 5114					 def->key_size, def->value_size,
 5115					 def->max_entries, &create_attr);
 5116	}
 5117
 5118	err = map->fd < 0 ? -errno : 0;
 5119
 5120	if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
 5121		if (obj->gen_loader)
 5122			map->inner_map->fd = -1;
 5123		bpf_map__destroy(map->inner_map);
 5124		zfree(&map->inner_map);
 5125	}
 5126
 5127	return err;
 5128}
 5129
 5130static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
 5131{
 5132	const struct bpf_map *targ_map;
 5133	unsigned int i;
 5134	int fd, err = 0;
 5135
 5136	for (i = 0; i < map->init_slots_sz; i++) {
 5137		if (!map->init_slots[i])
 5138			continue;
 5139
 5140		targ_map = map->init_slots[i];
 5141		fd = bpf_map__fd(targ_map);
 5142
 5143		if (obj->gen_loader) {
 5144			bpf_gen__populate_outer_map(obj->gen_loader,
 5145						    map - obj->maps, i,
 5146						    targ_map - obj->maps);
 5147		} else {
 5148			err = bpf_map_update_elem(map->fd, &i, &fd, 0);
 5149		}
 5150		if (err) {
 5151			err = -errno;
 5152			pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
 5153				map->name, i, targ_map->name, fd, err);
 5154			return err;
 5155		}
 5156		pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
 5157			 map->name, i, targ_map->name, fd);
 5158	}
 5159
 5160	zfree(&map->init_slots);
 5161	map->init_slots_sz = 0;
 5162
 5163	return 0;
 5164}
 5165
 5166static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
 5167{
 5168	const struct bpf_program *targ_prog;
 5169	unsigned int i;
 5170	int fd, err;
 5171
 5172	if (obj->gen_loader)
 5173		return -ENOTSUP;
 5174
 5175	for (i = 0; i < map->init_slots_sz; i++) {
 5176		if (!map->init_slots[i])
 5177			continue;
 5178
 5179		targ_prog = map->init_slots[i];
 5180		fd = bpf_program__fd(targ_prog);
 5181
 5182		err = bpf_map_update_elem(map->fd, &i, &fd, 0);
 5183		if (err) {
 5184			err = -errno;
 5185			pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
 5186				map->name, i, targ_prog->name, fd, err);
 5187			return err;
 5188		}
 5189		pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
 5190			 map->name, i, targ_prog->name, fd);
 5191	}
 5192
 5193	zfree(&map->init_slots);
 5194	map->init_slots_sz = 0;
 5195
 5196	return 0;
 5197}
 5198
 5199static int bpf_object_init_prog_arrays(struct bpf_object *obj)
 5200{
 5201	struct bpf_map *map;
 5202	int i, err;
 5203
 5204	for (i = 0; i < obj->nr_maps; i++) {
 5205		map = &obj->maps[i];
 
 5206
 5207		if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
 5208			continue;
 5209
 5210		err = init_prog_array_slots(obj, map);
 5211		if (err < 0) {
 5212			zclose(map->fd);
 
 
 
 
 
 
 5213			return err;
 5214		}
 5215	}
 5216	return 0;
 5217}
 5218
 5219static int map_set_def_max_entries(struct bpf_map *map)
 5220{
 5221	if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
 5222		int nr_cpus;
 5223
 5224		nr_cpus = libbpf_num_possible_cpus();
 5225		if (nr_cpus < 0) {
 5226			pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
 5227				map->name, nr_cpus);
 5228			return nr_cpus;
 5229		}
 5230		pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
 5231		map->def.max_entries = nr_cpus;
 5232	}
 5233
 5234	return 0;
 5235}
 5236
 5237static int
 5238bpf_object__create_maps(struct bpf_object *obj)
 5239{
 5240	struct bpf_map *map;
 5241	char *cp, errmsg[STRERR_BUFSIZE];
 5242	unsigned int i, j;
 5243	int err;
 5244	bool retried;
 5245
 5246	for (i = 0; i < obj->nr_maps; i++) {
 5247		map = &obj->maps[i];
 5248
 5249		/* To support old kernels, we skip creating global data maps
 5250		 * (.rodata, .data, .kconfig, etc); later on, during program
 5251		 * loading, if we detect that at least one of the to-be-loaded
 5252		 * programs is referencing any global data map, we'll error
 5253		 * out with program name and relocation index logged.
 5254		 * This approach allows to accommodate Clang emitting
 5255		 * unnecessary .rodata.str1.1 sections for string literals,
 5256		 * but also it allows to have CO-RE applications that use
 5257		 * global variables in some of BPF programs, but not others.
 5258		 * If those global variable-using programs are not loaded at
 5259		 * runtime due to bpf_program__set_autoload(prog, false),
 5260		 * bpf_object loading will succeed just fine even on old
 5261		 * kernels.
 5262		 */
 5263		if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
 5264			map->autocreate = false;
 5265
 5266		if (!map->autocreate) {
 5267			pr_debug("map '%s': skipped auto-creating...\n", map->name);
 5268			continue;
 5269		}
 5270
 5271		err = map_set_def_max_entries(map);
 5272		if (err)
 5273			goto err_out;
 5274
 5275		retried = false;
 5276retry:
 5277		if (map->pin_path) {
 5278			err = bpf_object__reuse_map(map);
 5279			if (err) {
 5280				pr_warn("map '%s': error reusing pinned map\n",
 5281					map->name);
 5282				goto err_out;
 5283			}
 5284			if (retried && map->fd < 0) {
 5285				pr_warn("map '%s': cannot find pinned map\n",
 5286					map->name);
 5287				err = -ENOENT;
 5288				goto err_out;
 5289			}
 5290		}
 5291
 5292		if (map->fd >= 0) {
 5293			pr_debug("map '%s': skipping creation (preset fd=%d)\n",
 5294				 map->name, map->fd);
 5295		} else {
 5296			err = bpf_object__create_map(obj, map, false);
 5297			if (err)
 5298				goto err_out;
 5299
 5300			pr_debug("map '%s': created successfully, fd=%d\n",
 5301				 map->name, map->fd);
 5302
 5303			if (bpf_map__is_internal(map)) {
 5304				err = bpf_object__populate_internal_map(obj, map);
 5305				if (err < 0) {
 5306					zclose(map->fd);
 5307					goto err_out;
 5308				}
 5309			}
 5310
 5311			if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
 5312				err = init_map_in_map_slots(obj, map);
 5313				if (err < 0) {
 5314					zclose(map->fd);
 5315					goto err_out;
 5316				}
 5317			}
 5318		}
 5319
 5320		if (map->pin_path && !map->pinned) {
 5321			err = bpf_map__pin(map, NULL);
 5322			if (err) {
 5323				zclose(map->fd);
 5324				if (!retried && err == -EEXIST) {
 5325					retried = true;
 5326					goto retry;
 5327				}
 5328				pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
 5329					map->name, map->pin_path, err);
 5330				goto err_out;
 5331			}
 5332		}
 5333	}
 5334
 5335	return 0;
 5336
 5337err_out:
 5338	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 5339	pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
 5340	pr_perm_msg(err);
 5341	for (j = 0; j < i; j++)
 5342		zclose(obj->maps[j].fd);
 5343	return err;
 5344}
 5345
 5346static bool bpf_core_is_flavor_sep(const char *s)
 5347{
 5348	/* check X___Y name pattern, where X and Y are not underscores */
 5349	return s[0] != '_' &&				      /* X */
 5350	       s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
 5351	       s[4] != '_';				      /* Y */
 5352}
 5353
 5354/* Given 'some_struct_name___with_flavor' return the length of a name prefix
 5355 * before last triple underscore. Struct name part after last triple
 5356 * underscore is ignored by BPF CO-RE relocation during relocation matching.
 5357 */
 5358size_t bpf_core_essential_name_len(const char *name)
 5359{
 5360	size_t n = strlen(name);
 5361	int i;
 5362
 5363	for (i = n - 5; i >= 0; i--) {
 5364		if (bpf_core_is_flavor_sep(name + i))
 5365			return i + 1;
 5366	}
 5367	return n;
 5368}
 5369
 5370void bpf_core_free_cands(struct bpf_core_cand_list *cands)
 5371{
 5372	if (!cands)
 5373		return;
 5374
 5375	free(cands->cands);
 5376	free(cands);
 5377}
 5378
 5379int bpf_core_add_cands(struct bpf_core_cand *local_cand,
 5380		       size_t local_essent_len,
 5381		       const struct btf *targ_btf,
 5382		       const char *targ_btf_name,
 5383		       int targ_start_id,
 5384		       struct bpf_core_cand_list *cands)
 5385{
 5386	struct bpf_core_cand *new_cands, *cand;
 5387	const struct btf_type *t, *local_t;
 5388	const char *targ_name, *local_name;
 5389	size_t targ_essent_len;
 5390	int n, i;
 5391
 5392	local_t = btf__type_by_id(local_cand->btf, local_cand->id);
 5393	local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
 5394
 5395	n = btf__type_cnt(targ_btf);
 5396	for (i = targ_start_id; i < n; i++) {
 5397		t = btf__type_by_id(targ_btf, i);
 5398		if (!btf_kind_core_compat(t, local_t))
 5399			continue;
 5400
 5401		targ_name = btf__name_by_offset(targ_btf, t->name_off);
 5402		if (str_is_empty(targ_name))
 5403			continue;
 5404
 5405		targ_essent_len = bpf_core_essential_name_len(targ_name);
 5406		if (targ_essent_len != local_essent_len)
 5407			continue;
 5408
 5409		if (strncmp(local_name, targ_name, local_essent_len) != 0)
 5410			continue;
 5411
 5412		pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
 5413			 local_cand->id, btf_kind_str(local_t),
 5414			 local_name, i, btf_kind_str(t), targ_name,
 5415			 targ_btf_name);
 5416		new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
 5417					      sizeof(*cands->cands));
 5418		if (!new_cands)
 5419			return -ENOMEM;
 5420
 5421		cand = &new_cands[cands->len];
 5422		cand->btf = targ_btf;
 5423		cand->id = i;
 5424
 5425		cands->cands = new_cands;
 5426		cands->len++;
 5427	}
 5428	return 0;
 5429}
 5430
 5431static int load_module_btfs(struct bpf_object *obj)
 5432{
 5433	struct bpf_btf_info info;
 5434	struct module_btf *mod_btf;
 5435	struct btf *btf;
 5436	char name[64];
 5437	__u32 id = 0, len;
 5438	int err, fd;
 5439
 5440	if (obj->btf_modules_loaded)
 5441		return 0;
 5442
 5443	if (obj->gen_loader)
 5444		return 0;
 5445
 5446	/* don't do this again, even if we find no module BTFs */
 5447	obj->btf_modules_loaded = true;
 5448
 5449	/* kernel too old to support module BTFs */
 5450	if (!kernel_supports(obj, FEAT_MODULE_BTF))
 5451		return 0;
 5452
 5453	while (true) {
 5454		err = bpf_btf_get_next_id(id, &id);
 5455		if (err && errno == ENOENT)
 5456			return 0;
 5457		if (err) {
 5458			err = -errno;
 5459			pr_warn("failed to iterate BTF objects: %d\n", err);
 5460			return err;
 5461		}
 5462
 5463		fd = bpf_btf_get_fd_by_id(id);
 5464		if (fd < 0) {
 5465			if (errno == ENOENT)
 5466				continue; /* expected race: BTF was unloaded */
 5467			err = -errno;
 5468			pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
 5469			return err;
 5470		}
 5471
 5472		len = sizeof(info);
 5473		memset(&info, 0, sizeof(info));
 5474		info.name = ptr_to_u64(name);
 5475		info.name_len = sizeof(name);
 5476
 5477		err = bpf_obj_get_info_by_fd(fd, &info, &len);
 5478		if (err) {
 5479			err = -errno;
 5480			pr_warn("failed to get BTF object #%d info: %d\n", id, err);
 5481			goto err_out;
 5482		}
 5483
 5484		/* ignore non-module BTFs */
 5485		if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
 5486			close(fd);
 5487			continue;
 5488		}
 5489
 5490		btf = btf_get_from_fd(fd, obj->btf_vmlinux);
 5491		err = libbpf_get_error(btf);
 5492		if (err) {
 5493			pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
 5494				name, id, err);
 5495			goto err_out;
 5496		}
 5497
 5498		err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
 5499					sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
 5500		if (err)
 5501			goto err_out;
 5502
 5503		mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
 5504
 5505		mod_btf->btf = btf;
 5506		mod_btf->id = id;
 5507		mod_btf->fd = fd;
 5508		mod_btf->name = strdup(name);
 5509		if (!mod_btf->name) {
 5510			err = -ENOMEM;
 5511			goto err_out;
 5512		}
 5513		continue;
 5514
 5515err_out:
 5516		close(fd);
 5517		return err;
 5518	}
 5519
 5520	return 0;
 5521}
 5522
 5523static struct bpf_core_cand_list *
 5524bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
 5525{
 5526	struct bpf_core_cand local_cand = {};
 5527	struct bpf_core_cand_list *cands;
 5528	const struct btf *main_btf;
 5529	const struct btf_type *local_t;
 5530	const char *local_name;
 5531	size_t local_essent_len;
 5532	int err, i;
 5533
 5534	local_cand.btf = local_btf;
 5535	local_cand.id = local_type_id;
 5536	local_t = btf__type_by_id(local_btf, local_type_id);
 5537	if (!local_t)
 5538		return ERR_PTR(-EINVAL);
 5539
 5540	local_name = btf__name_by_offset(local_btf, local_t->name_off);
 5541	if (str_is_empty(local_name))
 5542		return ERR_PTR(-EINVAL);
 5543	local_essent_len = bpf_core_essential_name_len(local_name);
 5544
 5545	cands = calloc(1, sizeof(*cands));
 5546	if (!cands)
 5547		return ERR_PTR(-ENOMEM);
 5548
 5549	/* Attempt to find target candidates in vmlinux BTF first */
 5550	main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
 5551	err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
 5552	if (err)
 5553		goto err_out;
 5554
 5555	/* if vmlinux BTF has any candidate, don't got for module BTFs */
 5556	if (cands->len)
 5557		return cands;
 5558
 5559	/* if vmlinux BTF was overridden, don't attempt to load module BTFs */
 5560	if (obj->btf_vmlinux_override)
 5561		return cands;
 5562
 5563	/* now look through module BTFs, trying to still find candidates */
 5564	err = load_module_btfs(obj);
 5565	if (err)
 5566		goto err_out;
 5567
 5568	for (i = 0; i < obj->btf_module_cnt; i++) {
 5569		err = bpf_core_add_cands(&local_cand, local_essent_len,
 5570					 obj->btf_modules[i].btf,
 5571					 obj->btf_modules[i].name,
 5572					 btf__type_cnt(obj->btf_vmlinux),
 5573					 cands);
 5574		if (err)
 5575			goto err_out;
 5576	}
 5577
 5578	return cands;
 5579err_out:
 5580	bpf_core_free_cands(cands);
 5581	return ERR_PTR(err);
 5582}
 5583
 5584/* Check local and target types for compatibility. This check is used for
 5585 * type-based CO-RE relocations and follow slightly different rules than
 5586 * field-based relocations. This function assumes that root types were already
 5587 * checked for name match. Beyond that initial root-level name check, names
 5588 * are completely ignored. Compatibility rules are as follows:
 5589 *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
 5590 *     kind should match for local and target types (i.e., STRUCT is not
 5591 *     compatible with UNION);
 5592 *   - for ENUMs, the size is ignored;
 5593 *   - for INT, size and signedness are ignored;
 5594 *   - for ARRAY, dimensionality is ignored, element types are checked for
 5595 *     compatibility recursively;
 5596 *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
 5597 *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
 5598 *   - FUNC_PROTOs are compatible if they have compatible signature: same
 5599 *     number of input args and compatible return and argument types.
 5600 * These rules are not set in stone and probably will be adjusted as we get
 5601 * more experience with using BPF CO-RE relocations.
 5602 */
 5603int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
 5604			      const struct btf *targ_btf, __u32 targ_id)
 5605{
 5606	return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
 5607}
 5608
 5609int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
 5610			 const struct btf *targ_btf, __u32 targ_id)
 5611{
 5612	return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
 5613}
 5614
 5615static size_t bpf_core_hash_fn(const long key, void *ctx)
 5616{
 5617	return key;
 5618}
 5619
 5620static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx)
 5621{
 5622	return k1 == k2;
 5623}
 5624
 5625static int record_relo_core(struct bpf_program *prog,
 5626			    const struct bpf_core_relo *core_relo, int insn_idx)
 5627{
 5628	struct reloc_desc *relos, *relo;
 5629
 5630	relos = libbpf_reallocarray(prog->reloc_desc,
 5631				    prog->nr_reloc + 1, sizeof(*relos));
 5632	if (!relos)
 5633		return -ENOMEM;
 5634	relo = &relos[prog->nr_reloc];
 5635	relo->type = RELO_CORE;
 5636	relo->insn_idx = insn_idx;
 5637	relo->core_relo = core_relo;
 5638	prog->reloc_desc = relos;
 5639	prog->nr_reloc++;
 5640	return 0;
 5641}
 5642
 5643static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
 5644{
 5645	struct reloc_desc *relo;
 5646	int i;
 5647
 5648	for (i = 0; i < prog->nr_reloc; i++) {
 5649		relo = &prog->reloc_desc[i];
 5650		if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
 5651			continue;
 5652
 5653		return relo->core_relo;
 5654	}
 5655
 5656	return NULL;
 5657}
 5658
 5659static int bpf_core_resolve_relo(struct bpf_program *prog,
 5660				 const struct bpf_core_relo *relo,
 5661				 int relo_idx,
 5662				 const struct btf *local_btf,
 5663				 struct hashmap *cand_cache,
 5664				 struct bpf_core_relo_res *targ_res)
 5665{
 5666	struct bpf_core_spec specs_scratch[3] = {};
 5667	struct bpf_core_cand_list *cands = NULL;
 5668	const char *prog_name = prog->name;
 5669	const struct btf_type *local_type;
 5670	const char *local_name;
 5671	__u32 local_id = relo->type_id;
 5672	int err;
 5673
 5674	local_type = btf__type_by_id(local_btf, local_id);
 5675	if (!local_type)
 5676		return -EINVAL;
 5677
 5678	local_name = btf__name_by_offset(local_btf, local_type->name_off);
 5679	if (!local_name)
 5680		return -EINVAL;
 5681
 5682	if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
 5683	    !hashmap__find(cand_cache, local_id, &cands)) {
 5684		cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
 5685		if (IS_ERR(cands)) {
 5686			pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
 5687				prog_name, relo_idx, local_id, btf_kind_str(local_type),
 5688				local_name, PTR_ERR(cands));
 5689			return PTR_ERR(cands);
 5690		}
 5691		err = hashmap__set(cand_cache, local_id, cands, NULL, NULL);
 5692		if (err) {
 5693			bpf_core_free_cands(cands);
 5694			return err;
 5695		}
 5696	}
 5697
 5698	return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
 5699				       targ_res);
 5700}
 5701
 5702static int
 5703bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
 5704{
 5705	const struct btf_ext_info_sec *sec;
 5706	struct bpf_core_relo_res targ_res;
 5707	const struct bpf_core_relo *rec;
 5708	const struct btf_ext_info *seg;
 5709	struct hashmap_entry *entry;
 5710	struct hashmap *cand_cache = NULL;
 5711	struct bpf_program *prog;
 5712	struct bpf_insn *insn;
 5713	const char *sec_name;
 5714	int i, err = 0, insn_idx, sec_idx, sec_num;
 5715
 5716	if (obj->btf_ext->core_relo_info.len == 0)
 5717		return 0;
 5718
 5719	if (targ_btf_path) {
 5720		obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
 5721		err = libbpf_get_error(obj->btf_vmlinux_override);
 5722		if (err) {
 5723			pr_warn("failed to parse target BTF: %d\n", err);
 5724			return err;
 5725		}
 5726	}
 5727
 5728	cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
 5729	if (IS_ERR(cand_cache)) {
 5730		err = PTR_ERR(cand_cache);
 5731		goto out;
 5732	}
 5733
 5734	seg = &obj->btf_ext->core_relo_info;
 5735	sec_num = 0;
 5736	for_each_btf_ext_sec(seg, sec) {
 5737		sec_idx = seg->sec_idxs[sec_num];
 5738		sec_num++;
 5739
 5740		sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
 5741		if (str_is_empty(sec_name)) {
 5742			err = -EINVAL;
 5743			goto out;
 5744		}
 5745
 5746		pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
 5747
 5748		for_each_btf_ext_rec(seg, sec, i, rec) {
 5749			if (rec->insn_off % BPF_INSN_SZ)
 5750				return -EINVAL;
 5751			insn_idx = rec->insn_off / BPF_INSN_SZ;
 5752			prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
 5753			if (!prog) {
 5754				/* When __weak subprog is "overridden" by another instance
 5755				 * of the subprog from a different object file, linker still
 5756				 * appends all the .BTF.ext info that used to belong to that
 5757				 * eliminated subprogram.
 5758				 * This is similar to what x86-64 linker does for relocations.
 5759				 * So just ignore such relocations just like we ignore
 5760				 * subprog instructions when discovering subprograms.
 5761				 */
 5762				pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
 5763					 sec_name, i, insn_idx);
 5764				continue;
 5765			}
 5766			/* no need to apply CO-RE relocation if the program is
 5767			 * not going to be loaded
 5768			 */
 5769			if (!prog->autoload)
 5770				continue;
 5771
 5772			/* adjust insn_idx from section frame of reference to the local
 5773			 * program's frame of reference; (sub-)program code is not yet
 5774			 * relocated, so it's enough to just subtract in-section offset
 5775			 */
 5776			insn_idx = insn_idx - prog->sec_insn_off;
 5777			if (insn_idx >= prog->insns_cnt)
 5778				return -EINVAL;
 5779			insn = &prog->insns[insn_idx];
 5780
 5781			err = record_relo_core(prog, rec, insn_idx);
 5782			if (err) {
 5783				pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
 5784					prog->name, i, err);
 5785				goto out;
 5786			}
 5787
 5788			if (prog->obj->gen_loader)
 5789				continue;
 5790
 5791			err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
 5792			if (err) {
 5793				pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
 5794					prog->name, i, err);
 5795				goto out;
 5796			}
 5797
 5798			err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
 5799			if (err) {
 5800				pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
 5801					prog->name, i, insn_idx, err);
 5802				goto out;
 5803			}
 5804		}
 5805	}
 5806
 5807out:
 5808	/* obj->btf_vmlinux and module BTFs are freed after object load */
 5809	btf__free(obj->btf_vmlinux_override);
 5810	obj->btf_vmlinux_override = NULL;
 5811
 5812	if (!IS_ERR_OR_NULL(cand_cache)) {
 5813		hashmap__for_each_entry(cand_cache, entry, i) {
 5814			bpf_core_free_cands(entry->pvalue);
 5815		}
 5816		hashmap__free(cand_cache);
 5817	}
 5818	return err;
 5819}
 5820
 5821/* base map load ldimm64 special constant, used also for log fixup logic */
 5822#define MAP_LDIMM64_POISON_BASE 2001000000
 5823#define MAP_LDIMM64_POISON_PFX "200100"
 5824
 5825static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
 5826			       int insn_idx, struct bpf_insn *insn,
 5827			       int map_idx, const struct bpf_map *map)
 5828{
 5829	int i;
 5830
 5831	pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
 5832		 prog->name, relo_idx, insn_idx, map_idx, map->name);
 5833
 5834	/* we turn single ldimm64 into two identical invalid calls */
 5835	for (i = 0; i < 2; i++) {
 5836		insn->code = BPF_JMP | BPF_CALL;
 5837		insn->dst_reg = 0;
 5838		insn->src_reg = 0;
 5839		insn->off = 0;
 5840		/* if this instruction is reachable (not a dead code),
 5841		 * verifier will complain with something like:
 5842		 * invalid func unknown#2001000123
 5843		 * where lower 123 is map index into obj->maps[] array
 5844		 */
 5845		insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
 5846
 5847		insn++;
 5848	}
 5849}
 5850
 5851/* Relocate data references within program code:
 5852 *  - map references;
 5853 *  - global variable references;
 5854 *  - extern references.
 5855 */
 5856static int
 5857bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
 5858{
 5859	int i;
 5860
 5861	for (i = 0; i < prog->nr_reloc; i++) {
 5862		struct reloc_desc *relo = &prog->reloc_desc[i];
 5863		struct bpf_insn *insn = &prog->insns[relo->insn_idx];
 5864		const struct bpf_map *map;
 5865		struct extern_desc *ext;
 5866
 5867		switch (relo->type) {
 5868		case RELO_LD64:
 5869			map = &obj->maps[relo->map_idx];
 5870			if (obj->gen_loader) {
 5871				insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
 5872				insn[0].imm = relo->map_idx;
 5873			} else if (map->autocreate) {
 5874				insn[0].src_reg = BPF_PSEUDO_MAP_FD;
 5875				insn[0].imm = map->fd;
 5876			} else {
 5877				poison_map_ldimm64(prog, i, relo->insn_idx, insn,
 5878						   relo->map_idx, map);
 5879			}
 5880			break;
 5881		case RELO_DATA:
 5882			map = &obj->maps[relo->map_idx];
 5883			insn[1].imm = insn[0].imm + relo->sym_off;
 5884			if (obj->gen_loader) {
 5885				insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
 5886				insn[0].imm = relo->map_idx;
 5887			} else if (map->autocreate) {
 5888				insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
 5889				insn[0].imm = map->fd;
 5890			} else {
 5891				poison_map_ldimm64(prog, i, relo->insn_idx, insn,
 5892						   relo->map_idx, map);
 5893			}
 5894			break;
 5895		case RELO_EXTERN_VAR:
 5896			ext = &obj->externs[relo->sym_off];
 5897			if (ext->type == EXT_KCFG) {
 5898				if (obj->gen_loader) {
 5899					insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
 5900					insn[0].imm = obj->kconfig_map_idx;
 5901				} else {
 5902					insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
 5903					insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
 5904				}
 5905				insn[1].imm = ext->kcfg.data_off;
 5906			} else /* EXT_KSYM */ {
 5907				if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
 5908					insn[0].src_reg = BPF_PSEUDO_BTF_ID;
 5909					insn[0].imm = ext->ksym.kernel_btf_id;
 5910					insn[1].imm = ext->ksym.kernel_btf_obj_fd;
 5911				} else { /* typeless ksyms or unresolved typed ksyms */
 5912					insn[0].imm = (__u32)ext->ksym.addr;
 5913					insn[1].imm = ext->ksym.addr >> 32;
 5914				}
 5915			}
 5916			break;
 5917		case RELO_EXTERN_FUNC:
 5918			ext = &obj->externs[relo->sym_off];
 5919			insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
 5920			if (ext->is_set) {
 5921				insn[0].imm = ext->ksym.kernel_btf_id;
 5922				insn[0].off = ext->ksym.btf_fd_idx;
 5923			} else { /* unresolved weak kfunc */
 5924				insn[0].imm = 0;
 5925				insn[0].off = 0;
 5926			}
 5927			break;
 5928		case RELO_SUBPROG_ADDR:
 5929			if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
 5930				pr_warn("prog '%s': relo #%d: bad insn\n",
 5931					prog->name, i);
 5932				return -EINVAL;
 5933			}
 5934			/* handled already */
 5935			break;
 5936		case RELO_CALL:
 5937			/* handled already */
 5938			break;
 5939		case RELO_CORE:
 5940			/* will be handled by bpf_program_record_relos() */
 5941			break;
 5942		default:
 5943			pr_warn("prog '%s': relo #%d: bad relo type %d\n",
 5944				prog->name, i, relo->type);
 5945			return -EINVAL;
 5946		}
 5947	}
 5948
 5949	return 0;
 5950}
 5951
 5952static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
 5953				    const struct bpf_program *prog,
 5954				    const struct btf_ext_info *ext_info,
 5955				    void **prog_info, __u32 *prog_rec_cnt,
 5956				    __u32 *prog_rec_sz)
 5957{
 5958	void *copy_start = NULL, *copy_end = NULL;
 5959	void *rec, *rec_end, *new_prog_info;
 5960	const struct btf_ext_info_sec *sec;
 5961	size_t old_sz, new_sz;
 5962	int i, sec_num, sec_idx, off_adj;
 5963
 5964	sec_num = 0;
 5965	for_each_btf_ext_sec(ext_info, sec) {
 5966		sec_idx = ext_info->sec_idxs[sec_num];
 5967		sec_num++;
 5968		if (prog->sec_idx != sec_idx)
 5969			continue;
 5970
 5971		for_each_btf_ext_rec(ext_info, sec, i, rec) {
 5972			__u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
 5973
 5974			if (insn_off < prog->sec_insn_off)
 5975				continue;
 5976			if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
 5977				break;
 5978
 5979			if (!copy_start)
 5980				copy_start = rec;
 5981			copy_end = rec + ext_info->rec_size;
 5982		}
 5983
 5984		if (!copy_start)
 5985			return -ENOENT;
 5986
 5987		/* append func/line info of a given (sub-)program to the main
 5988		 * program func/line info
 5989		 */
 5990		old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
 5991		new_sz = old_sz + (copy_end - copy_start);
 5992		new_prog_info = realloc(*prog_info, new_sz);
 5993		if (!new_prog_info)
 5994			return -ENOMEM;
 5995		*prog_info = new_prog_info;
 5996		*prog_rec_cnt = new_sz / ext_info->rec_size;
 5997		memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
 5998
 5999		/* Kernel instruction offsets are in units of 8-byte
 6000		 * instructions, while .BTF.ext instruction offsets generated
 6001		 * by Clang are in units of bytes. So convert Clang offsets
 6002		 * into kernel offsets and adjust offset according to program
 6003		 * relocated position.
 6004		 */
 6005		off_adj = prog->sub_insn_off - prog->sec_insn_off;
 6006		rec = new_prog_info + old_sz;
 6007		rec_end = new_prog_info + new_sz;
 6008		for (; rec < rec_end; rec += ext_info->rec_size) {
 6009			__u32 *insn_off = rec;
 6010
 6011			*insn_off = *insn_off / BPF_INSN_SZ + off_adj;
 6012		}
 6013		*prog_rec_sz = ext_info->rec_size;
 6014		return 0;
 6015	}
 6016
 6017	return -ENOENT;
 6018}
 6019
 6020static int
 6021reloc_prog_func_and_line_info(const struct bpf_object *obj,
 6022			      struct bpf_program *main_prog,
 6023			      const struct bpf_program *prog)
 6024{
 6025	int err;
 6026
 6027	/* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
 6028	 * supprot func/line info
 6029	 */
 6030	if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
 6031		return 0;
 6032
 6033	/* only attempt func info relocation if main program's func_info
 6034	 * relocation was successful
 6035	 */
 6036	if (main_prog != prog && !main_prog->func_info)
 6037		goto line_info;
 6038
 6039	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
 6040				       &main_prog->func_info,
 6041				       &main_prog->func_info_cnt,
 6042				       &main_prog->func_info_rec_size);
 6043	if (err) {
 6044		if (err != -ENOENT) {
 6045			pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
 6046				prog->name, err);
 6047			return err;
 6048		}
 6049		if (main_prog->func_info) {
 6050			/*
 6051			 * Some info has already been found but has problem
 6052			 * in the last btf_ext reloc. Must have to error out.
 6053			 */
 6054			pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
 6055			return err;
 6056		}
 6057		/* Have problem loading the very first info. Ignore the rest. */
 6058		pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
 6059			prog->name);
 6060	}
 6061
 6062line_info:
 6063	/* don't relocate line info if main program's relocation failed */
 6064	if (main_prog != prog && !main_prog->line_info)
 6065		return 0;
 6066
 6067	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
 6068				       &main_prog->line_info,
 6069				       &main_prog->line_info_cnt,
 6070				       &main_prog->line_info_rec_size);
 6071	if (err) {
 6072		if (err != -ENOENT) {
 6073			pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
 6074				prog->name, err);
 6075			return err;
 6076		}
 6077		if (main_prog->line_info) {
 6078			/*
 6079			 * Some info has already been found but has problem
 6080			 * in the last btf_ext reloc. Must have to error out.
 6081			 */
 6082			pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
 6083			return err;
 6084		}
 6085		/* Have problem loading the very first info. Ignore the rest. */
 6086		pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
 6087			prog->name);
 6088	}
 6089	return 0;
 6090}
 6091
 6092static int cmp_relo_by_insn_idx(const void *key, const void *elem)
 6093{
 6094	size_t insn_idx = *(const size_t *)key;
 6095	const struct reloc_desc *relo = elem;
 6096
 6097	if (insn_idx == relo->insn_idx)
 6098		return 0;
 6099	return insn_idx < relo->insn_idx ? -1 : 1;
 6100}
 6101
 6102static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
 6103{
 6104	if (!prog->nr_reloc)
 6105		return NULL;
 6106	return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
 6107		       sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
 6108}
 6109
 6110static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
 6111{
 6112	int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
 6113	struct reloc_desc *relos;
 6114	int i;
 6115
 6116	if (main_prog == subprog)
 6117		return 0;
 6118	relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
 6119	if (!relos)
 6120		return -ENOMEM;
 6121	if (subprog->nr_reloc)
 6122		memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
 6123		       sizeof(*relos) * subprog->nr_reloc);
 6124
 6125	for (i = main_prog->nr_reloc; i < new_cnt; i++)
 6126		relos[i].insn_idx += subprog->sub_insn_off;
 6127	/* After insn_idx adjustment the 'relos' array is still sorted
 6128	 * by insn_idx and doesn't break bsearch.
 6129	 */
 6130	main_prog->reloc_desc = relos;
 6131	main_prog->nr_reloc = new_cnt;
 6132	return 0;
 6133}
 6134
 6135static int
 6136bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
 6137		       struct bpf_program *prog)
 6138{
 6139	size_t sub_insn_idx, insn_idx, new_cnt;
 6140	struct bpf_program *subprog;
 6141	struct bpf_insn *insns, *insn;
 6142	struct reloc_desc *relo;
 6143	int err;
 6144
 6145	err = reloc_prog_func_and_line_info(obj, main_prog, prog);
 6146	if (err)
 6147		return err;
 6148
 6149	for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
 6150		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
 6151		if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
 6152			continue;
 6153
 6154		relo = find_prog_insn_relo(prog, insn_idx);
 6155		if (relo && relo->type == RELO_EXTERN_FUNC)
 6156			/* kfunc relocations will be handled later
 6157			 * in bpf_object__relocate_data()
 6158			 */
 6159			continue;
 6160		if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
 6161			pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
 6162				prog->name, insn_idx, relo->type);
 6163			return -LIBBPF_ERRNO__RELOC;
 6164		}
 6165		if (relo) {
 6166			/* sub-program instruction index is a combination of
 6167			 * an offset of a symbol pointed to by relocation and
 6168			 * call instruction's imm field; for global functions,
 6169			 * call always has imm = -1, but for static functions
 6170			 * relocation is against STT_SECTION and insn->imm
 6171			 * points to a start of a static function
 6172			 *
 6173			 * for subprog addr relocation, the relo->sym_off + insn->imm is
 6174			 * the byte offset in the corresponding section.
 6175			 */
 6176			if (relo->type == RELO_CALL)
 6177				sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
 6178			else
 6179				sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
 6180		} else if (insn_is_pseudo_func(insn)) {
 6181			/*
 6182			 * RELO_SUBPROG_ADDR relo is always emitted even if both
 6183			 * functions are in the same section, so it shouldn't reach here.
 6184			 */
 6185			pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
 6186				prog->name, insn_idx);
 6187			return -LIBBPF_ERRNO__RELOC;
 6188		} else {
 6189			/* if subprogram call is to a static function within
 6190			 * the same ELF section, there won't be any relocation
 6191			 * emitted, but it also means there is no additional
 6192			 * offset necessary, insns->imm is relative to
 6193			 * instruction's original position within the section
 6194			 */
 6195			sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
 6196		}
 6197
 6198		/* we enforce that sub-programs should be in .text section */
 6199		subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
 6200		if (!subprog) {
 6201			pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
 6202				prog->name);
 6203			return -LIBBPF_ERRNO__RELOC;
 6204		}
 6205
 6206		/* if it's the first call instruction calling into this
 6207		 * subprogram (meaning this subprog hasn't been processed
 6208		 * yet) within the context of current main program:
 6209		 *   - append it at the end of main program's instructions blog;
 6210		 *   - process is recursively, while current program is put on hold;
 6211		 *   - if that subprogram calls some other not yet processes
 6212		 *   subprogram, same thing will happen recursively until
 6213		 *   there are no more unprocesses subprograms left to append
 6214		 *   and relocate.
 6215		 */
 6216		if (subprog->sub_insn_off == 0) {
 6217			subprog->sub_insn_off = main_prog->insns_cnt;
 6218
 6219			new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
 6220			insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
 6221			if (!insns) {
 6222				pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
 6223				return -ENOMEM;
 6224			}
 6225			main_prog->insns = insns;
 6226			main_prog->insns_cnt = new_cnt;
 6227
 6228			memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
 6229			       subprog->insns_cnt * sizeof(*insns));
 6230
 6231			pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
 6232				 main_prog->name, subprog->insns_cnt, subprog->name);
 6233
 6234			/* The subprog insns are now appended. Append its relos too. */
 6235			err = append_subprog_relos(main_prog, subprog);
 6236			if (err)
 6237				return err;
 6238			err = bpf_object__reloc_code(obj, main_prog, subprog);
 6239			if (err)
 6240				return err;
 6241		}
 6242
 6243		/* main_prog->insns memory could have been re-allocated, so
 6244		 * calculate pointer again
 6245		 */
 6246		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
 6247		/* calculate correct instruction position within current main
 6248		 * prog; each main prog can have a different set of
 6249		 * subprograms appended (potentially in different order as
 6250		 * well), so position of any subprog can be different for
 6251		 * different main programs
 6252		 */
 6253		insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
 6254
 6255		pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
 6256			 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
 6257	}
 6258
 
 
 6259	return 0;
 6260}
 6261
 6262/*
 6263 * Relocate sub-program calls.
 6264 *
 6265 * Algorithm operates as follows. Each entry-point BPF program (referred to as
 6266 * main prog) is processed separately. For each subprog (non-entry functions,
 6267 * that can be called from either entry progs or other subprogs) gets their
 6268 * sub_insn_off reset to zero. This serves as indicator that this subprogram
 6269 * hasn't been yet appended and relocated within current main prog. Once its
 6270 * relocated, sub_insn_off will point at the position within current main prog
 6271 * where given subprog was appended. This will further be used to relocate all
 6272 * the call instructions jumping into this subprog.
 6273 *
 6274 * We start with main program and process all call instructions. If the call
 6275 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
 6276 * is zero), subprog instructions are appended at the end of main program's
 6277 * instruction array. Then main program is "put on hold" while we recursively
 6278 * process newly appended subprogram. If that subprogram calls into another
 6279 * subprogram that hasn't been appended, new subprogram is appended again to
 6280 * the *main* prog's instructions (subprog's instructions are always left
 6281 * untouched, as they need to be in unmodified state for subsequent main progs
 6282 * and subprog instructions are always sent only as part of a main prog) and
 6283 * the process continues recursively. Once all the subprogs called from a main
 6284 * prog or any of its subprogs are appended (and relocated), all their
 6285 * positions within finalized instructions array are known, so it's easy to
 6286 * rewrite call instructions with correct relative offsets, corresponding to
 6287 * desired target subprog.
 6288 *
 6289 * Its important to realize that some subprogs might not be called from some
 6290 * main prog and any of its called/used subprogs. Those will keep their
 6291 * subprog->sub_insn_off as zero at all times and won't be appended to current
 6292 * main prog and won't be relocated within the context of current main prog.
 6293 * They might still be used from other main progs later.
 6294 *
 6295 * Visually this process can be shown as below. Suppose we have two main
 6296 * programs mainA and mainB and BPF object contains three subprogs: subA,
 6297 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
 6298 * subC both call subB:
 6299 *
 6300 *        +--------+ +-------+
 6301 *        |        v v       |
 6302 *     +--+---+ +--+-+-+ +---+--+
 6303 *     | subA | | subB | | subC |
 6304 *     +--+---+ +------+ +---+--+
 6305 *        ^                  ^
 6306 *        |                  |
 6307 *    +---+-------+   +------+----+
 6308 *    |   mainA   |   |   mainB   |
 6309 *    +-----------+   +-----------+
 6310 *
 6311 * We'll start relocating mainA, will find subA, append it and start
 6312 * processing sub A recursively:
 6313 *
 6314 *    +-----------+------+
 6315 *    |   mainA   | subA |
 6316 *    +-----------+------+
 6317 *
 6318 * At this point we notice that subB is used from subA, so we append it and
 6319 * relocate (there are no further subcalls from subB):
 6320 *
 6321 *    +-----------+------+------+
 6322 *    |   mainA   | subA | subB |
 6323 *    +-----------+------+------+
 6324 *
 6325 * At this point, we relocate subA calls, then go one level up and finish with
 6326 * relocatin mainA calls. mainA is done.
 6327 *
 6328 * For mainB process is similar but results in different order. We start with
 6329 * mainB and skip subA and subB, as mainB never calls them (at least
 6330 * directly), but we see subC is needed, so we append and start processing it:
 6331 *
 6332 *    +-----------+------+
 6333 *    |   mainB   | subC |
 6334 *    +-----------+------+
 6335 * Now we see subC needs subB, so we go back to it, append and relocate it:
 6336 *
 6337 *    +-----------+------+------+
 6338 *    |   mainB   | subC | subB |
 6339 *    +-----------+------+------+
 6340 *
 6341 * At this point we unwind recursion, relocate calls in subC, then in mainB.
 6342 */
 6343static int
 6344bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
 6345{
 6346	struct bpf_program *subprog;
 6347	int i, err;
 6348
 6349	/* mark all subprogs as not relocated (yet) within the context of
 6350	 * current main program
 6351	 */
 6352	for (i = 0; i < obj->nr_programs; i++) {
 6353		subprog = &obj->programs[i];
 6354		if (!prog_is_subprog(obj, subprog))
 6355			continue;
 6356
 6357		subprog->sub_insn_off = 0;
 6358	}
 6359
 6360	err = bpf_object__reloc_code(obj, prog, prog);
 6361	if (err)
 6362		return err;
 6363
 6364	return 0;
 6365}
 6366
 6367static void
 6368bpf_object__free_relocs(struct bpf_object *obj)
 6369{
 6370	struct bpf_program *prog;
 6371	int i;
 6372
 6373	/* free up relocation descriptors */
 6374	for (i = 0; i < obj->nr_programs; i++) {
 6375		prog = &obj->programs[i];
 6376		zfree(&prog->reloc_desc);
 6377		prog->nr_reloc = 0;
 6378	}
 6379}
 6380
 6381static int cmp_relocs(const void *_a, const void *_b)
 6382{
 6383	const struct reloc_desc *a = _a;
 6384	const struct reloc_desc *b = _b;
 6385
 6386	if (a->insn_idx != b->insn_idx)
 6387		return a->insn_idx < b->insn_idx ? -1 : 1;
 6388
 6389	/* no two relocations should have the same insn_idx, but ... */
 6390	if (a->type != b->type)
 6391		return a->type < b->type ? -1 : 1;
 6392
 6393	return 0;
 6394}
 6395
 6396static void bpf_object__sort_relos(struct bpf_object *obj)
 6397{
 6398	int i;
 6399
 6400	for (i = 0; i < obj->nr_programs; i++) {
 6401		struct bpf_program *p = &obj->programs[i];
 6402
 6403		if (!p->nr_reloc)
 6404			continue;
 6405
 6406		qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
 6407	}
 6408}
 6409
 6410static int
 6411bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
 6412{
 6413	struct bpf_program *prog;
 6414	size_t i, j;
 6415	int err;
 6416
 6417	if (obj->btf_ext) {
 6418		err = bpf_object__relocate_core(obj, targ_btf_path);
 6419		if (err) {
 6420			pr_warn("failed to perform CO-RE relocations: %d\n",
 6421				err);
 6422			return err;
 6423		}
 6424		bpf_object__sort_relos(obj);
 6425	}
 6426
 6427	/* Before relocating calls pre-process relocations and mark
 6428	 * few ld_imm64 instructions that points to subprogs.
 6429	 * Otherwise bpf_object__reloc_code() later would have to consider
 6430	 * all ld_imm64 insns as relocation candidates. That would
 6431	 * reduce relocation speed, since amount of find_prog_insn_relo()
 6432	 * would increase and most of them will fail to find a relo.
 6433	 */
 6434	for (i = 0; i < obj->nr_programs; i++) {
 6435		prog = &obj->programs[i];
 6436		for (j = 0; j < prog->nr_reloc; j++) {
 6437			struct reloc_desc *relo = &prog->reloc_desc[j];
 6438			struct bpf_insn *insn = &prog->insns[relo->insn_idx];
 6439
 6440			/* mark the insn, so it's recognized by insn_is_pseudo_func() */
 6441			if (relo->type == RELO_SUBPROG_ADDR)
 6442				insn[0].src_reg = BPF_PSEUDO_FUNC;
 6443		}
 6444	}
 6445
 6446	/* relocate subprogram calls and append used subprograms to main
 6447	 * programs; each copy of subprogram code needs to be relocated
 6448	 * differently for each main program, because its code location might
 6449	 * have changed.
 6450	 * Append subprog relos to main programs to allow data relos to be
 6451	 * processed after text is completely relocated.
 6452	 */
 6453	for (i = 0; i < obj->nr_programs; i++) {
 6454		prog = &obj->programs[i];
 6455		/* sub-program's sub-calls are relocated within the context of
 6456		 * its main program only
 6457		 */
 6458		if (prog_is_subprog(obj, prog))
 6459			continue;
 6460		if (!prog->autoload)
 6461			continue;
 6462
 6463		err = bpf_object__relocate_calls(obj, prog);
 6464		if (err) {
 6465			pr_warn("prog '%s': failed to relocate calls: %d\n",
 6466				prog->name, err);
 6467			return err;
 6468		}
 6469	}
 6470	/* Process data relos for main programs */
 6471	for (i = 0; i < obj->nr_programs; i++) {
 6472		prog = &obj->programs[i];
 6473		if (prog_is_subprog(obj, prog))
 6474			continue;
 6475		if (!prog->autoload)
 6476			continue;
 6477		err = bpf_object__relocate_data(obj, prog);
 6478		if (err) {
 6479			pr_warn("prog '%s': failed to relocate data references: %d\n",
 6480				prog->name, err);
 6481			return err;
 6482		}
 6483	}
 6484
 6485	return 0;
 6486}
 6487
 6488static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
 6489					    Elf64_Shdr *shdr, Elf_Data *data);
 6490
 6491static int bpf_object__collect_map_relos(struct bpf_object *obj,
 6492					 Elf64_Shdr *shdr, Elf_Data *data)
 6493{
 6494	const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
 6495	int i, j, nrels, new_sz;
 6496	const struct btf_var_secinfo *vi = NULL;
 6497	const struct btf_type *sec, *var, *def;
 6498	struct bpf_map *map = NULL, *targ_map = NULL;
 6499	struct bpf_program *targ_prog = NULL;
 6500	bool is_prog_array, is_map_in_map;
 6501	const struct btf_member *member;
 6502	const char *name, *mname, *type;
 6503	unsigned int moff;
 6504	Elf64_Sym *sym;
 6505	Elf64_Rel *rel;
 6506	void *tmp;
 6507
 6508	if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
 6509		return -EINVAL;
 6510	sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
 6511	if (!sec)
 6512		return -EINVAL;
 6513
 6514	nrels = shdr->sh_size / shdr->sh_entsize;
 6515	for (i = 0; i < nrels; i++) {
 6516		rel = elf_rel_by_idx(data, i);
 6517		if (!rel) {
 6518			pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
 6519			return -LIBBPF_ERRNO__FORMAT;
 6520		}
 6521
 6522		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
 6523		if (!sym) {
 6524			pr_warn(".maps relo #%d: symbol %zx not found\n",
 6525				i, (size_t)ELF64_R_SYM(rel->r_info));
 6526			return -LIBBPF_ERRNO__FORMAT;
 6527		}
 6528		name = elf_sym_str(obj, sym->st_name) ?: "<?>";
 6529
 6530		pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
 6531			 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
 6532			 (size_t)rel->r_offset, sym->st_name, name);
 6533
 6534		for (j = 0; j < obj->nr_maps; j++) {
 6535			map = &obj->maps[j];
 6536			if (map->sec_idx != obj->efile.btf_maps_shndx)
 6537				continue;
 6538
 6539			vi = btf_var_secinfos(sec) + map->btf_var_idx;
 6540			if (vi->offset <= rel->r_offset &&
 6541			    rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
 6542				break;
 6543		}
 6544		if (j == obj->nr_maps) {
 6545			pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
 6546				i, name, (size_t)rel->r_offset);
 6547			return -EINVAL;
 6548		}
 6549
 6550		is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
 6551		is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
 6552		type = is_map_in_map ? "map" : "prog";
 6553		if (is_map_in_map) {
 6554			if (sym->st_shndx != obj->efile.btf_maps_shndx) {
 6555				pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
 6556					i, name);
 6557				return -LIBBPF_ERRNO__RELOC;
 6558			}
 6559			if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
 6560			    map->def.key_size != sizeof(int)) {
 6561				pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
 6562					i, map->name, sizeof(int));
 6563				return -EINVAL;
 6564			}
 6565			targ_map = bpf_object__find_map_by_name(obj, name);
 6566			if (!targ_map) {
 6567				pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
 6568					i, name);
 6569				return -ESRCH;
 6570			}
 6571		} else if (is_prog_array) {
 6572			targ_prog = bpf_object__find_program_by_name(obj, name);
 6573			if (!targ_prog) {
 6574				pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
 6575					i, name);
 6576				return -ESRCH;
 6577			}
 6578			if (targ_prog->sec_idx != sym->st_shndx ||
 6579			    targ_prog->sec_insn_off * 8 != sym->st_value ||
 6580			    prog_is_subprog(obj, targ_prog)) {
 6581				pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
 6582					i, name);
 6583				return -LIBBPF_ERRNO__RELOC;
 6584			}
 6585		} else {
 6586			return -EINVAL;
 6587		}
 6588
 6589		var = btf__type_by_id(obj->btf, vi->type);
 6590		def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
 6591		if (btf_vlen(def) == 0)
 6592			return -EINVAL;
 6593		member = btf_members(def) + btf_vlen(def) - 1;
 6594		mname = btf__name_by_offset(obj->btf, member->name_off);
 6595		if (strcmp(mname, "values"))
 6596			return -EINVAL;
 6597
 6598		moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
 6599		if (rel->r_offset - vi->offset < moff)
 6600			return -EINVAL;
 6601
 6602		moff = rel->r_offset - vi->offset - moff;
 6603		/* here we use BPF pointer size, which is always 64 bit, as we
 6604		 * are parsing ELF that was built for BPF target
 6605		 */
 6606		if (moff % bpf_ptr_sz)
 6607			return -EINVAL;
 6608		moff /= bpf_ptr_sz;
 6609		if (moff >= map->init_slots_sz) {
 6610			new_sz = moff + 1;
 6611			tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
 6612			if (!tmp)
 6613				return -ENOMEM;
 6614			map->init_slots = tmp;
 6615			memset(map->init_slots + map->init_slots_sz, 0,
 6616			       (new_sz - map->init_slots_sz) * host_ptr_sz);
 6617			map->init_slots_sz = new_sz;
 6618		}
 6619		map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
 6620
 6621		pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
 6622			 i, map->name, moff, type, name);
 6623	}
 6624
 6625	return 0;
 6626}
 6627
 6628static int bpf_object__collect_relos(struct bpf_object *obj)
 6629{
 6630	int i, err;
 6631
 6632	for (i = 0; i < obj->efile.sec_cnt; i++) {
 6633		struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
 6634		Elf64_Shdr *shdr;
 6635		Elf_Data *data;
 6636		int idx;
 6637
 6638		if (sec_desc->sec_type != SEC_RELO)
 6639			continue;
 6640
 6641		shdr = sec_desc->shdr;
 6642		data = sec_desc->data;
 6643		idx = shdr->sh_info;
 
 
 
 6644
 6645		if (shdr->sh_type != SHT_REL) {
 6646			pr_warn("internal error at %d\n", __LINE__);
 6647			return -LIBBPF_ERRNO__INTERNAL;
 6648		}
 6649
 6650		if (idx == obj->efile.st_ops_shndx)
 6651			err = bpf_object__collect_st_ops_relos(obj, shdr, data);
 6652		else if (idx == obj->efile.btf_maps_shndx)
 6653			err = bpf_object__collect_map_relos(obj, shdr, data);
 6654		else
 6655			err = bpf_object__collect_prog_relos(obj, shdr, data);
 6656		if (err)
 6657			return err;
 6658	}
 6659
 6660	bpf_object__sort_relos(obj);
 6661	return 0;
 6662}
 6663
 6664static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
 6665{
 6666	if (BPF_CLASS(insn->code) == BPF_JMP &&
 6667	    BPF_OP(insn->code) == BPF_CALL &&
 6668	    BPF_SRC(insn->code) == BPF_K &&
 6669	    insn->src_reg == 0 &&
 6670	    insn->dst_reg == 0) {
 6671		    *func_id = insn->imm;
 6672		    return true;
 6673	}
 6674	return false;
 6675}
 6676
 6677static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
 6678{
 6679	struct bpf_insn *insn = prog->insns;
 6680	enum bpf_func_id func_id;
 6681	int i;
 6682
 6683	if (obj->gen_loader)
 6684		return 0;
 6685
 6686	for (i = 0; i < prog->insns_cnt; i++, insn++) {
 6687		if (!insn_is_helper_call(insn, &func_id))
 6688			continue;
 6689
 6690		/* on kernels that don't yet support
 6691		 * bpf_probe_read_{kernel,user}[_str] helpers, fall back
 6692		 * to bpf_probe_read() which works well for old kernels
 6693		 */
 6694		switch (func_id) {
 6695		case BPF_FUNC_probe_read_kernel:
 6696		case BPF_FUNC_probe_read_user:
 6697			if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
 6698				insn->imm = BPF_FUNC_probe_read;
 6699			break;
 6700		case BPF_FUNC_probe_read_kernel_str:
 6701		case BPF_FUNC_probe_read_user_str:
 6702			if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
 6703				insn->imm = BPF_FUNC_probe_read_str;
 6704			break;
 6705		default:
 6706			break;
 6707		}
 6708	}
 6709	return 0;
 6710}
 6711
 6712static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
 6713				     int *btf_obj_fd, int *btf_type_id);
 6714
 6715/* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
 6716static int libbpf_prepare_prog_load(struct bpf_program *prog,
 6717				    struct bpf_prog_load_opts *opts, long cookie)
 6718{
 6719	enum sec_def_flags def = cookie;
 6720
 6721	/* old kernels might not support specifying expected_attach_type */
 6722	if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
 6723		opts->expected_attach_type = 0;
 6724
 6725	if (def & SEC_SLEEPABLE)
 6726		opts->prog_flags |= BPF_F_SLEEPABLE;
 6727
 6728	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
 6729		opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
 6730
 6731	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
 6732		int btf_obj_fd = 0, btf_type_id = 0, err;
 6733		const char *attach_name;
 6734
 6735		attach_name = strchr(prog->sec_name, '/');
 6736		if (!attach_name) {
 6737			/* if BPF program is annotated with just SEC("fentry")
 6738			 * (or similar) without declaratively specifying
 6739			 * target, then it is expected that target will be
 6740			 * specified with bpf_program__set_attach_target() at
 6741			 * runtime before BPF object load step. If not, then
 6742			 * there is nothing to load into the kernel as BPF
 6743			 * verifier won't be able to validate BPF program
 6744			 * correctness anyways.
 6745			 */
 6746			pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
 6747				prog->name);
 6748			return -EINVAL;
 6749		}
 6750		attach_name++; /* skip over / */
 6751
 6752		err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
 
 
 
 6753		if (err)
 6754			return err;
 6755
 6756		/* cache resolved BTF FD and BTF type ID in the prog */
 6757		prog->attach_btf_obj_fd = btf_obj_fd;
 6758		prog->attach_btf_id = btf_type_id;
 6759
 6760		/* but by now libbpf common logic is not utilizing
 6761		 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
 6762		 * this callback is called after opts were populated by
 6763		 * libbpf, so this callback has to update opts explicitly here
 6764		 */
 6765		opts->attach_btf_obj_fd = btf_obj_fd;
 6766		opts->attach_btf_id = btf_type_id;
 6767	}
 6768	return 0;
 6769}
 6770
 6771static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
 6772
 6773static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
 6774				struct bpf_insn *insns, int insns_cnt,
 6775				const char *license, __u32 kern_version, int *prog_fd)
 6776{
 6777	LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
 6778	const char *prog_name = NULL;
 6779	char *cp, errmsg[STRERR_BUFSIZE];
 6780	size_t log_buf_size = 0;
 6781	char *log_buf = NULL, *tmp;
 6782	int btf_fd, ret, err;
 6783	bool own_log_buf = true;
 6784	__u32 log_level = prog->log_level;
 6785
 6786	if (prog->type == BPF_PROG_TYPE_UNSPEC) {
 6787		/*
 6788		 * The program type must be set.  Most likely we couldn't find a proper
 6789		 * section definition at load time, and thus we didn't infer the type.
 6790		 */
 6791		pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
 6792			prog->name, prog->sec_name);
 6793		return -EINVAL;
 6794	}
 6795
 6796	if (!insns || !insns_cnt)
 6797		return -EINVAL;
 6798
 6799	load_attr.expected_attach_type = prog->expected_attach_type;
 6800	if (kernel_supports(obj, FEAT_PROG_NAME))
 6801		prog_name = prog->name;
 6802	load_attr.attach_prog_fd = prog->attach_prog_fd;
 6803	load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
 6804	load_attr.attach_btf_id = prog->attach_btf_id;
 6805	load_attr.kern_version = kern_version;
 6806	load_attr.prog_ifindex = prog->prog_ifindex;
 6807
 6808	/* specify func_info/line_info only if kernel supports them */
 6809	btf_fd = bpf_object__btf_fd(obj);
 6810	if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
 6811		load_attr.prog_btf_fd = btf_fd;
 6812		load_attr.func_info = prog->func_info;
 6813		load_attr.func_info_rec_size = prog->func_info_rec_size;
 6814		load_attr.func_info_cnt = prog->func_info_cnt;
 6815		load_attr.line_info = prog->line_info;
 6816		load_attr.line_info_rec_size = prog->line_info_rec_size;
 6817		load_attr.line_info_cnt = prog->line_info_cnt;
 6818	}
 6819	load_attr.log_level = log_level;
 6820	load_attr.prog_flags = prog->prog_flags;
 6821	load_attr.fd_array = obj->fd_array;
 6822
 6823	/* adjust load_attr if sec_def provides custom preload callback */
 6824	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
 6825		err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
 6826		if (err < 0) {
 6827			pr_warn("prog '%s': failed to prepare load attributes: %d\n",
 6828				prog->name, err);
 6829			return err;
 6830		}
 6831		insns = prog->insns;
 6832		insns_cnt = prog->insns_cnt;
 6833	}
 6834
 6835	if (obj->gen_loader) {
 6836		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
 6837				   license, insns, insns_cnt, &load_attr,
 6838				   prog - obj->programs);
 6839		*prog_fd = -1;
 6840		return 0;
 6841	}
 6842
 6843retry_load:
 6844	/* if log_level is zero, we don't request logs initially even if
 6845	 * custom log_buf is specified; if the program load fails, then we'll
 6846	 * bump log_level to 1 and use either custom log_buf or we'll allocate
 6847	 * our own and retry the load to get details on what failed
 6848	 */
 6849	if (log_level) {
 6850		if (prog->log_buf) {
 6851			log_buf = prog->log_buf;
 6852			log_buf_size = prog->log_size;
 6853			own_log_buf = false;
 6854		} else if (obj->log_buf) {
 6855			log_buf = obj->log_buf;
 6856			log_buf_size = obj->log_size;
 6857			own_log_buf = false;
 6858		} else {
 6859			log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
 6860			tmp = realloc(log_buf, log_buf_size);
 6861			if (!tmp) {
 6862				ret = -ENOMEM;
 6863				goto out;
 6864			}
 6865			log_buf = tmp;
 6866			log_buf[0] = '\0';
 6867			own_log_buf = true;
 6868		}
 6869	}
 6870
 6871	load_attr.log_buf = log_buf;
 6872	load_attr.log_size = log_buf_size;
 6873	load_attr.log_level = log_level;
 6874
 6875	ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
 6876	if (ret >= 0) {
 6877		if (log_level && own_log_buf) {
 6878			pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
 6879				 prog->name, log_buf);
 6880		}
 6881
 6882		if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
 6883			struct bpf_map *map;
 6884			int i;
 6885
 6886			for (i = 0; i < obj->nr_maps; i++) {
 6887				map = &prog->obj->maps[i];
 6888				if (map->libbpf_type != LIBBPF_MAP_RODATA)
 6889					continue;
 6890
 6891				if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) {
 6892					cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
 6893					pr_warn("prog '%s': failed to bind map '%s': %s\n",
 6894						prog->name, map->real_name, cp);
 6895					/* Don't fail hard if can't bind rodata. */
 6896				}
 6897			}
 6898		}
 6899
 6900		*prog_fd = ret;
 6901		ret = 0;
 6902		goto out;
 6903	}
 6904
 6905	if (log_level == 0) {
 6906		log_level = 1;
 6907		goto retry_load;
 6908	}
 6909	/* On ENOSPC, increase log buffer size and retry, unless custom
 6910	 * log_buf is specified.
 6911	 * Be careful to not overflow u32, though. Kernel's log buf size limit
 6912	 * isn't part of UAPI so it can always be bumped to full 4GB. So don't
 6913	 * multiply by 2 unless we are sure we'll fit within 32 bits.
 6914	 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
 6915	 */
 6916	if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
 6917		goto retry_load;
 6918
 6919	ret = -errno;
 6920
 6921	/* post-process verifier log to improve error descriptions */
 6922	fixup_verifier_log(prog, log_buf, log_buf_size);
 6923
 6924	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
 6925	pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
 6926	pr_perm_msg(ret);
 6927
 6928	if (own_log_buf && log_buf && log_buf[0] != '\0') {
 6929		pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
 6930			prog->name, log_buf);
 
 
 
 
 6931	}
 6932
 6933out:
 6934	if (own_log_buf)
 6935		free(log_buf);
 6936	return ret;
 6937}
 6938
 6939static char *find_prev_line(char *buf, char *cur)
 
 
 6940{
 6941	char *p;
 6942
 6943	if (cur == buf) /* end of a log buf */
 6944		return NULL;
 
 
 
 
 6945
 6946	p = cur - 1;
 6947	while (p - 1 >= buf && *(p - 1) != '\n')
 6948		p--;
 6949
 6950	return p;
 6951}
 6952
 6953static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
 6954		      char *orig, size_t orig_sz, const char *patch)
 6955{
 6956	/* size of the remaining log content to the right from the to-be-replaced part */
 6957	size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
 6958	size_t patch_sz = strlen(patch);
 6959
 6960	if (patch_sz != orig_sz) {
 6961		/* If patch line(s) are longer than original piece of verifier log,
 6962		 * shift log contents by (patch_sz - orig_sz) bytes to the right
 6963		 * starting from after to-be-replaced part of the log.
 6964		 *
 6965		 * If patch line(s) are shorter than original piece of verifier log,
 6966		 * shift log contents by (orig_sz - patch_sz) bytes to the left
 6967		 * starting from after to-be-replaced part of the log
 6968		 *
 6969		 * We need to be careful about not overflowing available
 6970		 * buf_sz capacity. If that's the case, we'll truncate the end
 6971		 * of the original log, as necessary.
 6972		 */
 6973		if (patch_sz > orig_sz) {
 6974			if (orig + patch_sz >= buf + buf_sz) {
 6975				/* patch is big enough to cover remaining space completely */
 6976				patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
 6977				rem_sz = 0;
 6978			} else if (patch_sz - orig_sz > buf_sz - log_sz) {
 6979				/* patch causes part of remaining log to be truncated */
 6980				rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
 6981			}
 6982		}
 6983		/* shift remaining log to the right by calculated amount */
 6984		memmove(orig + patch_sz, orig + orig_sz, rem_sz);
 6985	}
 6986
 6987	memcpy(orig, patch, patch_sz);
 6988}
 6989
 6990static void fixup_log_failed_core_relo(struct bpf_program *prog,
 6991				       char *buf, size_t buf_sz, size_t log_sz,
 6992				       char *line1, char *line2, char *line3)
 6993{
 6994	/* Expected log for failed and not properly guarded CO-RE relocation:
 6995	 * line1 -> 123: (85) call unknown#195896080
 6996	 * line2 -> invalid func unknown#195896080
 6997	 * line3 -> <anything else or end of buffer>
 6998	 *
 6999	 * "123" is the index of the instruction that was poisoned. We extract
 7000	 * instruction index to find corresponding CO-RE relocation and
 7001	 * replace this part of the log with more relevant information about
 7002	 * failed CO-RE relocation.
 7003	 */
 7004	const struct bpf_core_relo *relo;
 7005	struct bpf_core_spec spec;
 7006	char patch[512], spec_buf[256];
 7007	int insn_idx, err, spec_len;
 7008
 7009	if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
 7010		return;
 7011
 7012	relo = find_relo_core(prog, insn_idx);
 7013	if (!relo)
 7014		return;
 7015
 7016	err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
 7017	if (err)
 7018		return;
 7019
 7020	spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
 7021	snprintf(patch, sizeof(patch),
 7022		 "%d: <invalid CO-RE relocation>\n"
 7023		 "failed to resolve CO-RE relocation %s%s\n",
 7024		 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
 7025
 7026	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
 7027}
 7028
 7029static void fixup_log_missing_map_load(struct bpf_program *prog,
 7030				       char *buf, size_t buf_sz, size_t log_sz,
 7031				       char *line1, char *line2, char *line3)
 7032{
 7033	/* Expected log for failed and not properly guarded CO-RE relocation:
 7034	 * line1 -> 123: (85) call unknown#2001000345
 7035	 * line2 -> invalid func unknown#2001000345
 7036	 * line3 -> <anything else or end of buffer>
 7037	 *
 7038	 * "123" is the index of the instruction that was poisoned.
 7039	 * "345" in "2001000345" are map index in obj->maps to fetch map name.
 7040	 */
 7041	struct bpf_object *obj = prog->obj;
 7042	const struct bpf_map *map;
 7043	int insn_idx, map_idx;
 7044	char patch[128];
 7045
 7046	if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
 7047		return;
 7048
 7049	map_idx -= MAP_LDIMM64_POISON_BASE;
 7050	if (map_idx < 0 || map_idx >= obj->nr_maps)
 7051		return;
 7052	map = &obj->maps[map_idx];
 7053
 7054	snprintf(patch, sizeof(patch),
 7055		 "%d: <invalid BPF map reference>\n"
 7056		 "BPF map '%s' is referenced but wasn't created\n",
 7057		 insn_idx, map->name);
 7058
 7059	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
 7060}
 7061
 7062static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
 7063{
 7064	/* look for familiar error patterns in last N lines of the log */
 7065	const size_t max_last_line_cnt = 10;
 7066	char *prev_line, *cur_line, *next_line;
 7067	size_t log_sz;
 7068	int i;
 7069
 7070	if (!buf)
 7071		return;
 7072
 7073	log_sz = strlen(buf) + 1;
 7074	next_line = buf + log_sz - 1;
 7075
 7076	for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
 7077		cur_line = find_prev_line(buf, next_line);
 7078		if (!cur_line)
 7079			return;
 7080
 7081		/* failed CO-RE relocation case */
 7082		if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
 7083			prev_line = find_prev_line(buf, cur_line);
 7084			if (!prev_line)
 7085				continue;
 7086
 7087			fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
 7088						   prev_line, cur_line, next_line);
 7089			return;
 7090		} else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
 7091			prev_line = find_prev_line(buf, cur_line);
 7092			if (!prev_line)
 7093				continue;
 7094
 7095			fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
 7096						   prev_line, cur_line, next_line);
 7097			return;
 7098		}
 
 
 
 
 
 7099	}
 7100}
 7101
 7102static int bpf_program_record_relos(struct bpf_program *prog)
 7103{
 7104	struct bpf_object *obj = prog->obj;
 7105	int i;
 
 
 
 
 
 
 
 
 7106
 7107	for (i = 0; i < prog->nr_reloc; i++) {
 7108		struct reloc_desc *relo = &prog->reloc_desc[i];
 7109		struct extern_desc *ext = &obj->externs[relo->sym_off];
 
 
 
 
 
 7110
 7111		switch (relo->type) {
 7112		case RELO_EXTERN_VAR:
 7113			if (ext->type != EXT_KSYM)
 7114				continue;
 7115			bpf_gen__record_extern(obj->gen_loader, ext->name,
 7116					       ext->is_weak, !ext->ksym.type_id,
 7117					       BTF_KIND_VAR, relo->insn_idx);
 7118			break;
 7119		case RELO_EXTERN_FUNC:
 7120			bpf_gen__record_extern(obj->gen_loader, ext->name,
 7121					       ext->is_weak, false, BTF_KIND_FUNC,
 7122					       relo->insn_idx);
 7123			break;
 7124		case RELO_CORE: {
 7125			struct bpf_core_relo cr = {
 7126				.insn_off = relo->insn_idx * 8,
 7127				.type_id = relo->core_relo->type_id,
 7128				.access_str_off = relo->core_relo->access_str_off,
 7129				.kind = relo->core_relo->kind,
 7130			};
 7131
 7132			bpf_gen__record_relo_core(obj->gen_loader, &cr);
 7133			break;
 7134		}
 7135		default:
 7136			continue;
 7137		}
 
 
 
 
 7138	}
 7139	return 0;
 
 
 
 
 
 
 7140}
 7141
 7142static int
 7143bpf_object__load_progs(struct bpf_object *obj, int log_level)
 7144{
 7145	struct bpf_program *prog;
 7146	size_t i;
 7147	int err;
 7148
 7149	for (i = 0; i < obj->nr_programs; i++) {
 7150		prog = &obj->programs[i];
 7151		err = bpf_object__sanitize_prog(obj, prog);
 
 7152		if (err)
 7153			return err;
 7154	}
 7155
 7156	for (i = 0; i < obj->nr_programs; i++) {
 7157		prog = &obj->programs[i];
 7158		if (prog_is_subprog(obj, prog))
 7159			continue;
 7160		if (!prog->autoload) {
 7161			pr_debug("prog '%s': skipped loading\n", prog->name);
 7162			continue;
 7163		}
 7164		prog->log_level |= log_level;
 7165
 7166		if (obj->gen_loader)
 7167			bpf_program_record_relos(prog);
 7168
 7169		err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
 7170					   obj->license, obj->kern_version, &prog->fd);
 7171		if (err) {
 7172			pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
 7173			return err;
 7174		}
 7175	}
 7176
 7177	bpf_object__free_relocs(obj);
 7178	return 0;
 7179}
 7180
 7181static const struct bpf_sec_def *find_sec_def(const char *sec_name);
 7182
 7183static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
 7184{
 7185	struct bpf_program *prog;
 7186	int err;
 7187
 7188	bpf_object__for_each_program(prog, obj) {
 7189		prog->sec_def = find_sec_def(prog->sec_name);
 7190		if (!prog->sec_def) {
 7191			/* couldn't guess, but user might manually specify */
 7192			pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
 7193				prog->name, prog->sec_name);
 7194			continue;
 7195		}
 7196
 7197		prog->type = prog->sec_def->prog_type;
 7198		prog->expected_attach_type = prog->sec_def->expected_attach_type;
 7199
 7200		/* sec_def can have custom callback which should be called
 7201		 * after bpf_program is initialized to adjust its properties
 7202		 */
 7203		if (prog->sec_def->prog_setup_fn) {
 7204			err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
 7205			if (err < 0) {
 7206				pr_warn("prog '%s': failed to initialize: %d\n",
 7207					prog->name, err);
 7208				return err;
 7209			}
 7210		}
 7211	}
 7212
 7213	return 0;
 7214}
 7215
 7216static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
 7217					  const struct bpf_object_open_opts *opts)
 7218{
 7219	const char *obj_name, *kconfig, *btf_tmp_path;
 7220	struct bpf_object *obj;
 7221	char tmp_name[64];
 7222	int err;
 7223	char *log_buf;
 7224	size_t log_size;
 7225	__u32 log_level;
 7226
 7227	if (elf_version(EV_CURRENT) == EV_NONE) {
 7228		pr_warn("failed to init libelf for %s\n",
 7229			path ? : "(mem buf)");
 7230		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
 7231	}
 7232
 7233	if (!OPTS_VALID(opts, bpf_object_open_opts))
 7234		return ERR_PTR(-EINVAL);
 7235
 7236	obj_name = OPTS_GET(opts, object_name, NULL);
 7237	if (obj_buf) {
 7238		if (!obj_name) {
 7239			snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
 7240				 (unsigned long)obj_buf,
 7241				 (unsigned long)obj_buf_sz);
 7242			obj_name = tmp_name;
 7243		}
 7244		path = obj_name;
 7245		pr_debug("loading object '%s' from buffer\n", obj_name);
 7246	}
 7247
 7248	log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
 7249	log_size = OPTS_GET(opts, kernel_log_size, 0);
 7250	log_level = OPTS_GET(opts, kernel_log_level, 0);
 7251	if (log_size > UINT_MAX)
 7252		return ERR_PTR(-EINVAL);
 7253	if (log_size && !log_buf)
 7254		return ERR_PTR(-EINVAL);
 7255
 7256	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
 7257	if (IS_ERR(obj))
 7258		return obj;
 7259
 7260	obj->log_buf = log_buf;
 7261	obj->log_size = log_size;
 7262	obj->log_level = log_level;
 7263
 7264	btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
 7265	if (btf_tmp_path) {
 7266		if (strlen(btf_tmp_path) >= PATH_MAX) {
 7267			err = -ENAMETOOLONG;
 7268			goto out;
 7269		}
 7270		obj->btf_custom_path = strdup(btf_tmp_path);
 7271		if (!obj->btf_custom_path) {
 7272			err = -ENOMEM;
 7273			goto out;
 7274		}
 7275	}
 7276
 7277	kconfig = OPTS_GET(opts, kconfig, NULL);
 7278	if (kconfig) {
 7279		obj->kconfig = strdup(kconfig);
 7280		if (!obj->kconfig) {
 7281			err = -ENOMEM;
 7282			goto out;
 7283		}
 7284	}
 7285
 7286	err = bpf_object__elf_init(obj);
 7287	err = err ? : bpf_object__check_endianness(obj);
 7288	err = err ? : bpf_object__elf_collect(obj);
 7289	err = err ? : bpf_object__collect_externs(obj);
 7290	err = err ? : bpf_object_fixup_btf(obj);
 7291	err = err ? : bpf_object__init_maps(obj, opts);
 7292	err = err ? : bpf_object_init_progs(obj, opts);
 7293	err = err ? : bpf_object__collect_relos(obj);
 7294	if (err)
 7295		goto out;
 7296
 7297	bpf_object__elf_finish(obj);
 7298
 7299	return obj;
 7300out:
 7301	bpf_object__close(obj);
 7302	return ERR_PTR(err);
 7303}
 7304
 7305struct bpf_object *
 7306bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
 7307{
 
 7308	if (!path)
 7309		return libbpf_err_ptr(-EINVAL);
 7310
 7311	pr_debug("loading %s\n", path);
 7312
 7313	return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
 7314}
 7315
 7316struct bpf_object *bpf_object__open(const char *path)
 
 
 7317{
 7318	return bpf_object__open_file(path, NULL);
 7319}
 7320
 7321struct bpf_object *
 7322bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
 7323		     const struct bpf_object_open_opts *opts)
 7324{
 7325	if (!obj_buf || obj_buf_sz == 0)
 7326		return libbpf_err_ptr(-EINVAL);
 
 
 
 
 
 
 
 7327
 7328	return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
 7329}
 7330
 7331static int bpf_object_unload(struct bpf_object *obj)
 7332{
 7333	size_t i;
 7334
 7335	if (!obj)
 7336		return libbpf_err(-EINVAL);
 7337
 7338	for (i = 0; i < obj->nr_maps; i++) {
 7339		zclose(obj->maps[i].fd);
 7340		if (obj->maps[i].st_ops)
 7341			zfree(&obj->maps[i].st_ops->kern_vdata);
 7342	}
 7343
 7344	for (i = 0; i < obj->nr_programs; i++)
 7345		bpf_program__unload(&obj->programs[i]);
 7346
 7347	return 0;
 7348}
 7349
 7350static int bpf_object__sanitize_maps(struct bpf_object *obj)
 7351{
 7352	struct bpf_map *m;
 7353
 7354	bpf_object__for_each_map(m, obj) {
 7355		if (!bpf_map__is_internal(m))
 7356			continue;
 7357		if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
 7358			m->def.map_flags ^= BPF_F_MMAPABLE;
 7359	}
 7360
 7361	return 0;
 7362}
 7363
 7364int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
 7365{
 7366	char sym_type, sym_name[500];
 7367	unsigned long long sym_addr;
 7368	int ret, err = 0;
 7369	FILE *f;
 7370
 7371	f = fopen("/proc/kallsyms", "r");
 7372	if (!f) {
 7373		err = -errno;
 7374		pr_warn("failed to open /proc/kallsyms: %d\n", err);
 7375		return err;
 7376	}
 7377
 7378	while (true) {
 7379		ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
 7380			     &sym_addr, &sym_type, sym_name);
 7381		if (ret == EOF && feof(f))
 7382			break;
 7383		if (ret != 3) {
 7384			pr_warn("failed to read kallsyms entry: %d\n", ret);
 7385			err = -EINVAL;
 7386			break;
 7387		}
 7388
 7389		err = cb(sym_addr, sym_type, sym_name, ctx);
 7390		if (err)
 7391			break;
 7392	}
 7393
 7394	fclose(f);
 7395	return err;
 7396}
 7397
 7398static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
 7399		       const char *sym_name, void *ctx)
 7400{
 7401	struct bpf_object *obj = ctx;
 7402	const struct btf_type *t;
 7403	struct extern_desc *ext;
 7404
 7405	ext = find_extern_by_name(obj, sym_name);
 7406	if (!ext || ext->type != EXT_KSYM)
 7407		return 0;
 7408
 7409	t = btf__type_by_id(obj->btf, ext->btf_id);
 7410	if (!btf_is_var(t))
 7411		return 0;
 7412
 7413	if (ext->is_set && ext->ksym.addr != sym_addr) {
 7414		pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
 7415			sym_name, ext->ksym.addr, sym_addr);
 7416		return -EINVAL;
 7417	}
 7418	if (!ext->is_set) {
 7419		ext->is_set = true;
 7420		ext->ksym.addr = sym_addr;
 7421		pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
 7422	}
 7423	return 0;
 7424}
 7425
 7426static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
 7427{
 7428	return libbpf_kallsyms_parse(kallsyms_cb, obj);
 7429}
 7430
 7431static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
 7432			    __u16 kind, struct btf **res_btf,
 7433			    struct module_btf **res_mod_btf)
 7434{
 7435	struct module_btf *mod_btf;
 7436	struct btf *btf;
 7437	int i, id, err;
 7438
 7439	btf = obj->btf_vmlinux;
 7440	mod_btf = NULL;
 7441	id = btf__find_by_name_kind(btf, ksym_name, kind);
 7442
 7443	if (id == -ENOENT) {
 7444		err = load_module_btfs(obj);
 7445		if (err)
 7446			return err;
 7447
 7448		for (i = 0; i < obj->btf_module_cnt; i++) {
 7449			/* we assume module_btf's BTF FD is always >0 */
 7450			mod_btf = &obj->btf_modules[i];
 7451			btf = mod_btf->btf;
 7452			id = btf__find_by_name_kind_own(btf, ksym_name, kind);
 7453			if (id != -ENOENT)
 7454				break;
 7455		}
 7456	}
 7457	if (id <= 0)
 7458		return -ESRCH;
 7459
 7460	*res_btf = btf;
 7461	*res_mod_btf = mod_btf;
 7462	return id;
 7463}
 7464
 7465static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
 7466					       struct extern_desc *ext)
 7467{
 7468	const struct btf_type *targ_var, *targ_type;
 7469	__u32 targ_type_id, local_type_id;
 7470	struct module_btf *mod_btf = NULL;
 7471	const char *targ_var_name;
 7472	struct btf *btf = NULL;
 7473	int id, err;
 7474
 7475	id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
 7476	if (id < 0) {
 7477		if (id == -ESRCH && ext->is_weak)
 7478			return 0;
 7479		pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
 7480			ext->name);
 7481		return id;
 7482	}
 7483
 7484	/* find local type_id */
 7485	local_type_id = ext->ksym.type_id;
 7486
 7487	/* find target type_id */
 7488	targ_var = btf__type_by_id(btf, id);
 7489	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
 7490	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
 7491
 7492	err = bpf_core_types_are_compat(obj->btf, local_type_id,
 7493					btf, targ_type_id);
 7494	if (err <= 0) {
 7495		const struct btf_type *local_type;
 7496		const char *targ_name, *local_name;
 7497
 7498		local_type = btf__type_by_id(obj->btf, local_type_id);
 7499		local_name = btf__name_by_offset(obj->btf, local_type->name_off);
 7500		targ_name = btf__name_by_offset(btf, targ_type->name_off);
 7501
 7502		pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
 7503			ext->name, local_type_id,
 7504			btf_kind_str(local_type), local_name, targ_type_id,
 7505			btf_kind_str(targ_type), targ_name);
 7506		return -EINVAL;
 7507	}
 7508
 7509	ext->is_set = true;
 7510	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
 7511	ext->ksym.kernel_btf_id = id;
 7512	pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
 7513		 ext->name, id, btf_kind_str(targ_var), targ_var_name);
 7514
 7515	return 0;
 7516}
 7517
 7518static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
 7519						struct extern_desc *ext)
 7520{
 7521	int local_func_proto_id, kfunc_proto_id, kfunc_id;
 7522	struct module_btf *mod_btf = NULL;
 7523	const struct btf_type *kern_func;
 7524	struct btf *kern_btf = NULL;
 7525	int ret;
 7526
 7527	local_func_proto_id = ext->ksym.type_id;
 7528
 7529	kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
 7530	if (kfunc_id < 0) {
 7531		if (kfunc_id == -ESRCH && ext->is_weak)
 7532			return 0;
 7533		pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
 7534			ext->name);
 7535		return kfunc_id;
 7536	}
 7537
 7538	kern_func = btf__type_by_id(kern_btf, kfunc_id);
 7539	kfunc_proto_id = kern_func->type;
 7540
 7541	ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
 7542					kern_btf, kfunc_proto_id);
 7543	if (ret <= 0) {
 7544		pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
 7545			ext->name, local_func_proto_id, kfunc_proto_id);
 7546		return -EINVAL;
 7547	}
 7548
 7549	/* set index for module BTF fd in fd_array, if unset */
 7550	if (mod_btf && !mod_btf->fd_array_idx) {
 7551		/* insn->off is s16 */
 7552		if (obj->fd_array_cnt == INT16_MAX) {
 7553			pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
 7554				ext->name, mod_btf->fd_array_idx);
 7555			return -E2BIG;
 7556		}
 7557		/* Cannot use index 0 for module BTF fd */
 7558		if (!obj->fd_array_cnt)
 7559			obj->fd_array_cnt = 1;
 7560
 7561		ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
 7562					obj->fd_array_cnt + 1);
 7563		if (ret)
 7564			return ret;
 7565		mod_btf->fd_array_idx = obj->fd_array_cnt;
 7566		/* we assume module BTF FD is always >0 */
 7567		obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
 7568	}
 7569
 7570	ext->is_set = true;
 7571	ext->ksym.kernel_btf_id = kfunc_id;
 7572	ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
 7573	pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
 7574		 ext->name, kfunc_id);
 7575
 7576	return 0;
 7577}
 7578
 7579static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
 7580{
 7581	const struct btf_type *t;
 7582	struct extern_desc *ext;
 7583	int i, err;
 7584
 7585	for (i = 0; i < obj->nr_extern; i++) {
 7586		ext = &obj->externs[i];
 7587		if (ext->type != EXT_KSYM || !ext->ksym.type_id)
 7588			continue;
 7589
 7590		if (obj->gen_loader) {
 7591			ext->is_set = true;
 7592			ext->ksym.kernel_btf_obj_fd = 0;
 7593			ext->ksym.kernel_btf_id = 0;
 7594			continue;
 7595		}
 7596		t = btf__type_by_id(obj->btf, ext->btf_id);
 7597		if (btf_is_var(t))
 7598			err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
 7599		else
 7600			err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
 7601		if (err)
 7602			return err;
 7603	}
 7604	return 0;
 7605}
 7606
 7607static int bpf_object__resolve_externs(struct bpf_object *obj,
 7608				       const char *extra_kconfig)
 7609{
 7610	bool need_config = false, need_kallsyms = false;
 7611	bool need_vmlinux_btf = false;
 7612	struct extern_desc *ext;
 7613	void *kcfg_data = NULL;
 7614	int err, i;
 7615
 7616	if (obj->nr_extern == 0)
 7617		return 0;
 7618
 7619	if (obj->kconfig_map_idx >= 0)
 7620		kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
 7621
 7622	for (i = 0; i < obj->nr_extern; i++) {
 7623		ext = &obj->externs[i];
 7624
 7625		if (ext->type == EXT_KSYM) {
 7626			if (ext->ksym.type_id)
 7627				need_vmlinux_btf = true;
 7628			else
 7629				need_kallsyms = true;
 7630			continue;
 7631		} else if (ext->type == EXT_KCFG) {
 7632			void *ext_ptr = kcfg_data + ext->kcfg.data_off;
 7633			__u64 value = 0;
 7634
 7635			/* Kconfig externs need actual /proc/config.gz */
 7636			if (str_has_pfx(ext->name, "CONFIG_")) {
 7637				need_config = true;
 7638				continue;
 7639			}
 7640
 7641			/* Virtual kcfg externs are customly handled by libbpf */
 7642			if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
 7643				value = get_kernel_version();
 7644				if (!value) {
 7645					pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
 7646					return -EINVAL;
 7647				}
 7648			} else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
 7649				value = kernel_supports(obj, FEAT_BPF_COOKIE);
 7650			} else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
 7651				value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
 7652			} else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
 7653				/* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
 7654				 * __kconfig externs, where LINUX_ ones are virtual and filled out
 7655				 * customly by libbpf (their values don't come from Kconfig).
 7656				 * If LINUX_xxx variable is not recognized by libbpf, but is marked
 7657				 * __weak, it defaults to zero value, just like for CONFIG_xxx
 7658				 * externs.
 7659				 */
 7660				pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
 7661				return -EINVAL;
 7662			}
 7663
 7664			err = set_kcfg_value_num(ext, ext_ptr, value);
 7665			if (err)
 7666				return err;
 7667			pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
 7668				 ext->name, (long long)value);
 7669		} else {
 7670			pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
 7671			return -EINVAL;
 7672		}
 7673	}
 7674	if (need_config && extra_kconfig) {
 7675		err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
 7676		if (err)
 7677			return -EINVAL;
 7678		need_config = false;
 7679		for (i = 0; i < obj->nr_extern; i++) {
 7680			ext = &obj->externs[i];
 7681			if (ext->type == EXT_KCFG && !ext->is_set) {
 7682				need_config = true;
 7683				break;
 7684			}
 7685		}
 7686	}
 7687	if (need_config) {
 7688		err = bpf_object__read_kconfig_file(obj, kcfg_data);
 7689		if (err)
 7690			return -EINVAL;
 7691	}
 7692	if (need_kallsyms) {
 7693		err = bpf_object__read_kallsyms_file(obj);
 7694		if (err)
 7695			return -EINVAL;
 7696	}
 7697	if (need_vmlinux_btf) {
 7698		err = bpf_object__resolve_ksyms_btf_id(obj);
 7699		if (err)
 7700			return -EINVAL;
 7701	}
 7702	for (i = 0; i < obj->nr_extern; i++) {
 7703		ext = &obj->externs[i];
 7704
 7705		if (!ext->is_set && !ext->is_weak) {
 7706			pr_warn("extern '%s' (strong): not resolved\n", ext->name);
 7707			return -ESRCH;
 7708		} else if (!ext->is_set) {
 7709			pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
 7710				 ext->name);
 7711		}
 7712	}
 7713
 7714	return 0;
 7715}
 7716
 7717static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
 7718{
 7719	int err, i;
 7720
 7721	if (!obj)
 7722		return libbpf_err(-EINVAL);
 7723
 7724	if (obj->loaded) {
 7725		pr_warn("object '%s': load can't be attempted twice\n", obj->name);
 7726		return libbpf_err(-EINVAL);
 7727	}
 7728
 7729	if (obj->gen_loader)
 7730		bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
 7731
 7732	err = bpf_object__probe_loading(obj);
 7733	err = err ? : bpf_object__load_vmlinux_btf(obj, false);
 7734	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
 7735	err = err ? : bpf_object__sanitize_and_load_btf(obj);
 7736	err = err ? : bpf_object__sanitize_maps(obj);
 7737	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
 7738	err = err ? : bpf_object__create_maps(obj);
 7739	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
 7740	err = err ? : bpf_object__load_progs(obj, extra_log_level);
 7741	err = err ? : bpf_object_init_prog_arrays(obj);
 7742
 7743	if (obj->gen_loader) {
 7744		/* reset FDs */
 7745		if (obj->btf)
 7746			btf__set_fd(obj->btf, -1);
 7747		for (i = 0; i < obj->nr_maps; i++)
 7748			obj->maps[i].fd = -1;
 7749		if (!err)
 7750			err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
 7751	}
 7752
 7753	/* clean up fd_array */
 7754	zfree(&obj->fd_array);
 7755
 7756	/* clean up module BTFs */
 7757	for (i = 0; i < obj->btf_module_cnt; i++) {
 7758		close(obj->btf_modules[i].fd);
 7759		btf__free(obj->btf_modules[i].btf);
 7760		free(obj->btf_modules[i].name);
 7761	}
 7762	free(obj->btf_modules);
 7763
 7764	/* clean up vmlinux BTF */
 7765	btf__free(obj->btf_vmlinux);
 7766	obj->btf_vmlinux = NULL;
 7767
 7768	obj->loaded = true; /* doesn't matter if successfully or not */
 7769
 7770	if (err)
 7771		goto out;
 
 7772
 7773	return 0;
 7774out:
 7775	/* unpin any maps that were auto-pinned during load */
 7776	for (i = 0; i < obj->nr_maps; i++)
 7777		if (obj->maps[i].pinned && !obj->maps[i].reused)
 7778			bpf_map__unpin(&obj->maps[i], NULL);
 7779
 7780	bpf_object_unload(obj);
 7781	pr_warn("failed to load object '%s'\n", obj->path);
 7782	return libbpf_err(err);
 7783}
 7784
 7785int bpf_object__load(struct bpf_object *obj)
 7786{
 7787	return bpf_object_load(obj, 0, NULL);
 7788}
 7789
 7790static int make_parent_dir(const char *path)
 7791{
 7792	char *cp, errmsg[STRERR_BUFSIZE];
 7793	char *dname, *dir;
 7794	int err = 0;
 7795
 7796	dname = strdup(path);
 7797	if (dname == NULL)
 7798		return -ENOMEM;
 7799
 7800	dir = dirname(dname);
 7801	if (mkdir(dir, 0700) && errno != EEXIST)
 7802		err = -errno;
 7803
 7804	free(dname);
 7805	if (err) {
 7806		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
 7807		pr_warn("failed to mkdir %s: %s\n", path, cp);
 7808	}
 7809	return err;
 7810}
 7811
 7812static int check_path(const char *path)
 7813{
 7814	char *cp, errmsg[STRERR_BUFSIZE];
 7815	struct statfs st_fs;
 7816	char *dname, *dir;
 7817	int err = 0;
 7818
 7819	if (path == NULL)
 7820		return -EINVAL;
 7821
 7822	dname = strdup(path);
 7823	if (dname == NULL)
 7824		return -ENOMEM;
 7825
 7826	dir = dirname(dname);
 7827	if (statfs(dir, &st_fs)) {
 7828		cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
 7829		pr_warn("failed to statfs %s: %s\n", dir, cp);
 7830		err = -errno;
 7831	}
 7832	free(dname);
 7833
 7834	if (!err && st_fs.f_type != BPF_FS_MAGIC) {
 7835		pr_warn("specified path %s is not on BPF FS\n", path);
 7836		err = -EINVAL;
 7837	}
 7838
 7839	return err;
 7840}
 7841
 7842int bpf_program__pin(struct bpf_program *prog, const char *path)
 7843{
 7844	char *cp, errmsg[STRERR_BUFSIZE];
 7845	int err;
 7846
 7847	if (prog->fd < 0) {
 7848		pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
 7849		return libbpf_err(-EINVAL);
 7850	}
 7851
 7852	err = make_parent_dir(path);
 7853	if (err)
 7854		return libbpf_err(err);
 7855
 7856	err = check_path(path);
 7857	if (err)
 7858		return libbpf_err(err);
 7859
 7860	if (bpf_obj_pin(prog->fd, path)) {
 7861		err = -errno;
 7862		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 7863		pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
 7864		return libbpf_err(err);
 7865	}
 7866
 7867	pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
 7868	return 0;
 7869}
 7870
 7871int bpf_program__unpin(struct bpf_program *prog, const char *path)
 7872{
 7873	int err;
 7874
 7875	if (prog->fd < 0) {
 7876		pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
 7877		return libbpf_err(-EINVAL);
 7878	}
 7879
 7880	err = check_path(path);
 7881	if (err)
 7882		return libbpf_err(err);
 7883
 7884	err = unlink(path);
 7885	if (err)
 7886		return libbpf_err(-errno);
 7887
 7888	pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
 7889	return 0;
 7890}
 7891
 7892int bpf_map__pin(struct bpf_map *map, const char *path)
 7893{
 7894	char *cp, errmsg[STRERR_BUFSIZE];
 7895	int err;
 7896
 7897	if (map == NULL) {
 7898		pr_warn("invalid map pointer\n");
 7899		return libbpf_err(-EINVAL);
 7900	}
 7901
 7902	if (map->pin_path) {
 7903		if (path && strcmp(path, map->pin_path)) {
 7904			pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
 7905				bpf_map__name(map), map->pin_path, path);
 7906			return libbpf_err(-EINVAL);
 7907		} else if (map->pinned) {
 7908			pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
 7909				 bpf_map__name(map), map->pin_path);
 7910			return 0;
 7911		}
 7912	} else {
 7913		if (!path) {
 7914			pr_warn("missing a path to pin map '%s' at\n",
 7915				bpf_map__name(map));
 7916			return libbpf_err(-EINVAL);
 7917		} else if (map->pinned) {
 7918			pr_warn("map '%s' already pinned\n", bpf_map__name(map));
 7919			return libbpf_err(-EEXIST);
 7920		}
 7921
 7922		map->pin_path = strdup(path);
 7923		if (!map->pin_path) {
 7924			err = -errno;
 7925			goto out_err;
 7926		}
 7927	}
 7928
 7929	err = make_parent_dir(map->pin_path);
 7930	if (err)
 7931		return libbpf_err(err);
 7932
 7933	err = check_path(map->pin_path);
 7934	if (err)
 7935		return libbpf_err(err);
 7936
 7937	if (bpf_obj_pin(map->fd, map->pin_path)) {
 7938		err = -errno;
 7939		goto out_err;
 7940	}
 7941
 7942	map->pinned = true;
 7943	pr_debug("pinned map '%s'\n", map->pin_path);
 7944
 7945	return 0;
 7946
 7947out_err:
 7948	cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
 7949	pr_warn("failed to pin map: %s\n", cp);
 7950	return libbpf_err(err);
 7951}
 7952
 7953int bpf_map__unpin(struct bpf_map *map, const char *path)
 7954{
 7955	int err;
 7956
 7957	if (map == NULL) {
 7958		pr_warn("invalid map pointer\n");
 7959		return libbpf_err(-EINVAL);
 7960	}
 7961
 7962	if (map->pin_path) {
 7963		if (path && strcmp(path, map->pin_path)) {
 7964			pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
 7965				bpf_map__name(map), map->pin_path, path);
 7966			return libbpf_err(-EINVAL);
 7967		}
 7968		path = map->pin_path;
 7969	} else if (!path) {
 7970		pr_warn("no path to unpin map '%s' from\n",
 7971			bpf_map__name(map));
 7972		return libbpf_err(-EINVAL);
 7973	}
 7974
 7975	err = check_path(path);
 7976	if (err)
 7977		return libbpf_err(err);
 7978
 7979	err = unlink(path);
 7980	if (err != 0)
 7981		return libbpf_err(-errno);
 7982
 7983	map->pinned = false;
 7984	pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
 7985
 7986	return 0;
 7987}
 7988
 7989int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
 7990{
 7991	char *new = NULL;
 7992
 7993	if (path) {
 7994		new = strdup(path);
 7995		if (!new)
 7996			return libbpf_err(-errno);
 7997	}
 7998
 7999	free(map->pin_path);
 8000	map->pin_path = new;
 8001	return 0;
 8002}
 8003
 8004__alias(bpf_map__pin_path)
 8005const char *bpf_map__get_pin_path(const struct bpf_map *map);
 8006
 8007const char *bpf_map__pin_path(const struct bpf_map *map)
 8008{
 8009	return map->pin_path;
 8010}
 8011
 8012bool bpf_map__is_pinned(const struct bpf_map *map)
 8013{
 8014	return map->pinned;
 8015}
 8016
 8017static void sanitize_pin_path(char *s)
 8018{
 8019	/* bpffs disallows periods in path names */
 8020	while (*s) {
 8021		if (*s == '.')
 8022			*s = '_';
 8023		s++;
 8024	}
 8025}
 8026
 8027int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
 8028{
 8029	struct bpf_map *map;
 8030	int err;
 8031
 8032	if (!obj)
 8033		return libbpf_err(-ENOENT);
 8034
 8035	if (!obj->loaded) {
 8036		pr_warn("object not yet loaded; load it first\n");
 8037		return libbpf_err(-ENOENT);
 8038	}
 8039
 8040	bpf_object__for_each_map(map, obj) {
 8041		char *pin_path = NULL;
 8042		char buf[PATH_MAX];
 8043
 8044		if (!map->autocreate)
 8045			continue;
 8046
 8047		if (path) {
 8048			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
 8049			if (err)
 8050				goto err_unpin_maps;
 8051			sanitize_pin_path(buf);
 8052			pin_path = buf;
 8053		} else if (!map->pin_path) {
 8054			continue;
 8055		}
 8056
 8057		err = bpf_map__pin(map, pin_path);
 8058		if (err)
 8059			goto err_unpin_maps;
 8060	}
 8061
 8062	return 0;
 8063
 8064err_unpin_maps:
 8065	while ((map = bpf_object__prev_map(obj, map))) {
 8066		if (!map->pin_path)
 8067			continue;
 8068
 8069		bpf_map__unpin(map, NULL);
 8070	}
 8071
 8072	return libbpf_err(err);
 8073}
 8074
 8075int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
 8076{
 8077	struct bpf_map *map;
 8078	int err;
 8079
 8080	if (!obj)
 8081		return libbpf_err(-ENOENT);
 8082
 8083	bpf_object__for_each_map(map, obj) {
 8084		char *pin_path = NULL;
 8085		char buf[PATH_MAX];
 8086
 8087		if (path) {
 8088			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
 8089			if (err)
 8090				return libbpf_err(err);
 8091			sanitize_pin_path(buf);
 8092			pin_path = buf;
 8093		} else if (!map->pin_path) {
 8094			continue;
 8095		}
 8096
 8097		err = bpf_map__unpin(map, pin_path);
 8098		if (err)
 8099			return libbpf_err(err);
 8100	}
 8101
 8102	return 0;
 8103}
 8104
 8105int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
 8106{
 8107	struct bpf_program *prog;
 8108	char buf[PATH_MAX];
 8109	int err;
 8110
 8111	if (!obj)
 8112		return libbpf_err(-ENOENT);
 8113
 8114	if (!obj->loaded) {
 8115		pr_warn("object not yet loaded; load it first\n");
 8116		return libbpf_err(-ENOENT);
 8117	}
 8118
 8119	bpf_object__for_each_program(prog, obj) {
 8120		err = pathname_concat(buf, sizeof(buf), path, prog->name);
 8121		if (err)
 8122			goto err_unpin_programs;
 8123
 8124		err = bpf_program__pin(prog, buf);
 8125		if (err)
 8126			goto err_unpin_programs;
 8127	}
 8128
 8129	return 0;
 8130
 8131err_unpin_programs:
 8132	while ((prog = bpf_object__prev_program(obj, prog))) {
 8133		if (pathname_concat(buf, sizeof(buf), path, prog->name))
 8134			continue;
 8135
 8136		bpf_program__unpin(prog, buf);
 8137	}
 8138
 8139	return libbpf_err(err);
 8140}
 8141
 8142int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
 8143{
 8144	struct bpf_program *prog;
 8145	int err;
 8146
 8147	if (!obj)
 8148		return libbpf_err(-ENOENT);
 8149
 8150	bpf_object__for_each_program(prog, obj) {
 8151		char buf[PATH_MAX];
 8152
 8153		err = pathname_concat(buf, sizeof(buf), path, prog->name);
 8154		if (err)
 8155			return libbpf_err(err);
 8156
 8157		err = bpf_program__unpin(prog, buf);
 8158		if (err)
 8159			return libbpf_err(err);
 8160	}
 8161
 8162	return 0;
 8163}
 8164
 8165int bpf_object__pin(struct bpf_object *obj, const char *path)
 8166{
 8167	int err;
 8168
 8169	err = bpf_object__pin_maps(obj, path);
 8170	if (err)
 8171		return libbpf_err(err);
 8172
 8173	err = bpf_object__pin_programs(obj, path);
 8174	if (err) {
 8175		bpf_object__unpin_maps(obj, path);
 8176		return libbpf_err(err);
 8177	}
 8178
 8179	return 0;
 8180}
 8181
 8182static void bpf_map__destroy(struct bpf_map *map)
 8183{
 8184	if (map->inner_map) {
 8185		bpf_map__destroy(map->inner_map);
 8186		zfree(&map->inner_map);
 8187	}
 8188
 8189	zfree(&map->init_slots);
 8190	map->init_slots_sz = 0;
 8191
 8192	if (map->mmaped) {
 8193		munmap(map->mmaped, bpf_map_mmap_sz(map));
 8194		map->mmaped = NULL;
 8195	}
 8196
 8197	if (map->st_ops) {
 8198		zfree(&map->st_ops->data);
 8199		zfree(&map->st_ops->progs);
 8200		zfree(&map->st_ops->kern_func_off);
 8201		zfree(&map->st_ops);
 8202	}
 8203
 8204	zfree(&map->name);
 8205	zfree(&map->real_name);
 8206	zfree(&map->pin_path);
 8207
 8208	if (map->fd >= 0)
 8209		zclose(map->fd);
 8210}
 8211
 8212void bpf_object__close(struct bpf_object *obj)
 8213{
 8214	size_t i;
 8215
 8216	if (IS_ERR_OR_NULL(obj))
 8217		return;
 8218
 8219	usdt_manager_free(obj->usdt_man);
 8220	obj->usdt_man = NULL;
 8221
 8222	bpf_gen__free(obj->gen_loader);
 8223	bpf_object__elf_finish(obj);
 8224	bpf_object_unload(obj);
 8225	btf__free(obj->btf);
 8226	btf_ext__free(obj->btf_ext);
 8227
 8228	for (i = 0; i < obj->nr_maps; i++)
 8229		bpf_map__destroy(&obj->maps[i]);
 8230
 8231	zfree(&obj->btf_custom_path);
 8232	zfree(&obj->kconfig);
 8233	zfree(&obj->externs);
 8234	obj->nr_extern = 0;
 8235
 
 
 
 
 
 
 
 
 8236	zfree(&obj->maps);
 8237	obj->nr_maps = 0;
 8238
 8239	if (obj->programs && obj->nr_programs) {
 8240		for (i = 0; i < obj->nr_programs; i++)
 8241			bpf_program__exit(&obj->programs[i]);
 8242	}
 8243	zfree(&obj->programs);
 8244
 
 8245	free(obj);
 8246}
 8247
 8248const char *bpf_object__name(const struct bpf_object *obj)
 
 8249{
 8250	return obj ? obj->name : libbpf_err_ptr(-EINVAL);
 8251}
 8252
 8253unsigned int bpf_object__kversion(const struct bpf_object *obj)
 8254{
 8255	return obj ? obj->kern_version : 0;
 8256}
 
 
 8257
 8258struct btf *bpf_object__btf(const struct bpf_object *obj)
 8259{
 8260	return obj ? obj->btf : NULL;
 8261}
 8262
 8263int bpf_object__btf_fd(const struct bpf_object *obj)
 8264{
 8265	return obj->btf ? btf__fd(obj->btf) : -1;
 8266}
 8267
 8268int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
 
 8269{
 8270	if (obj->loaded)
 8271		return libbpf_err(-EINVAL);
 8272
 8273	obj->kern_version = kern_version;
 8274
 8275	return 0;
 8276}
 8277
 8278int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
 
 8279{
 8280	struct bpf_gen *gen;
 8281
 8282	if (!opts)
 8283		return -EFAULT;
 8284	if (!OPTS_VALID(opts, gen_loader_opts))
 8285		return -EINVAL;
 8286	gen = calloc(sizeof(*gen), 1);
 8287	if (!gen)
 8288		return -ENOMEM;
 8289	gen->opts = opts;
 8290	obj->gen_loader = gen;
 8291	return 0;
 8292}
 8293
 8294static struct bpf_program *
 8295__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
 8296		    bool forward)
 8297{
 8298	size_t nr_programs = obj->nr_programs;
 8299	ssize_t idx;
 8300
 8301	if (!nr_programs)
 8302		return NULL;
 
 
 
 8303
 8304	if (!p)
 8305		/* Iter from the beginning */
 8306		return forward ? &obj->programs[0] :
 8307			&obj->programs[nr_programs - 1];
 8308
 8309	if (p->obj != obj) {
 8310		pr_warn("error: program handler doesn't match object\n");
 8311		return errno = EINVAL, NULL;
 8312	}
 8313
 8314	idx = (p - obj->programs) + (forward ? 1 : -1);
 8315	if (idx >= obj->nr_programs || idx < 0)
 8316		return NULL;
 8317	return &obj->programs[idx];
 8318}
 8319
 8320struct bpf_program *
 8321bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
 8322{
 8323	struct bpf_program *prog = prev;
 8324
 8325	do {
 8326		prog = __bpf_program__iter(prog, obj, true);
 8327	} while (prog && prog_is_subprog(obj, prog));
 8328
 8329	return prog;
 8330}
 8331
 8332struct bpf_program *
 8333bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
 8334{
 8335	struct bpf_program *prog = next;
 8336
 8337	do {
 8338		prog = __bpf_program__iter(prog, obj, false);
 8339	} while (prog && prog_is_subprog(obj, prog));
 8340
 8341	return prog;
 8342}
 8343
 8344void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
 8345{
 8346	prog->prog_ifindex = ifindex;
 8347}
 8348
 8349const char *bpf_program__name(const struct bpf_program *prog)
 8350{
 8351	return prog->name;
 8352}
 8353
 8354const char *bpf_program__section_name(const struct bpf_program *prog)
 8355{
 8356	return prog->sec_name;
 8357}
 8358
 8359bool bpf_program__autoload(const struct bpf_program *prog)
 8360{
 8361	return prog->autoload;
 8362}
 8363
 8364int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
 8365{
 8366	if (prog->obj->loaded)
 8367		return libbpf_err(-EINVAL);
 8368
 8369	prog->autoload = autoload;
 8370	return 0;
 8371}
 8372
 8373bool bpf_program__autoattach(const struct bpf_program *prog)
 8374{
 8375	return prog->autoattach;
 8376}
 8377
 8378void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
 8379{
 8380	prog->autoattach = autoattach;
 8381}
 8382
 8383const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
 8384{
 8385	return prog->insns;
 8386}
 8387
 8388size_t bpf_program__insn_cnt(const struct bpf_program *prog)
 8389{
 8390	return prog->insns_cnt;
 8391}
 8392
 8393int bpf_program__set_insns(struct bpf_program *prog,
 8394			   struct bpf_insn *new_insns, size_t new_insn_cnt)
 8395{
 8396	struct bpf_insn *insns;
 8397
 8398	if (prog->obj->loaded)
 8399		return -EBUSY;
 8400
 8401	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
 8402	if (!insns) {
 8403		pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
 8404		return -ENOMEM;
 8405	}
 8406	memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
 8407
 8408	prog->insns = insns;
 8409	prog->insns_cnt = new_insn_cnt;
 8410	return 0;
 8411}
 8412
 8413int bpf_program__fd(const struct bpf_program *prog)
 8414{
 8415	if (!prog)
 8416		return libbpf_err(-EINVAL);
 8417
 8418	if (prog->fd < 0)
 8419		return libbpf_err(-ENOENT);
 8420
 8421	return prog->fd;
 8422}
 8423
 8424__alias(bpf_program__type)
 8425enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
 8426
 8427enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
 8428{
 8429	return prog->type;
 8430}
 8431
 8432int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
 8433{
 8434	if (prog->obj->loaded)
 8435		return libbpf_err(-EBUSY);
 8436
 8437	prog->type = type;
 
 8438	return 0;
 8439}
 8440
 8441__alias(bpf_program__expected_attach_type)
 8442enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
 8443
 8444enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
 8445{
 8446	return prog->expected_attach_type;
 8447}
 8448
 8449int bpf_program__set_expected_attach_type(struct bpf_program *prog,
 8450					   enum bpf_attach_type type)
 8451{
 8452	if (prog->obj->loaded)
 8453		return libbpf_err(-EBUSY);
 8454
 8455	prog->expected_attach_type = type;
 8456	return 0;
 8457}
 8458
 8459__u32 bpf_program__flags(const struct bpf_program *prog)
 8460{
 8461	return prog->prog_flags;
 8462}
 8463
 8464int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
 8465{
 8466	if (prog->obj->loaded)
 8467		return libbpf_err(-EBUSY);
 8468
 8469	prog->prog_flags = flags;
 8470	return 0;
 8471}
 8472
 8473__u32 bpf_program__log_level(const struct bpf_program *prog)
 8474{
 8475	return prog->log_level;
 8476}
 8477
 8478int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
 8479{
 8480	if (prog->obj->loaded)
 8481		return libbpf_err(-EBUSY);
 8482
 8483	prog->log_level = log_level;
 8484	return 0;
 8485}
 8486
 8487const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
 8488{
 8489	*log_size = prog->log_size;
 8490	return prog->log_buf;
 8491}
 8492
 8493int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
 8494{
 8495	if (log_size && !log_buf)
 8496		return -EINVAL;
 8497	if (prog->log_size > UINT_MAX)
 8498		return -EINVAL;
 8499	if (prog->obj->loaded)
 8500		return -EBUSY;
 8501
 8502	prog->log_buf = log_buf;
 8503	prog->log_size = log_size;
 8504	return 0;
 8505}
 8506
 8507#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {			    \
 8508	.sec = (char *)sec_pfx,						    \
 8509	.prog_type = BPF_PROG_TYPE_##ptype,				    \
 8510	.expected_attach_type = atype,					    \
 8511	.cookie = (long)(flags),					    \
 8512	.prog_prepare_load_fn = libbpf_prepare_prog_load,		    \
 8513	__VA_ARGS__							    \
 8514}
 8515
 8516static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8517static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8518static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8519static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8520static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8521static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8522static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8523static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8524static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8525static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
 8526
 8527static const struct bpf_sec_def section_defs[] = {
 8528	SEC_DEF("socket",		SOCKET_FILTER, 0, SEC_NONE),
 8529	SEC_DEF("sk_reuseport/migrate",	SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
 8530	SEC_DEF("sk_reuseport",		SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
 8531	SEC_DEF("kprobe+",		KPROBE,	0, SEC_NONE, attach_kprobe),
 8532	SEC_DEF("uprobe+",		KPROBE,	0, SEC_NONE, attach_uprobe),
 8533	SEC_DEF("uprobe.s+",		KPROBE,	0, SEC_SLEEPABLE, attach_uprobe),
 8534	SEC_DEF("kretprobe+",		KPROBE, 0, SEC_NONE, attach_kprobe),
 8535	SEC_DEF("uretprobe+",		KPROBE, 0, SEC_NONE, attach_uprobe),
 8536	SEC_DEF("uretprobe.s+",		KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
 8537	SEC_DEF("kprobe.multi+",	KPROBE,	BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
 8538	SEC_DEF("kretprobe.multi+",	KPROBE,	BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
 8539	SEC_DEF("ksyscall+",		KPROBE,	0, SEC_NONE, attach_ksyscall),
 8540	SEC_DEF("kretsyscall+",		KPROBE, 0, SEC_NONE, attach_ksyscall),
 8541	SEC_DEF("usdt+",		KPROBE,	0, SEC_NONE, attach_usdt),
 8542	SEC_DEF("tc",			SCHED_CLS, 0, SEC_NONE),
 8543	SEC_DEF("classifier",		SCHED_CLS, 0, SEC_NONE),
 8544	SEC_DEF("action",		SCHED_ACT, 0, SEC_NONE),
 8545	SEC_DEF("tracepoint+",		TRACEPOINT, 0, SEC_NONE, attach_tp),
 8546	SEC_DEF("tp+",			TRACEPOINT, 0, SEC_NONE, attach_tp),
 8547	SEC_DEF("raw_tracepoint+",	RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
 8548	SEC_DEF("raw_tp+",		RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
 8549	SEC_DEF("raw_tracepoint.w+",	RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
 8550	SEC_DEF("raw_tp.w+",		RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
 8551	SEC_DEF("tp_btf+",		TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
 8552	SEC_DEF("fentry+",		TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
 8553	SEC_DEF("fmod_ret+",		TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
 8554	SEC_DEF("fexit+",		TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
 8555	SEC_DEF("fentry.s+",		TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
 8556	SEC_DEF("fmod_ret.s+",		TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
 8557	SEC_DEF("fexit.s+",		TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
 8558	SEC_DEF("freplace+",		EXT, 0, SEC_ATTACH_BTF, attach_trace),
 8559	SEC_DEF("lsm+",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 8560	SEC_DEF("lsm.s+",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 8561	SEC_DEF("lsm_cgroup+",		LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
 8562	SEC_DEF("iter+",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
 8563	SEC_DEF("iter.s+",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 8564	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 8565	SEC_DEF("xdp.frags/devmap",	XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
 8566	SEC_DEF("xdp/devmap",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 8567	SEC_DEF("xdp.frags/cpumap",	XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
 8568	SEC_DEF("xdp/cpumap",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
 8569	SEC_DEF("xdp.frags",		XDP, BPF_XDP, SEC_XDP_FRAGS),
 8570	SEC_DEF("xdp",			XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
 8571	SEC_DEF("perf_event",		PERF_EVENT, 0, SEC_NONE),
 8572	SEC_DEF("lwt_in",		LWT_IN, 0, SEC_NONE),
 8573	SEC_DEF("lwt_out",		LWT_OUT, 0, SEC_NONE),
 8574	SEC_DEF("lwt_xmit",		LWT_XMIT, 0, SEC_NONE),
 8575	SEC_DEF("lwt_seg6local",	LWT_SEG6LOCAL, 0, SEC_NONE),
 8576	SEC_DEF("sockops",		SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
 8577	SEC_DEF("sk_skb/stream_parser",	SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
 8578	SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
 8579	SEC_DEF("sk_skb",		SK_SKB, 0, SEC_NONE),
 8580	SEC_DEF("sk_msg",		SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
 8581	SEC_DEF("lirc_mode2",		LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
 8582	SEC_DEF("flow_dissector",	FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
 8583	SEC_DEF("cgroup_skb/ingress",	CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
 8584	SEC_DEF("cgroup_skb/egress",	CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
 8585	SEC_DEF("cgroup/skb",		CGROUP_SKB, 0, SEC_NONE),
 8586	SEC_DEF("cgroup/sock_create",	CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
 8587	SEC_DEF("cgroup/sock_release",	CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
 8588	SEC_DEF("cgroup/sock",		CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
 8589	SEC_DEF("cgroup/post_bind4",	CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
 8590	SEC_DEF("cgroup/post_bind6",	CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
 8591	SEC_DEF("cgroup/bind4",		CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
 8592	SEC_DEF("cgroup/bind6",		CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
 8593	SEC_DEF("cgroup/connect4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
 8594	SEC_DEF("cgroup/connect6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
 8595	SEC_DEF("cgroup/sendmsg4",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
 8596	SEC_DEF("cgroup/sendmsg6",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
 8597	SEC_DEF("cgroup/recvmsg4",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
 8598	SEC_DEF("cgroup/recvmsg6",	CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
 8599	SEC_DEF("cgroup/getpeername4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
 8600	SEC_DEF("cgroup/getpeername6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
 8601	SEC_DEF("cgroup/getsockname4",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
 8602	SEC_DEF("cgroup/getsockname6",	CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
 8603	SEC_DEF("cgroup/sysctl",	CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
 8604	SEC_DEF("cgroup/getsockopt",	CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
 8605	SEC_DEF("cgroup/setsockopt",	CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
 8606	SEC_DEF("cgroup/dev",		CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
 8607	SEC_DEF("struct_ops+",		STRUCT_OPS, 0, SEC_NONE),
 8608	SEC_DEF("sk_lookup",		SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
 8609};
 8610
 8611static size_t custom_sec_def_cnt;
 8612static struct bpf_sec_def *custom_sec_defs;
 8613static struct bpf_sec_def custom_fallback_def;
 8614static bool has_custom_fallback_def;
 8615
 8616static int last_custom_sec_def_handler_id;
 8617
 8618int libbpf_register_prog_handler(const char *sec,
 8619				 enum bpf_prog_type prog_type,
 8620				 enum bpf_attach_type exp_attach_type,
 8621				 const struct libbpf_prog_handler_opts *opts)
 8622{
 8623	struct bpf_sec_def *sec_def;
 8624
 8625	if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
 8626		return libbpf_err(-EINVAL);
 8627
 8628	if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
 8629		return libbpf_err(-E2BIG);
 8630
 8631	if (sec) {
 8632		sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
 8633					      sizeof(*sec_def));
 8634		if (!sec_def)
 8635			return libbpf_err(-ENOMEM);
 8636
 8637		custom_sec_defs = sec_def;
 8638		sec_def = &custom_sec_defs[custom_sec_def_cnt];
 8639	} else {
 8640		if (has_custom_fallback_def)
 8641			return libbpf_err(-EBUSY);
 8642
 8643		sec_def = &custom_fallback_def;
 8644	}
 8645
 8646	sec_def->sec = sec ? strdup(sec) : NULL;
 8647	if (sec && !sec_def->sec)
 8648		return libbpf_err(-ENOMEM);
 8649
 8650	sec_def->prog_type = prog_type;
 8651	sec_def->expected_attach_type = exp_attach_type;
 8652	sec_def->cookie = OPTS_GET(opts, cookie, 0);
 8653
 8654	sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
 8655	sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
 8656	sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
 8657
 8658	sec_def->handler_id = ++last_custom_sec_def_handler_id;
 8659
 8660	if (sec)
 8661		custom_sec_def_cnt++;
 8662	else
 8663		has_custom_fallback_def = true;
 8664
 8665	return sec_def->handler_id;
 8666}
 8667
 8668int libbpf_unregister_prog_handler(int handler_id)
 8669{
 8670	struct bpf_sec_def *sec_defs;
 8671	int i;
 8672
 8673	if (handler_id <= 0)
 8674		return libbpf_err(-EINVAL);
 8675
 8676	if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
 8677		memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
 8678		has_custom_fallback_def = false;
 8679		return 0;
 8680	}
 8681
 8682	for (i = 0; i < custom_sec_def_cnt; i++) {
 8683		if (custom_sec_defs[i].handler_id == handler_id)
 8684			break;
 8685	}
 8686
 8687	if (i == custom_sec_def_cnt)
 8688		return libbpf_err(-ENOENT);
 8689
 8690	free(custom_sec_defs[i].sec);
 8691	for (i = i + 1; i < custom_sec_def_cnt; i++)
 8692		custom_sec_defs[i - 1] = custom_sec_defs[i];
 8693	custom_sec_def_cnt--;
 8694
 8695	/* try to shrink the array, but it's ok if we couldn't */
 8696	sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
 8697	if (sec_defs)
 8698		custom_sec_defs = sec_defs;
 8699
 8700	return 0;
 8701}
 8702
 8703static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
 8704{
 8705	size_t len = strlen(sec_def->sec);
 8706
 8707	/* "type/" always has to have proper SEC("type/extras") form */
 8708	if (sec_def->sec[len - 1] == '/') {
 8709		if (str_has_pfx(sec_name, sec_def->sec))
 8710			return true;
 8711		return false;
 8712	}
 8713
 8714	/* "type+" means it can be either exact SEC("type") or
 8715	 * well-formed SEC("type/extras") with proper '/' separator
 8716	 */
 8717	if (sec_def->sec[len - 1] == '+') {
 8718		len--;
 8719		/* not even a prefix */
 8720		if (strncmp(sec_name, sec_def->sec, len) != 0)
 8721			return false;
 8722		/* exact match or has '/' separator */
 8723		if (sec_name[len] == '\0' || sec_name[len] == '/')
 8724			return true;
 8725		return false;
 8726	}
 8727
 8728	return strcmp(sec_name, sec_def->sec) == 0;
 8729}
 8730
 8731static const struct bpf_sec_def *find_sec_def(const char *sec_name)
 8732{
 8733	const struct bpf_sec_def *sec_def;
 8734	int i, n;
 8735
 8736	n = custom_sec_def_cnt;
 8737	for (i = 0; i < n; i++) {
 8738		sec_def = &custom_sec_defs[i];
 8739		if (sec_def_matches(sec_def, sec_name))
 8740			return sec_def;
 8741	}
 8742
 8743	n = ARRAY_SIZE(section_defs);
 8744	for (i = 0; i < n; i++) {
 8745		sec_def = &section_defs[i];
 8746		if (sec_def_matches(sec_def, sec_name))
 8747			return sec_def;
 8748	}
 8749
 8750	if (has_custom_fallback_def)
 8751		return &custom_fallback_def;
 8752
 8753	return NULL;
 8754}
 8755
 8756#define MAX_TYPE_NAME_SIZE 32
 8757
 8758static char *libbpf_get_type_names(bool attach_type)
 8759{
 8760	int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
 8761	char *buf;
 8762
 8763	buf = malloc(len);
 8764	if (!buf)
 8765		return NULL;
 8766
 8767	buf[0] = '\0';
 8768	/* Forge string buf with all available names */
 8769	for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
 8770		const struct bpf_sec_def *sec_def = &section_defs[i];
 8771
 8772		if (attach_type) {
 8773			if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
 8774				continue;
 8775
 8776			if (!(sec_def->cookie & SEC_ATTACHABLE))
 8777				continue;
 8778		}
 8779
 8780		if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
 8781			free(buf);
 8782			return NULL;
 8783		}
 8784		strcat(buf, " ");
 8785		strcat(buf, section_defs[i].sec);
 8786	}
 8787
 8788	return buf;
 8789}
 8790
 8791int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
 8792			     enum bpf_attach_type *expected_attach_type)
 8793{
 8794	const struct bpf_sec_def *sec_def;
 8795	char *type_names;
 8796
 8797	if (!name)
 8798		return libbpf_err(-EINVAL);
 8799
 8800	sec_def = find_sec_def(name);
 8801	if (sec_def) {
 8802		*prog_type = sec_def->prog_type;
 8803		*expected_attach_type = sec_def->expected_attach_type;
 8804		return 0;
 8805	}
 8806
 8807	pr_debug("failed to guess program type from ELF section '%s'\n", name);
 8808	type_names = libbpf_get_type_names(false);
 8809	if (type_names != NULL) {
 8810		pr_debug("supported section(type) names are:%s\n", type_names);
 8811		free(type_names);
 8812	}
 8813
 8814	return libbpf_err(-ESRCH);
 8815}
 8816
 8817const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
 
 8818{
 8819	if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
 8820		return NULL;
 8821
 8822	return attach_type_name[t];
 8823}
 8824
 8825const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
 8826{
 8827	if (t < 0 || t >= ARRAY_SIZE(link_type_name))
 8828		return NULL;
 8829
 8830	return link_type_name[t];
 8831}
 8832
 8833const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
 8834{
 8835	if (t < 0 || t >= ARRAY_SIZE(map_type_name))
 8836		return NULL;
 8837
 8838	return map_type_name[t];
 8839}
 8840
 8841const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
 8842{
 8843	if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
 8844		return NULL;
 8845
 8846	return prog_type_name[t];
 8847}
 8848
 8849static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
 8850						     size_t offset)
 8851{
 8852	struct bpf_map *map;
 8853	size_t i;
 8854
 8855	for (i = 0; i < obj->nr_maps; i++) {
 8856		map = &obj->maps[i];
 8857		if (!bpf_map__is_struct_ops(map))
 8858			continue;
 8859		if (map->sec_offset <= offset &&
 8860		    offset - map->sec_offset < map->def.value_size)
 8861			return map;
 8862	}
 8863
 8864	return NULL;
 8865}
 8866
 8867/* Collect the reloc from ELF and populate the st_ops->progs[] */
 8868static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
 8869					    Elf64_Shdr *shdr, Elf_Data *data)
 8870{
 8871	const struct btf_member *member;
 8872	struct bpf_struct_ops *st_ops;
 8873	struct bpf_program *prog;
 8874	unsigned int shdr_idx;
 8875	const struct btf *btf;
 8876	struct bpf_map *map;
 8877	unsigned int moff, insn_idx;
 8878	const char *name;
 8879	__u32 member_idx;
 8880	Elf64_Sym *sym;
 8881	Elf64_Rel *rel;
 8882	int i, nrels;
 8883
 8884	btf = obj->btf;
 8885	nrels = shdr->sh_size / shdr->sh_entsize;
 8886	for (i = 0; i < nrels; i++) {
 8887		rel = elf_rel_by_idx(data, i);
 8888		if (!rel) {
 8889			pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
 8890			return -LIBBPF_ERRNO__FORMAT;
 8891		}
 8892
 8893		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
 8894		if (!sym) {
 8895			pr_warn("struct_ops reloc: symbol %zx not found\n",
 8896				(size_t)ELF64_R_SYM(rel->r_info));
 8897			return -LIBBPF_ERRNO__FORMAT;
 8898		}
 8899
 8900		name = elf_sym_str(obj, sym->st_name) ?: "<?>";
 8901		map = find_struct_ops_map_by_offset(obj, rel->r_offset);
 8902		if (!map) {
 8903			pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
 8904				(size_t)rel->r_offset);
 8905			return -EINVAL;
 8906		}
 8907
 8908		moff = rel->r_offset - map->sec_offset;
 8909		shdr_idx = sym->st_shndx;
 8910		st_ops = map->st_ops;
 8911		pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
 8912			 map->name,
 8913			 (long long)(rel->r_info >> 32),
 8914			 (long long)sym->st_value,
 8915			 shdr_idx, (size_t)rel->r_offset,
 8916			 map->sec_offset, sym->st_name, name);
 8917
 8918		if (shdr_idx >= SHN_LORESERVE) {
 8919			pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
 8920				map->name, (size_t)rel->r_offset, shdr_idx);
 8921			return -LIBBPF_ERRNO__RELOC;
 8922		}
 8923		if (sym->st_value % BPF_INSN_SZ) {
 8924			pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
 8925				map->name, (unsigned long long)sym->st_value);
 8926			return -LIBBPF_ERRNO__FORMAT;
 8927		}
 8928		insn_idx = sym->st_value / BPF_INSN_SZ;
 8929
 8930		member = find_member_by_offset(st_ops->type, moff * 8);
 8931		if (!member) {
 8932			pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
 8933				map->name, moff);
 8934			return -EINVAL;
 8935		}
 8936		member_idx = member - btf_members(st_ops->type);
 8937		name = btf__name_by_offset(btf, member->name_off);
 8938
 8939		if (!resolve_func_ptr(btf, member->type, NULL)) {
 8940			pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
 8941				map->name, name);
 8942			return -EINVAL;
 8943		}
 8944
 8945		prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
 8946		if (!prog) {
 8947			pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
 8948				map->name, shdr_idx, name);
 8949			return -EINVAL;
 8950		}
 8951
 8952		/* prevent the use of BPF prog with invalid type */
 8953		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
 8954			pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
 8955				map->name, prog->name);
 8956			return -EINVAL;
 8957		}
 8958
 8959		/* if we haven't yet processed this BPF program, record proper
 8960		 * attach_btf_id and member_idx
 8961		 */
 8962		if (!prog->attach_btf_id) {
 8963			prog->attach_btf_id = st_ops->type_id;
 8964			prog->expected_attach_type = member_idx;
 8965		}
 8966
 8967		/* struct_ops BPF prog can be re-used between multiple
 8968		 * .struct_ops as long as it's the same struct_ops struct
 8969		 * definition and the same function pointer field
 8970		 */
 8971		if (prog->attach_btf_id != st_ops->type_id ||
 8972		    prog->expected_attach_type != member_idx) {
 8973			pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
 8974				map->name, prog->name, prog->sec_name, prog->type,
 8975				prog->attach_btf_id, prog->expected_attach_type, name);
 8976			return -EINVAL;
 8977		}
 8978
 8979		st_ops->progs[member_idx] = prog;
 8980	}
 8981
 8982	return 0;
 8983}
 8984
 8985#define BTF_TRACE_PREFIX "btf_trace_"
 8986#define BTF_LSM_PREFIX "bpf_lsm_"
 8987#define BTF_ITER_PREFIX "bpf_iter_"
 8988#define BTF_MAX_NAME_SIZE 128
 8989
 8990void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
 8991				const char **prefix, int *kind)
 8992{
 8993	switch (attach_type) {
 8994	case BPF_TRACE_RAW_TP:
 8995		*prefix = BTF_TRACE_PREFIX;
 8996		*kind = BTF_KIND_TYPEDEF;
 8997		break;
 8998	case BPF_LSM_MAC:
 8999	case BPF_LSM_CGROUP:
 9000		*prefix = BTF_LSM_PREFIX;
 9001		*kind = BTF_KIND_FUNC;
 9002		break;
 9003	case BPF_TRACE_ITER:
 9004		*prefix = BTF_ITER_PREFIX;
 9005		*kind = BTF_KIND_FUNC;
 9006		break;
 9007	default:
 9008		*prefix = "";
 9009		*kind = BTF_KIND_FUNC;
 9010	}
 9011}
 9012
 9013static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 9014				   const char *name, __u32 kind)
 9015{
 9016	char btf_type_name[BTF_MAX_NAME_SIZE];
 9017	int ret;
 9018
 9019	ret = snprintf(btf_type_name, sizeof(btf_type_name),
 9020		       "%s%s", prefix, name);
 9021	/* snprintf returns the number of characters written excluding the
 9022	 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
 9023	 * indicates truncation.
 9024	 */
 9025	if (ret < 0 || ret >= sizeof(btf_type_name))
 9026		return -ENAMETOOLONG;
 9027	return btf__find_by_name_kind(btf, btf_type_name, kind);
 9028}
 9029
 9030static inline int find_attach_btf_id(struct btf *btf, const char *name,
 9031				     enum bpf_attach_type attach_type)
 9032{
 9033	const char *prefix;
 9034	int kind;
 9035
 9036	btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
 9037	return find_btf_by_prefix_kind(btf, prefix, name, kind);
 9038}
 9039
 9040int libbpf_find_vmlinux_btf_id(const char *name,
 9041			       enum bpf_attach_type attach_type)
 9042{
 9043	struct btf *btf;
 9044	int err;
 9045
 9046	btf = btf__load_vmlinux_btf();
 9047	err = libbpf_get_error(btf);
 9048	if (err) {
 9049		pr_warn("vmlinux BTF is not found\n");
 9050		return libbpf_err(err);
 9051	}
 9052
 9053	err = find_attach_btf_id(btf, name, attach_type);
 9054	if (err <= 0)
 9055		pr_warn("%s is not found in vmlinux BTF\n", name);
 9056
 9057	btf__free(btf);
 9058	return libbpf_err(err);
 
 
 9059}
 9060
 9061static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
 9062{
 9063	struct bpf_prog_info info;
 9064	__u32 info_len = sizeof(info);
 9065	struct btf *btf;
 9066	int err;
 9067
 9068	memset(&info, 0, info_len);
 9069	err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
 9070	if (err) {
 9071		pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
 9072			attach_prog_fd, err);
 9073		return err;
 9074	}
 9075
 9076	err = -EINVAL;
 9077	if (!info.btf_id) {
 9078		pr_warn("The target program doesn't have BTF\n");
 9079		goto out;
 9080	}
 9081	btf = btf__load_from_kernel_by_id(info.btf_id);
 9082	err = libbpf_get_error(btf);
 9083	if (err) {
 9084		pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
 9085		goto out;
 9086	}
 9087	err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
 9088	btf__free(btf);
 9089	if (err <= 0) {
 9090		pr_warn("%s is not found in prog's BTF\n", name);
 9091		goto out;
 9092	}
 9093out:
 9094	return err;
 9095}
 9096
 9097static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
 9098			      enum bpf_attach_type attach_type,
 9099			      int *btf_obj_fd, int *btf_type_id)
 9100{
 9101	int ret, i;
 9102
 9103	ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
 9104	if (ret > 0) {
 9105		*btf_obj_fd = 0; /* vmlinux BTF */
 9106		*btf_type_id = ret;
 9107		return 0;
 9108	}
 9109	if (ret != -ENOENT)
 9110		return ret;
 9111
 9112	ret = load_module_btfs(obj);
 9113	if (ret)
 9114		return ret;
 9115
 9116	for (i = 0; i < obj->btf_module_cnt; i++) {
 9117		const struct module_btf *mod = &obj->btf_modules[i];
 9118
 9119		ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
 9120		if (ret > 0) {
 9121			*btf_obj_fd = mod->fd;
 9122			*btf_type_id = ret;
 9123			return 0;
 9124		}
 9125		if (ret == -ENOENT)
 9126			continue;
 9127
 9128		return ret;
 9129	}
 9130
 9131	return -ESRCH;
 9132}
 9133
 9134static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
 9135				     int *btf_obj_fd, int *btf_type_id)
 9136{
 9137	enum bpf_attach_type attach_type = prog->expected_attach_type;
 9138	__u32 attach_prog_fd = prog->attach_prog_fd;
 9139	int err = 0;
 9140
 9141	/* BPF program's BTF ID */
 9142	if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
 9143		if (!attach_prog_fd) {
 9144			pr_warn("prog '%s': attach program FD is not set\n", prog->name);
 9145			return -EINVAL;
 9146		}
 9147		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
 9148		if (err < 0) {
 9149			pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
 9150				 prog->name, attach_prog_fd, attach_name, err);
 9151			return err;
 9152		}
 9153		*btf_obj_fd = 0;
 9154		*btf_type_id = err;
 9155		return 0;
 9156	}
 9157
 9158	/* kernel/module BTF ID */
 9159	if (prog->obj->gen_loader) {
 9160		bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
 9161		*btf_obj_fd = 0;
 9162		*btf_type_id = 1;
 9163	} else {
 9164		err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
 9165	}
 9166	if (err) {
 9167		pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
 9168			prog->name, attach_name, err);
 9169		return err;
 9170	}
 9171	return 0;
 9172}
 9173
 9174int libbpf_attach_type_by_name(const char *name,
 9175			       enum bpf_attach_type *attach_type)
 9176{
 9177	char *type_names;
 9178	const struct bpf_sec_def *sec_def;
 9179
 9180	if (!name)
 9181		return libbpf_err(-EINVAL);
 9182
 9183	sec_def = find_sec_def(name);
 9184	if (!sec_def) {
 9185		pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
 9186		type_names = libbpf_get_type_names(true);
 9187		if (type_names != NULL) {
 9188			pr_debug("attachable section(type) names are:%s\n", type_names);
 9189			free(type_names);
 9190		}
 9191
 9192		return libbpf_err(-EINVAL);
 9193	}
 9194
 9195	if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
 9196		return libbpf_err(-EINVAL);
 9197	if (!(sec_def->cookie & SEC_ATTACHABLE))
 9198		return libbpf_err(-EINVAL);
 9199
 9200	*attach_type = sec_def->expected_attach_type;
 9201	return 0;
 9202}
 9203
 9204int bpf_map__fd(const struct bpf_map *map)
 9205{
 9206	return map ? map->fd : libbpf_err(-EINVAL);
 9207}
 9208
 9209static bool map_uses_real_name(const struct bpf_map *map)
 9210{
 9211	/* Since libbpf started to support custom .data.* and .rodata.* maps,
 9212	 * their user-visible name differs from kernel-visible name. Users see
 9213	 * such map's corresponding ELF section name as a map name.
 9214	 * This check distinguishes .data/.rodata from .data.* and .rodata.*
 9215	 * maps to know which name has to be returned to the user.
 9216	 */
 9217	if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
 9218		return true;
 9219	if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
 9220		return true;
 9221	return false;
 9222}
 9223
 9224const char *bpf_map__name(const struct bpf_map *map)
 9225{
 9226	if (!map)
 9227		return NULL;
 9228
 9229	if (map_uses_real_name(map))
 9230		return map->real_name;
 9231
 9232	return map->name;
 9233}
 9234
 9235enum bpf_map_type bpf_map__type(const struct bpf_map *map)
 9236{
 9237	return map->def.type;
 9238}
 9239
 9240int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
 9241{
 9242	if (map->fd >= 0)
 9243		return libbpf_err(-EBUSY);
 9244	map->def.type = type;
 9245	return 0;
 9246}
 9247
 9248__u32 bpf_map__map_flags(const struct bpf_map *map)
 9249{
 9250	return map->def.map_flags;
 9251}
 9252
 9253int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
 9254{
 9255	if (map->fd >= 0)
 9256		return libbpf_err(-EBUSY);
 9257	map->def.map_flags = flags;
 9258	return 0;
 9259}
 9260
 9261__u64 bpf_map__map_extra(const struct bpf_map *map)
 9262{
 9263	return map->map_extra;
 9264}
 9265
 9266int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
 9267{
 9268	if (map->fd >= 0)
 9269		return libbpf_err(-EBUSY);
 9270	map->map_extra = map_extra;
 9271	return 0;
 9272}
 9273
 9274__u32 bpf_map__numa_node(const struct bpf_map *map)
 9275{
 9276	return map->numa_node;
 9277}
 9278
 9279int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
 9280{
 9281	if (map->fd >= 0)
 9282		return libbpf_err(-EBUSY);
 9283	map->numa_node = numa_node;
 9284	return 0;
 9285}
 9286
 9287__u32 bpf_map__key_size(const struct bpf_map *map)
 9288{
 9289	return map->def.key_size;
 9290}
 9291
 9292int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
 9293{
 9294	if (map->fd >= 0)
 9295		return libbpf_err(-EBUSY);
 9296	map->def.key_size = size;
 9297	return 0;
 9298}
 9299
 9300__u32 bpf_map__value_size(const struct bpf_map *map)
 9301{
 9302	return map->def.value_size;
 9303}
 9304
 9305int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
 9306{
 9307	if (map->fd >= 0)
 9308		return libbpf_err(-EBUSY);
 9309	map->def.value_size = size;
 9310	return 0;
 9311}
 9312
 9313__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
 9314{
 9315	return map ? map->btf_key_type_id : 0;
 9316}
 9317
 9318__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
 9319{
 9320	return map ? map->btf_value_type_id : 0;
 9321}
 9322
 9323int bpf_map__set_initial_value(struct bpf_map *map,
 9324			       const void *data, size_t size)
 9325{
 9326	if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
 9327	    size != map->def.value_size || map->fd >= 0)
 9328		return libbpf_err(-EINVAL);
 9329
 9330	memcpy(map->mmaped, data, size);
 
 9331	return 0;
 9332}
 9333
 9334const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
 9335{
 9336	if (!map->mmaped)
 9337		return NULL;
 9338	*psize = map->def.value_size;
 9339	return map->mmaped;
 9340}
 9341
 9342bool bpf_map__is_internal(const struct bpf_map *map)
 9343{
 9344	return map->libbpf_type != LIBBPF_MAP_UNSPEC;
 9345}
 9346
 9347__u32 bpf_map__ifindex(const struct bpf_map *map)
 9348{
 9349	return map->map_ifindex;
 9350}
 9351
 9352int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
 9353{
 9354	if (map->fd >= 0)
 9355		return libbpf_err(-EBUSY);
 9356	map->map_ifindex = ifindex;
 9357	return 0;
 9358}
 9359
 9360int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
 9361{
 9362	if (!bpf_map_type__is_map_in_map(map->def.type)) {
 9363		pr_warn("error: unsupported map type\n");
 9364		return libbpf_err(-EINVAL);
 9365	}
 9366	if (map->inner_map_fd != -1) {
 9367		pr_warn("error: inner_map_fd already specified\n");
 9368		return libbpf_err(-EINVAL);
 9369	}
 9370	if (map->inner_map) {
 9371		bpf_map__destroy(map->inner_map);
 9372		zfree(&map->inner_map);
 9373	}
 9374	map->inner_map_fd = fd;
 9375	return 0;
 9376}
 9377
 9378static struct bpf_map *
 9379__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
 9380{
 9381	ssize_t idx;
 9382	struct bpf_map *s, *e;
 9383
 9384	if (!obj || !obj->maps)
 9385		return errno = EINVAL, NULL;
 9386
 9387	s = obj->maps;
 9388	e = obj->maps + obj->nr_maps;
 9389
 9390	if ((m < s) || (m >= e)) {
 9391		pr_warn("error in %s: map handler doesn't belong to object\n",
 9392			 __func__);
 9393		return errno = EINVAL, NULL;
 9394	}
 9395
 9396	idx = (m - obj->maps) + i;
 9397	if (idx >= obj->nr_maps || idx < 0)
 9398		return NULL;
 9399	return &obj->maps[idx];
 9400}
 9401
 9402struct bpf_map *
 9403bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
 9404{
 9405	if (prev == NULL)
 9406		return obj->maps;
 9407
 9408	return __bpf_map__iter(prev, obj, 1);
 9409}
 9410
 9411struct bpf_map *
 9412bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
 9413{
 9414	if (next == NULL) {
 9415		if (!obj->nr_maps)
 9416			return NULL;
 9417		return obj->maps + obj->nr_maps - 1;
 9418	}
 9419
 9420	return __bpf_map__iter(next, obj, -1);
 
 
 
 9421}
 9422
 9423struct bpf_map *
 9424bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
 9425{
 9426	struct bpf_map *pos;
 9427
 9428	bpf_object__for_each_map(pos, obj) {
 9429		/* if it's a special internal map name (which always starts
 9430		 * with dot) then check if that special name matches the
 9431		 * real map name (ELF section name)
 9432		 */
 9433		if (name[0] == '.') {
 9434			if (pos->real_name && strcmp(pos->real_name, name) == 0)
 9435				return pos;
 9436			continue;
 9437		}
 9438		/* otherwise map name has to be an exact match */
 9439		if (map_uses_real_name(pos)) {
 9440			if (strcmp(pos->real_name, name) == 0)
 9441				return pos;
 9442			continue;
 9443		}
 9444		if (strcmp(pos->name, name) == 0)
 9445			return pos;
 9446	}
 9447	return errno = ENOENT, NULL;
 9448}
 9449
 9450int
 9451bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
 9452{
 9453	return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
 9454}
 9455
 9456static int validate_map_op(const struct bpf_map *map, size_t key_sz,
 9457			   size_t value_sz, bool check_value_sz)
 9458{
 9459	if (map->fd <= 0)
 9460		return -ENOENT;
 9461
 9462	if (map->def.key_size != key_sz) {
 9463		pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
 9464			map->name, key_sz, map->def.key_size);
 9465		return -EINVAL;
 9466	}
 9467
 9468	if (!check_value_sz)
 9469		return 0;
 9470
 9471	switch (map->def.type) {
 9472	case BPF_MAP_TYPE_PERCPU_ARRAY:
 9473	case BPF_MAP_TYPE_PERCPU_HASH:
 9474	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
 9475	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
 9476		int num_cpu = libbpf_num_possible_cpus();
 9477		size_t elem_sz = roundup(map->def.value_size, 8);
 9478
 9479		if (value_sz != num_cpu * elem_sz) {
 9480			pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
 9481				map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
 9482			return -EINVAL;
 9483		}
 9484		break;
 9485	}
 9486	default:
 9487		if (map->def.value_size != value_sz) {
 9488			pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
 9489				map->name, value_sz, map->def.value_size);
 9490			return -EINVAL;
 9491		}
 9492		break;
 9493	}
 9494	return 0;
 9495}
 9496
 9497int bpf_map__lookup_elem(const struct bpf_map *map,
 9498			 const void *key, size_t key_sz,
 9499			 void *value, size_t value_sz, __u64 flags)
 9500{
 9501	int err;
 9502
 9503	err = validate_map_op(map, key_sz, value_sz, true);
 9504	if (err)
 9505		return libbpf_err(err);
 9506
 9507	return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
 9508}
 9509
 9510int bpf_map__update_elem(const struct bpf_map *map,
 9511			 const void *key, size_t key_sz,
 9512			 const void *value, size_t value_sz, __u64 flags)
 9513{
 9514	int err;
 9515
 9516	err = validate_map_op(map, key_sz, value_sz, true);
 9517	if (err)
 9518		return libbpf_err(err);
 9519
 9520	return bpf_map_update_elem(map->fd, key, value, flags);
 9521}
 9522
 9523int bpf_map__delete_elem(const struct bpf_map *map,
 9524			 const void *key, size_t key_sz, __u64 flags)
 9525{
 9526	int err;
 9527
 9528	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
 9529	if (err)
 9530		return libbpf_err(err);
 9531
 9532	return bpf_map_delete_elem_flags(map->fd, key, flags);
 9533}
 9534
 9535int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
 9536				    const void *key, size_t key_sz,
 9537				    void *value, size_t value_sz, __u64 flags)
 9538{
 9539	int err;
 9540
 9541	err = validate_map_op(map, key_sz, value_sz, true);
 9542	if (err)
 9543		return libbpf_err(err);
 9544
 9545	return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
 9546}
 9547
 9548int bpf_map__get_next_key(const struct bpf_map *map,
 9549			  const void *cur_key, void *next_key, size_t key_sz)
 9550{
 9551	int err;
 9552
 9553	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
 9554	if (err)
 9555		return libbpf_err(err);
 9556
 9557	return bpf_map_get_next_key(map->fd, cur_key, next_key);
 9558}
 9559
 9560long libbpf_get_error(const void *ptr)
 9561{
 9562	if (!IS_ERR_OR_NULL(ptr))
 9563		return 0;
 9564
 9565	if (IS_ERR(ptr))
 9566		errno = -PTR_ERR(ptr);
 9567
 9568	/* If ptr == NULL, then errno should be already set by the failing
 9569	 * API, because libbpf never returns NULL on success and it now always
 9570	 * sets errno on error. So no extra errno handling for ptr == NULL
 9571	 * case.
 9572	 */
 9573	return -errno;
 9574}
 9575
 9576/* Replace link's underlying BPF program with the new one */
 9577int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
 9578{
 9579	int ret;
 9580
 9581	ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
 9582	return libbpf_err_errno(ret);
 9583}
 9584
 9585/* Release "ownership" of underlying BPF resource (typically, BPF program
 9586 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
 9587 * link, when destructed through bpf_link__destroy() call won't attempt to
 9588 * detach/unregisted that BPF resource. This is useful in situations where,
 9589 * say, attached BPF program has to outlive userspace program that attached it
 9590 * in the system. Depending on type of BPF program, though, there might be
 9591 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
 9592 * exit of userspace program doesn't trigger automatic detachment and clean up
 9593 * inside the kernel.
 9594 */
 9595void bpf_link__disconnect(struct bpf_link *link)
 9596{
 9597	link->disconnected = true;
 9598}
 9599
 9600int bpf_link__destroy(struct bpf_link *link)
 9601{
 9602	int err = 0;
 9603
 9604	if (IS_ERR_OR_NULL(link))
 9605		return 0;
 9606
 9607	if (!link->disconnected && link->detach)
 9608		err = link->detach(link);
 9609	if (link->pin_path)
 9610		free(link->pin_path);
 9611	if (link->dealloc)
 9612		link->dealloc(link);
 9613	else
 9614		free(link);
 9615
 9616	return libbpf_err(err);
 9617}
 9618
 9619int bpf_link__fd(const struct bpf_link *link)
 9620{
 9621	return link->fd;
 9622}
 9623
 9624const char *bpf_link__pin_path(const struct bpf_link *link)
 9625{
 9626	return link->pin_path;
 9627}
 9628
 9629static int bpf_link__detach_fd(struct bpf_link *link)
 9630{
 9631	return libbpf_err_errno(close(link->fd));
 9632}
 9633
 9634struct bpf_link *bpf_link__open(const char *path)
 9635{
 9636	struct bpf_link *link;
 9637	int fd;
 9638
 9639	fd = bpf_obj_get(path);
 9640	if (fd < 0) {
 9641		fd = -errno;
 9642		pr_warn("failed to open link at %s: %d\n", path, fd);
 9643		return libbpf_err_ptr(fd);
 9644	}
 9645
 9646	link = calloc(1, sizeof(*link));
 9647	if (!link) {
 9648		close(fd);
 9649		return libbpf_err_ptr(-ENOMEM);
 9650	}
 9651	link->detach = &bpf_link__detach_fd;
 9652	link->fd = fd;
 9653
 9654	link->pin_path = strdup(path);
 9655	if (!link->pin_path) {
 9656		bpf_link__destroy(link);
 9657		return libbpf_err_ptr(-ENOMEM);
 9658	}
 9659
 9660	return link;
 9661}
 9662
 9663int bpf_link__detach(struct bpf_link *link)
 9664{
 9665	return bpf_link_detach(link->fd) ? -errno : 0;
 9666}
 9667
 9668int bpf_link__pin(struct bpf_link *link, const char *path)
 9669{
 9670	int err;
 9671
 9672	if (link->pin_path)
 9673		return libbpf_err(-EBUSY);
 9674	err = make_parent_dir(path);
 9675	if (err)
 9676		return libbpf_err(err);
 9677	err = check_path(path);
 9678	if (err)
 9679		return libbpf_err(err);
 9680
 9681	link->pin_path = strdup(path);
 9682	if (!link->pin_path)
 9683		return libbpf_err(-ENOMEM);
 9684
 9685	if (bpf_obj_pin(link->fd, link->pin_path)) {
 9686		err = -errno;
 9687		zfree(&link->pin_path);
 9688		return libbpf_err(err);
 9689	}
 9690
 9691	pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
 9692	return 0;
 9693}
 9694
 9695int bpf_link__unpin(struct bpf_link *link)
 9696{
 9697	int err;
 9698
 9699	if (!link->pin_path)
 9700		return libbpf_err(-EINVAL);
 9701
 9702	err = unlink(link->pin_path);
 9703	if (err != 0)
 9704		return -errno;
 9705
 9706	pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
 9707	zfree(&link->pin_path);
 9708	return 0;
 9709}
 9710
 9711struct bpf_link_perf {
 9712	struct bpf_link link;
 9713	int perf_event_fd;
 9714	/* legacy kprobe support: keep track of probe identifier and type */
 9715	char *legacy_probe_name;
 9716	bool legacy_is_kprobe;
 9717	bool legacy_is_retprobe;
 9718};
 9719
 9720static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
 9721static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
 9722
 9723static int bpf_link_perf_detach(struct bpf_link *link)
 9724{
 9725	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
 9726	int err = 0;
 9727
 9728	if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
 9729		err = -errno;
 9730
 9731	if (perf_link->perf_event_fd != link->fd)
 9732		close(perf_link->perf_event_fd);
 9733	close(link->fd);
 9734
 9735	/* legacy uprobe/kprobe needs to be removed after perf event fd closure */
 9736	if (perf_link->legacy_probe_name) {
 9737		if (perf_link->legacy_is_kprobe) {
 9738			err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
 9739							 perf_link->legacy_is_retprobe);
 9740		} else {
 9741			err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
 9742							 perf_link->legacy_is_retprobe);
 9743		}
 9744	}
 9745
 9746	return err;
 9747}
 9748
 9749static void bpf_link_perf_dealloc(struct bpf_link *link)
 9750{
 9751	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
 9752
 9753	free(perf_link->legacy_probe_name);
 9754	free(perf_link);
 9755}
 9756
 9757struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
 9758						     const struct bpf_perf_event_opts *opts)
 9759{
 9760	char errmsg[STRERR_BUFSIZE];
 9761	struct bpf_link_perf *link;
 9762	int prog_fd, link_fd = -1, err;
 9763
 9764	if (!OPTS_VALID(opts, bpf_perf_event_opts))
 9765		return libbpf_err_ptr(-EINVAL);
 9766
 9767	if (pfd < 0) {
 9768		pr_warn("prog '%s': invalid perf event FD %d\n",
 9769			prog->name, pfd);
 9770		return libbpf_err_ptr(-EINVAL);
 9771	}
 9772	prog_fd = bpf_program__fd(prog);
 9773	if (prog_fd < 0) {
 9774		pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
 9775			prog->name);
 9776		return libbpf_err_ptr(-EINVAL);
 9777	}
 9778
 9779	link = calloc(1, sizeof(*link));
 9780	if (!link)
 9781		return libbpf_err_ptr(-ENOMEM);
 9782	link->link.detach = &bpf_link_perf_detach;
 9783	link->link.dealloc = &bpf_link_perf_dealloc;
 9784	link->perf_event_fd = pfd;
 9785
 9786	if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
 9787		DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
 9788			.perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
 9789
 9790		link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
 9791		if (link_fd < 0) {
 9792			err = -errno;
 9793			pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
 9794				prog->name, pfd,
 9795				err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
 9796			goto err_out;
 9797		}
 9798		link->link.fd = link_fd;
 9799	} else {
 9800		if (OPTS_GET(opts, bpf_cookie, 0)) {
 9801			pr_warn("prog '%s': user context value is not supported\n", prog->name);
 9802			err = -EOPNOTSUPP;
 9803			goto err_out;
 9804		}
 9805
 9806		if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
 9807			err = -errno;
 9808			pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
 9809				prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
 9810			if (err == -EPROTO)
 9811				pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
 9812					prog->name, pfd);
 9813			goto err_out;
 9814		}
 9815		link->link.fd = pfd;
 9816	}
 9817	if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
 9818		err = -errno;
 9819		pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
 9820			prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
 9821		goto err_out;
 9822	}
 9823
 9824	return &link->link;
 9825err_out:
 9826	if (link_fd >= 0)
 9827		close(link_fd);
 9828	free(link);
 9829	return libbpf_err_ptr(err);
 9830}
 9831
 9832struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
 9833{
 9834	return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
 9835}
 9836
 9837/*
 9838 * this function is expected to parse integer in the range of [0, 2^31-1] from
 9839 * given file using scanf format string fmt. If actual parsed value is
 9840 * negative, the result might be indistinguishable from error
 9841 */
 9842static int parse_uint_from_file(const char *file, const char *fmt)
 9843{
 9844	char buf[STRERR_BUFSIZE];
 9845	int err, ret;
 9846	FILE *f;
 9847
 9848	f = fopen(file, "r");
 9849	if (!f) {
 9850		err = -errno;
 9851		pr_debug("failed to open '%s': %s\n", file,
 9852			 libbpf_strerror_r(err, buf, sizeof(buf)));
 9853		return err;
 9854	}
 9855	err = fscanf(f, fmt, &ret);
 9856	if (err != 1) {
 9857		err = err == EOF ? -EIO : -errno;
 9858		pr_debug("failed to parse '%s': %s\n", file,
 9859			libbpf_strerror_r(err, buf, sizeof(buf)));
 9860		fclose(f);
 9861		return err;
 9862	}
 9863	fclose(f);
 9864	return ret;
 9865}
 9866
 9867static int determine_kprobe_perf_type(void)
 9868{
 9869	const char *file = "/sys/bus/event_source/devices/kprobe/type";
 9870
 9871	return parse_uint_from_file(file, "%d\n");
 9872}
 9873
 9874static int determine_uprobe_perf_type(void)
 9875{
 9876	const char *file = "/sys/bus/event_source/devices/uprobe/type";
 9877
 9878	return parse_uint_from_file(file, "%d\n");
 9879}
 9880
 9881static int determine_kprobe_retprobe_bit(void)
 9882{
 9883	const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
 9884
 9885	return parse_uint_from_file(file, "config:%d\n");
 9886}
 9887
 9888static int determine_uprobe_retprobe_bit(void)
 9889{
 9890	const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
 9891
 9892	return parse_uint_from_file(file, "config:%d\n");
 9893}
 9894
 9895#define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
 9896#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
 9897
 9898static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
 9899				 uint64_t offset, int pid, size_t ref_ctr_off)
 9900{
 9901	const size_t attr_sz = sizeof(struct perf_event_attr);
 9902	struct perf_event_attr attr;
 9903	char errmsg[STRERR_BUFSIZE];
 9904	int type, pfd;
 9905
 9906	if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
 9907		return -EINVAL;
 9908
 9909	memset(&attr, 0, attr_sz);
 9910
 9911	type = uprobe ? determine_uprobe_perf_type()
 9912		      : determine_kprobe_perf_type();
 9913	if (type < 0) {
 9914		pr_warn("failed to determine %s perf type: %s\n",
 9915			uprobe ? "uprobe" : "kprobe",
 9916			libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
 9917		return type;
 9918	}
 9919	if (retprobe) {
 9920		int bit = uprobe ? determine_uprobe_retprobe_bit()
 9921				 : determine_kprobe_retprobe_bit();
 9922
 9923		if (bit < 0) {
 9924			pr_warn("failed to determine %s retprobe bit: %s\n",
 9925				uprobe ? "uprobe" : "kprobe",
 9926				libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
 9927			return bit;
 9928		}
 9929		attr.config |= 1 << bit;
 9930	}
 9931	attr.size = attr_sz;
 9932	attr.type = type;
 9933	attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
 9934	attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
 9935	attr.config2 = offset;		 /* kprobe_addr or probe_offset */
 9936
 9937	/* pid filter is meaningful only for uprobes */
 9938	pfd = syscall(__NR_perf_event_open, &attr,
 9939		      pid < 0 ? -1 : pid /* pid */,
 9940		      pid == -1 ? 0 : -1 /* cpu */,
 9941		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
 9942	return pfd >= 0 ? pfd : -errno;
 9943}
 9944
 9945static int append_to_file(const char *file, const char *fmt, ...)
 9946{
 9947	int fd, n, err = 0;
 9948	va_list ap;
 9949
 9950	fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
 9951	if (fd < 0)
 9952		return -errno;
 9953
 9954	va_start(ap, fmt);
 9955	n = vdprintf(fd, fmt, ap);
 9956	va_end(ap);
 9957
 9958	if (n < 0)
 9959		err = -errno;
 9960
 9961	close(fd);
 9962	return err;
 9963}
 9964
 9965#define DEBUGFS "/sys/kernel/debug/tracing"
 9966#define TRACEFS "/sys/kernel/tracing"
 9967
 9968static bool use_debugfs(void)
 9969{
 9970	static int has_debugfs = -1;
 9971
 9972	if (has_debugfs < 0)
 9973		has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
 9974
 9975	return has_debugfs == 1;
 9976}
 9977
 9978static const char *tracefs_path(void)
 9979{
 9980	return use_debugfs() ? DEBUGFS : TRACEFS;
 9981}
 9982
 9983static const char *tracefs_kprobe_events(void)
 9984{
 9985	return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
 9986}
 9987
 9988static const char *tracefs_uprobe_events(void)
 9989{
 9990	return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
 9991}
 9992
 9993static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
 9994					 const char *kfunc_name, size_t offset)
 9995{
 9996	static int index = 0;
 9997
 9998	snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
 9999		 __sync_fetch_and_add(&index, 1));
10000}
10001
10002static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
10003				   const char *kfunc_name, size_t offset)
10004{
10005	return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
10006			      retprobe ? 'r' : 'p',
10007			      retprobe ? "kretprobes" : "kprobes",
10008			      probe_name, kfunc_name, offset);
10009}
10010
10011static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
10012{
10013	return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
10014			      retprobe ? "kretprobes" : "kprobes", probe_name);
10015}
10016
10017static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10018{
10019	char file[256];
10020
10021	snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10022		 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
10023
10024	return parse_uint_from_file(file, "%d\n");
10025}
10026
10027static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
10028					 const char *kfunc_name, size_t offset, int pid)
10029{
10030	const size_t attr_sz = sizeof(struct perf_event_attr);
10031	struct perf_event_attr attr;
10032	char errmsg[STRERR_BUFSIZE];
10033	int type, pfd, err;
10034
10035	err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
10036	if (err < 0) {
10037		pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
10038			kfunc_name, offset,
10039			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10040		return err;
10041	}
10042	type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
10043	if (type < 0) {
10044		err = type;
10045		pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
10046			kfunc_name, offset,
10047			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10048		goto err_clean_legacy;
10049	}
10050
10051	memset(&attr, 0, attr_sz);
10052	attr.size = attr_sz;
10053	attr.config = type;
10054	attr.type = PERF_TYPE_TRACEPOINT;
10055
10056	pfd = syscall(__NR_perf_event_open, &attr,
10057		      pid < 0 ? -1 : pid, /* pid */
10058		      pid == -1 ? 0 : -1, /* cpu */
10059		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10060	if (pfd < 0) {
10061		err = -errno;
10062		pr_warn("legacy kprobe perf_event_open() failed: %s\n",
10063			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10064		goto err_clean_legacy;
10065	}
10066	return pfd;
10067
10068err_clean_legacy:
10069	/* Clear the newly added legacy kprobe_event */
10070	remove_kprobe_event_legacy(probe_name, retprobe);
10071	return err;
10072}
10073
10074static const char *arch_specific_syscall_pfx(void)
10075{
10076#if defined(__x86_64__)
10077	return "x64";
10078#elif defined(__i386__)
10079	return "ia32";
10080#elif defined(__s390x__)
10081	return "s390x";
10082#elif defined(__s390__)
10083	return "s390";
10084#elif defined(__arm__)
10085	return "arm";
10086#elif defined(__aarch64__)
10087	return "arm64";
10088#elif defined(__mips__)
10089	return "mips";
10090#elif defined(__riscv)
10091	return "riscv";
10092#elif defined(__powerpc__)
10093	return "powerpc";
10094#elif defined(__powerpc64__)
10095	return "powerpc64";
10096#else
10097	return NULL;
10098#endif
10099}
10100
10101static int probe_kern_syscall_wrapper(void)
10102{
10103	char syscall_name[64];
10104	const char *ksys_pfx;
10105
10106	ksys_pfx = arch_specific_syscall_pfx();
10107	if (!ksys_pfx)
10108		return 0;
10109
10110	snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
10111
10112	if (determine_kprobe_perf_type() >= 0) {
10113		int pfd;
10114
10115		pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
10116		if (pfd >= 0)
10117			close(pfd);
10118
10119		return pfd >= 0 ? 1 : 0;
10120	} else { /* legacy mode */
10121		char probe_name[128];
10122
10123		gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
10124		if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
10125			return 0;
10126
10127		(void)remove_kprobe_event_legacy(probe_name, false);
10128		return 1;
10129	}
10130}
10131
10132struct bpf_link *
10133bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10134				const char *func_name,
10135				const struct bpf_kprobe_opts *opts)
10136{
10137	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10138	char errmsg[STRERR_BUFSIZE];
10139	char *legacy_probe = NULL;
10140	struct bpf_link *link;
10141	size_t offset;
10142	bool retprobe, legacy;
10143	int pfd, err;
10144
10145	if (!OPTS_VALID(opts, bpf_kprobe_opts))
10146		return libbpf_err_ptr(-EINVAL);
10147
10148	retprobe = OPTS_GET(opts, retprobe, false);
10149	offset = OPTS_GET(opts, offset, 0);
10150	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10151
10152	legacy = determine_kprobe_perf_type() < 0;
10153	if (!legacy) {
10154		pfd = perf_event_open_probe(false /* uprobe */, retprobe,
10155					    func_name, offset,
10156					    -1 /* pid */, 0 /* ref_ctr_off */);
10157	} else {
10158		char probe_name[256];
10159
10160		gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
10161					     func_name, offset);
10162
10163		legacy_probe = strdup(probe_name);
10164		if (!legacy_probe)
10165			return libbpf_err_ptr(-ENOMEM);
10166
10167		pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
10168						    offset, -1 /* pid */);
10169	}
10170	if (pfd < 0) {
10171		err = -errno;
10172		pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10173			prog->name, retprobe ? "kretprobe" : "kprobe",
10174			func_name, offset,
10175			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10176		goto err_out;
10177	}
10178	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10179	err = libbpf_get_error(link);
10180	if (err) {
10181		close(pfd);
10182		pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10183			prog->name, retprobe ? "kretprobe" : "kprobe",
10184			func_name, offset,
10185			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10186		goto err_clean_legacy;
10187	}
10188	if (legacy) {
10189		struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10190
10191		perf_link->legacy_probe_name = legacy_probe;
10192		perf_link->legacy_is_kprobe = true;
10193		perf_link->legacy_is_retprobe = retprobe;
10194	}
10195
10196	return link;
10197
10198err_clean_legacy:
10199	if (legacy)
10200		remove_kprobe_event_legacy(legacy_probe, retprobe);
10201err_out:
10202	free(legacy_probe);
10203	return libbpf_err_ptr(err);
10204}
10205
10206struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10207					    bool retprobe,
10208					    const char *func_name)
10209{
10210	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
10211		.retprobe = retprobe,
10212	);
10213
10214	return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10215}
10216
10217struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10218					      const char *syscall_name,
10219					      const struct bpf_ksyscall_opts *opts)
10220{
10221	LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
10222	char func_name[128];
10223
10224	if (!OPTS_VALID(opts, bpf_ksyscall_opts))
10225		return libbpf_err_ptr(-EINVAL);
10226
10227	if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10228		/* arch_specific_syscall_pfx() should never return NULL here
10229		 * because it is guarded by kernel_supports(). However, since
10230		 * compiler does not know that we have an explicit conditional
10231		 * as well.
10232		 */
10233		snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
10234			 arch_specific_syscall_pfx() ? : "", syscall_name);
10235	} else {
10236		snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
10237	}
10238
10239	kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
10240	kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10241
10242	return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10243}
10244
10245/* Adapted from perf/util/string.c */
10246static bool glob_match(const char *str, const char *pat)
10247{
10248	while (*str && *pat && *pat != '*') {
10249		if (*pat == '?') {      /* Matches any single character */
10250			str++;
10251			pat++;
10252			continue;
10253		}
10254		if (*str != *pat)
10255			return false;
10256		str++;
10257		pat++;
10258	}
10259	/* Check wild card */
10260	if (*pat == '*') {
10261		while (*pat == '*')
10262			pat++;
10263		if (!*pat) /* Tail wild card matches all */
10264			return true;
10265		while (*str)
10266			if (glob_match(str++, pat))
10267				return true;
10268	}
10269	return !*str && !*pat;
10270}
10271
10272struct kprobe_multi_resolve {
10273	const char *pattern;
10274	unsigned long *addrs;
10275	size_t cap;
10276	size_t cnt;
10277};
10278
10279static int
10280resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
10281			const char *sym_name, void *ctx)
10282{
10283	struct kprobe_multi_resolve *res = ctx;
10284	int err;
10285
10286	if (!glob_match(sym_name, res->pattern))
10287		return 0;
10288
10289	err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10290				res->cnt + 1);
10291	if (err)
10292		return err;
10293
10294	res->addrs[res->cnt++] = (unsigned long) sym_addr;
10295	return 0;
10296}
10297
10298struct bpf_link *
10299bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10300				      const char *pattern,
10301				      const struct bpf_kprobe_multi_opts *opts)
10302{
10303	LIBBPF_OPTS(bpf_link_create_opts, lopts);
10304	struct kprobe_multi_resolve res = {
10305		.pattern = pattern,
10306	};
10307	struct bpf_link *link = NULL;
10308	char errmsg[STRERR_BUFSIZE];
10309	const unsigned long *addrs;
10310	int err, link_fd, prog_fd;
10311	const __u64 *cookies;
10312	const char **syms;
10313	bool retprobe;
10314	size_t cnt;
10315
10316	if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
10317		return libbpf_err_ptr(-EINVAL);
10318
10319	syms    = OPTS_GET(opts, syms, false);
10320	addrs   = OPTS_GET(opts, addrs, false);
10321	cnt     = OPTS_GET(opts, cnt, false);
10322	cookies = OPTS_GET(opts, cookies, false);
10323
10324	if (!pattern && !addrs && !syms)
10325		return libbpf_err_ptr(-EINVAL);
10326	if (pattern && (addrs || syms || cookies || cnt))
10327		return libbpf_err_ptr(-EINVAL);
10328	if (!pattern && !cnt)
10329		return libbpf_err_ptr(-EINVAL);
10330	if (addrs && syms)
10331		return libbpf_err_ptr(-EINVAL);
10332
10333	if (pattern) {
10334		err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
10335		if (err)
10336			goto error;
10337		if (!res.cnt) {
10338			err = -ENOENT;
10339			goto error;
10340		}
10341		addrs = res.addrs;
10342		cnt = res.cnt;
10343	}
10344
10345	retprobe = OPTS_GET(opts, retprobe, false);
10346
10347	lopts.kprobe_multi.syms = syms;
10348	lopts.kprobe_multi.addrs = addrs;
10349	lopts.kprobe_multi.cookies = cookies;
10350	lopts.kprobe_multi.cnt = cnt;
10351	lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
10352
10353	link = calloc(1, sizeof(*link));
10354	if (!link) {
10355		err = -ENOMEM;
10356		goto error;
10357	}
10358	link->detach = &bpf_link__detach_fd;
10359
10360	prog_fd = bpf_program__fd(prog);
10361	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
10362	if (link_fd < 0) {
10363		err = -errno;
10364		pr_warn("prog '%s': failed to attach: %s\n",
10365			prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10366		goto error;
10367	}
10368	link->fd = link_fd;
10369	free(res.addrs);
10370	return link;
10371
10372error:
10373	free(link);
10374	free(res.addrs);
10375	return libbpf_err_ptr(err);
10376}
10377
10378static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10379{
10380	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
10381	unsigned long offset = 0;
10382	const char *func_name;
10383	char *func;
10384	int n;
10385
10386	*link = NULL;
10387
10388	/* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10389	if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10390		return 0;
10391
10392	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10393	if (opts.retprobe)
10394		func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10395	else
10396		func_name = prog->sec_name + sizeof("kprobe/") - 1;
10397
10398	n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10399	if (n < 1) {
10400		pr_warn("kprobe name is invalid: %s\n", func_name);
10401		return -EINVAL;
10402	}
10403	if (opts.retprobe && offset != 0) {
10404		free(func);
10405		pr_warn("kretprobes do not support offset specification\n");
10406		return -EINVAL;
10407	}
10408
10409	opts.offset = offset;
10410	*link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10411	free(func);
10412	return libbpf_get_error(*link);
10413}
10414
10415static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10416{
10417	LIBBPF_OPTS(bpf_ksyscall_opts, opts);
10418	const char *syscall_name;
10419
10420	*link = NULL;
10421
10422	/* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10423	if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10424		return 0;
10425
10426	opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10427	if (opts.retprobe)
10428		syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10429	else
10430		syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10431
10432	*link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10433	return *link ? 0 : -errno;
10434}
10435
10436static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10437{
10438	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
10439	const char *spec;
10440	char *pattern;
10441	int n;
10442
10443	*link = NULL;
10444
10445	/* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10446	if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10447	    strcmp(prog->sec_name, "kretprobe.multi") == 0)
10448		return 0;
10449
10450	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10451	if (opts.retprobe)
10452		spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10453	else
10454		spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10455
10456	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10457	if (n < 1) {
10458		pr_warn("kprobe multi pattern is invalid: %s\n", pattern);
10459		return -EINVAL;
10460	}
10461
10462	*link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10463	free(pattern);
10464	return libbpf_get_error(*link);
10465}
10466
10467static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
10468					 const char *binary_path, uint64_t offset)
10469{
10470	int i;
10471
10472	snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
10473
10474	/* sanitize binary_path in the probe name */
10475	for (i = 0; buf[i]; i++) {
10476		if (!isalnum(buf[i]))
10477			buf[i] = '_';
10478	}
10479}
10480
10481static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
10482					  const char *binary_path, size_t offset)
10483{
10484	return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
10485			      retprobe ? 'r' : 'p',
10486			      retprobe ? "uretprobes" : "uprobes",
10487			      probe_name, binary_path, offset);
10488}
10489
10490static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
10491{
10492	return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10493			      retprobe ? "uretprobes" : "uprobes", probe_name);
10494}
10495
10496static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10497{
10498	char file[512];
10499
10500	snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10501		 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
10502
10503	return parse_uint_from_file(file, "%d\n");
10504}
10505
10506static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
10507					 const char *binary_path, size_t offset, int pid)
10508{
10509	const size_t attr_sz = sizeof(struct perf_event_attr);
10510	struct perf_event_attr attr;
10511	int type, pfd, err;
10512
10513	err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
10514	if (err < 0) {
10515		pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
10516			binary_path, (size_t)offset, err);
10517		return err;
10518	}
10519	type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
10520	if (type < 0) {
10521		err = type;
10522		pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
10523			binary_path, offset, err);
10524		goto err_clean_legacy;
10525	}
10526
10527	memset(&attr, 0, attr_sz);
10528	attr.size = attr_sz;
10529	attr.config = type;
10530	attr.type = PERF_TYPE_TRACEPOINT;
10531
10532	pfd = syscall(__NR_perf_event_open, &attr,
10533		      pid < 0 ? -1 : pid, /* pid */
10534		      pid == -1 ? 0 : -1, /* cpu */
10535		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10536	if (pfd < 0) {
10537		err = -errno;
10538		pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
10539		goto err_clean_legacy;
10540	}
10541	return pfd;
10542
10543err_clean_legacy:
10544	/* Clear the newly added legacy uprobe_event */
10545	remove_uprobe_event_legacy(probe_name, retprobe);
10546	return err;
10547}
10548
10549/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
10550static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
10551{
10552	while ((scn = elf_nextscn(elf, scn)) != NULL) {
10553		GElf_Shdr sh;
10554
10555		if (!gelf_getshdr(scn, &sh))
10556			continue;
10557		if (sh.sh_type == sh_type)
10558			return scn;
10559	}
10560	return NULL;
10561}
10562
10563/* Find offset of function name in object specified by path.  "name" matches
10564 * symbol name or name@@LIB for library functions.
10565 */
10566static long elf_find_func_offset(const char *binary_path, const char *name)
10567{
10568	int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
10569	bool is_shared_lib, is_name_qualified;
10570	char errmsg[STRERR_BUFSIZE];
10571	long ret = -ENOENT;
10572	size_t name_len;
10573	GElf_Ehdr ehdr;
10574	Elf *elf;
10575
10576	fd = open(binary_path, O_RDONLY | O_CLOEXEC);
10577	if (fd < 0) {
10578		ret = -errno;
10579		pr_warn("failed to open %s: %s\n", binary_path,
10580			libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
10581		return ret;
10582	}
10583	elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
10584	if (!elf) {
10585		pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10586		close(fd);
10587		return -LIBBPF_ERRNO__FORMAT;
10588	}
10589	if (!gelf_getehdr(elf, &ehdr)) {
10590		pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10591		ret = -LIBBPF_ERRNO__FORMAT;
10592		goto out;
10593	}
10594	/* for shared lib case, we do not need to calculate relative offset */
10595	is_shared_lib = ehdr.e_type == ET_DYN;
10596
10597	name_len = strlen(name);
10598	/* Does name specify "@@LIB"? */
10599	is_name_qualified = strstr(name, "@@") != NULL;
10600
10601	/* Search SHT_DYNSYM, SHT_SYMTAB for symbol.  This search order is used because if
10602	 * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
10603	 * linked binary may not have SHT_DYMSYM, so absence of a section should not be
10604	 * reported as a warning/error.
10605	 */
10606	for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
10607		size_t nr_syms, strtabidx, idx;
10608		Elf_Data *symbols = NULL;
10609		Elf_Scn *scn = NULL;
10610		int last_bind = -1;
10611		const char *sname;
10612		GElf_Shdr sh;
10613
10614		scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
10615		if (!scn) {
10616			pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
10617				 binary_path);
10618			continue;
10619		}
10620		if (!gelf_getshdr(scn, &sh))
10621			continue;
10622		strtabidx = sh.sh_link;
10623		symbols = elf_getdata(scn, 0);
10624		if (!symbols) {
10625			pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
10626				binary_path, elf_errmsg(-1));
10627			ret = -LIBBPF_ERRNO__FORMAT;
10628			goto out;
10629		}
10630		nr_syms = symbols->d_size / sh.sh_entsize;
10631
10632		for (idx = 0; idx < nr_syms; idx++) {
10633			int curr_bind;
10634			GElf_Sym sym;
10635			Elf_Scn *sym_scn;
10636			GElf_Shdr sym_sh;
10637
10638			if (!gelf_getsym(symbols, idx, &sym))
10639				continue;
10640
10641			if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
10642				continue;
10643
10644			sname = elf_strptr(elf, strtabidx, sym.st_name);
10645			if (!sname)
10646				continue;
10647
10648			curr_bind = GELF_ST_BIND(sym.st_info);
10649
10650			/* User can specify func, func@@LIB or func@@LIB_VERSION. */
10651			if (strncmp(sname, name, name_len) != 0)
10652				continue;
10653			/* ...but we don't want a search for "foo" to match 'foo2" also, so any
10654			 * additional characters in sname should be of the form "@@LIB".
10655			 */
10656			if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
10657				continue;
10658
10659			if (ret >= 0) {
10660				/* handle multiple matches */
10661				if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
10662					/* Only accept one non-weak bind. */
10663					pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
10664						sname, name, binary_path);
10665					ret = -LIBBPF_ERRNO__FORMAT;
10666					goto out;
10667				} else if (curr_bind == STB_WEAK) {
10668					/* already have a non-weak bind, and
10669					 * this is a weak bind, so ignore.
10670					 */
10671					continue;
10672				}
10673			}
10674
10675			/* Transform symbol's virtual address (absolute for
10676			 * binaries and relative for shared libs) into file
10677			 * offset, which is what kernel is expecting for
10678			 * uprobe/uretprobe attachment.
10679			 * See Documentation/trace/uprobetracer.rst for more
10680			 * details.
10681			 * This is done by looking up symbol's containing
10682			 * section's header and using it's virtual address
10683			 * (sh_addr) and corresponding file offset (sh_offset)
10684			 * to transform sym.st_value (virtual address) into
10685			 * desired final file offset.
10686			 */
10687			sym_scn = elf_getscn(elf, sym.st_shndx);
10688			if (!sym_scn)
10689				continue;
10690			if (!gelf_getshdr(sym_scn, &sym_sh))
10691				continue;
10692
10693			ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
10694			last_bind = curr_bind;
10695		}
10696		if (ret > 0)
10697			break;
10698	}
10699
10700	if (ret > 0) {
10701		pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
10702			 ret);
10703	} else {
10704		if (ret == 0) {
10705			pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
10706				is_shared_lib ? "should not be 0 in a shared library" :
10707						"try using shared library path instead");
10708			ret = -ENOENT;
10709		} else {
10710			pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
10711		}
10712	}
10713out:
10714	elf_end(elf);
10715	close(fd);
10716	return ret;
10717}
10718
10719static const char *arch_specific_lib_paths(void)
10720{
10721	/*
10722	 * Based on https://packages.debian.org/sid/libc6.
10723	 *
10724	 * Assume that the traced program is built for the same architecture
10725	 * as libbpf, which should cover the vast majority of cases.
10726	 */
10727#if defined(__x86_64__)
10728	return "/lib/x86_64-linux-gnu";
10729#elif defined(__i386__)
10730	return "/lib/i386-linux-gnu";
10731#elif defined(__s390x__)
10732	return "/lib/s390x-linux-gnu";
10733#elif defined(__s390__)
10734	return "/lib/s390-linux-gnu";
10735#elif defined(__arm__) && defined(__SOFTFP__)
10736	return "/lib/arm-linux-gnueabi";
10737#elif defined(__arm__) && !defined(__SOFTFP__)
10738	return "/lib/arm-linux-gnueabihf";
10739#elif defined(__aarch64__)
10740	return "/lib/aarch64-linux-gnu";
10741#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
10742	return "/lib/mips64el-linux-gnuabi64";
10743#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
10744	return "/lib/mipsel-linux-gnu";
10745#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
10746	return "/lib/powerpc64le-linux-gnu";
10747#elif defined(__sparc__) && defined(__arch64__)
10748	return "/lib/sparc64-linux-gnu";
10749#elif defined(__riscv) && __riscv_xlen == 64
10750	return "/lib/riscv64-linux-gnu";
10751#else
10752	return NULL;
10753#endif
10754}
10755
10756/* Get full path to program/shared library. */
10757static int resolve_full_path(const char *file, char *result, size_t result_sz)
10758{
10759	const char *search_paths[3] = {};
10760	int i, perm;
10761
10762	if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
10763		search_paths[0] = getenv("LD_LIBRARY_PATH");
10764		search_paths[1] = "/usr/lib64:/usr/lib";
10765		search_paths[2] = arch_specific_lib_paths();
10766		perm = R_OK;
10767	} else {
10768		search_paths[0] = getenv("PATH");
10769		search_paths[1] = "/usr/bin:/usr/sbin";
10770		perm = R_OK | X_OK;
10771	}
10772
10773	for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
10774		const char *s;
10775
10776		if (!search_paths[i])
10777			continue;
10778		for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
10779			char *next_path;
10780			int seg_len;
10781
10782			if (s[0] == ':')
10783				s++;
10784			next_path = strchr(s, ':');
10785			seg_len = next_path ? next_path - s : strlen(s);
10786			if (!seg_len)
10787				continue;
10788			snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
10789			/* ensure it has required permissions */
10790			if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
10791				continue;
10792			pr_debug("resolved '%s' to '%s'\n", file, result);
10793			return 0;
10794		}
10795	}
10796	return -ENOENT;
10797}
10798
10799LIBBPF_API struct bpf_link *
10800bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
10801				const char *binary_path, size_t func_offset,
10802				const struct bpf_uprobe_opts *opts)
10803{
10804	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10805	char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
10806	char full_binary_path[PATH_MAX];
10807	struct bpf_link *link;
10808	size_t ref_ctr_off;
10809	int pfd, err;
10810	bool retprobe, legacy;
10811	const char *func_name;
10812
10813	if (!OPTS_VALID(opts, bpf_uprobe_opts))
10814		return libbpf_err_ptr(-EINVAL);
10815
10816	retprobe = OPTS_GET(opts, retprobe, false);
10817	ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
10818	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10819
10820	if (!binary_path)
10821		return libbpf_err_ptr(-EINVAL);
10822
10823	if (!strchr(binary_path, '/')) {
10824		err = resolve_full_path(binary_path, full_binary_path,
10825					sizeof(full_binary_path));
10826		if (err) {
10827			pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10828				prog->name, binary_path, err);
10829			return libbpf_err_ptr(err);
10830		}
10831		binary_path = full_binary_path;
10832	}
10833	func_name = OPTS_GET(opts, func_name, NULL);
10834	if (func_name) {
10835		long sym_off;
10836
10837		sym_off = elf_find_func_offset(binary_path, func_name);
10838		if (sym_off < 0)
10839			return libbpf_err_ptr(sym_off);
10840		func_offset += sym_off;
10841	}
10842
10843	legacy = determine_uprobe_perf_type() < 0;
10844	if (!legacy) {
10845		pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
10846					    func_offset, pid, ref_ctr_off);
10847	} else {
10848		char probe_name[PATH_MAX + 64];
10849
10850		if (ref_ctr_off)
10851			return libbpf_err_ptr(-EINVAL);
10852
10853		gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
10854					     binary_path, func_offset);
10855
10856		legacy_probe = strdup(probe_name);
10857		if (!legacy_probe)
10858			return libbpf_err_ptr(-ENOMEM);
10859
10860		pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
10861						    binary_path, func_offset, pid);
10862	}
10863	if (pfd < 0) {
10864		err = -errno;
10865		pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10866			prog->name, retprobe ? "uretprobe" : "uprobe",
10867			binary_path, func_offset,
10868			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10869		goto err_out;
10870	}
10871
10872	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10873	err = libbpf_get_error(link);
10874	if (err) {
10875		close(pfd);
10876		pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10877			prog->name, retprobe ? "uretprobe" : "uprobe",
10878			binary_path, func_offset,
10879			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10880		goto err_clean_legacy;
10881	}
10882	if (legacy) {
10883		struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10884
10885		perf_link->legacy_probe_name = legacy_probe;
10886		perf_link->legacy_is_kprobe = false;
10887		perf_link->legacy_is_retprobe = retprobe;
10888	}
10889	return link;
10890
10891err_clean_legacy:
10892	if (legacy)
10893		remove_uprobe_event_legacy(legacy_probe, retprobe);
10894err_out:
10895	free(legacy_probe);
10896	return libbpf_err_ptr(err);
10897}
10898
10899/* Format of u[ret]probe section definition supporting auto-attach:
10900 * u[ret]probe/binary:function[+offset]
10901 *
10902 * binary can be an absolute/relative path or a filename; the latter is resolved to a
10903 * full binary path via bpf_program__attach_uprobe_opts.
10904 *
10905 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
10906 * specified (and auto-attach is not possible) or the above format is specified for
10907 * auto-attach.
10908 */
10909static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10910{
10911	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
10912	char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
10913	int n, ret = -EINVAL;
10914	long offset = 0;
10915
10916	*link = NULL;
10917
10918	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
10919		   &probe_type, &binary_path, &func_name, &offset);
10920	switch (n) {
10921	case 1:
10922		/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
10923		ret = 0;
10924		break;
10925	case 2:
10926		pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
10927			prog->name, prog->sec_name);
10928		break;
10929	case 3:
10930	case 4:
10931		opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
10932				strcmp(probe_type, "uretprobe.s") == 0;
10933		if (opts.retprobe && offset != 0) {
10934			pr_warn("prog '%s': uretprobes do not support offset specification\n",
10935				prog->name);
10936			break;
10937		}
10938		opts.func_name = func_name;
10939		*link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
10940		ret = libbpf_get_error(*link);
10941		break;
10942	default:
10943		pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
10944			prog->sec_name);
10945		break;
10946	}
10947	free(probe_type);
10948	free(binary_path);
10949	free(func_name);
10950
10951	return ret;
10952}
10953
10954struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
10955					    bool retprobe, pid_t pid,
10956					    const char *binary_path,
10957					    size_t func_offset)
10958{
10959	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
10960
10961	return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
10962}
10963
10964struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
10965					  pid_t pid, const char *binary_path,
10966					  const char *usdt_provider, const char *usdt_name,
10967					  const struct bpf_usdt_opts *opts)
10968{
10969	char resolved_path[512];
10970	struct bpf_object *obj = prog->obj;
10971	struct bpf_link *link;
10972	__u64 usdt_cookie;
10973	int err;
10974
10975	if (!OPTS_VALID(opts, bpf_uprobe_opts))
10976		return libbpf_err_ptr(-EINVAL);
10977
10978	if (bpf_program__fd(prog) < 0) {
10979		pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10980			prog->name);
10981		return libbpf_err_ptr(-EINVAL);
10982	}
10983
10984	if (!binary_path)
10985		return libbpf_err_ptr(-EINVAL);
10986
10987	if (!strchr(binary_path, '/')) {
10988		err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
10989		if (err) {
10990			pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10991				prog->name, binary_path, err);
10992			return libbpf_err_ptr(err);
10993		}
10994		binary_path = resolved_path;
10995	}
10996
10997	/* USDT manager is instantiated lazily on first USDT attach. It will
10998	 * be destroyed together with BPF object in bpf_object__close().
10999	 */
11000	if (IS_ERR(obj->usdt_man))
11001		return libbpf_ptr(obj->usdt_man);
11002	if (!obj->usdt_man) {
11003		obj->usdt_man = usdt_manager_new(obj);
11004		if (IS_ERR(obj->usdt_man))
11005			return libbpf_ptr(obj->usdt_man);
11006	}
11007
11008	usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
11009	link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
11010					usdt_provider, usdt_name, usdt_cookie);
11011	err = libbpf_get_error(link);
11012	if (err)
11013		return libbpf_err_ptr(err);
11014	return link;
11015}
11016
11017static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11018{
11019	char *path = NULL, *provider = NULL, *name = NULL;
11020	const char *sec_name;
11021	int n, err;
11022
11023	sec_name = bpf_program__section_name(prog);
11024	if (strcmp(sec_name, "usdt") == 0) {
11025		/* no auto-attach for just SEC("usdt") */
11026		*link = NULL;
11027		return 0;
11028	}
11029
11030	n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
11031	if (n != 3) {
11032		pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
11033			sec_name);
11034		err = -EINVAL;
11035	} else {
11036		*link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
11037						 provider, name, NULL);
11038		err = libbpf_get_error(*link);
11039	}
11040	free(path);
11041	free(provider);
11042	free(name);
11043	return err;
11044}
11045
11046static int determine_tracepoint_id(const char *tp_category,
11047				   const char *tp_name)
11048{
11049	char file[PATH_MAX];
11050	int ret;
11051
11052	ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
11053		       tracefs_path(), tp_category, tp_name);
11054	if (ret < 0)
11055		return -errno;
11056	if (ret >= sizeof(file)) {
11057		pr_debug("tracepoint %s/%s path is too long\n",
11058			 tp_category, tp_name);
11059		return -E2BIG;
11060	}
11061	return parse_uint_from_file(file, "%d\n");
11062}
11063
11064static int perf_event_open_tracepoint(const char *tp_category,
11065				      const char *tp_name)
11066{
11067	const size_t attr_sz = sizeof(struct perf_event_attr);
11068	struct perf_event_attr attr;
11069	char errmsg[STRERR_BUFSIZE];
11070	int tp_id, pfd, err;
11071
11072	tp_id = determine_tracepoint_id(tp_category, tp_name);
11073	if (tp_id < 0) {
11074		pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
11075			tp_category, tp_name,
11076			libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
11077		return tp_id;
11078	}
11079
11080	memset(&attr, 0, attr_sz);
11081	attr.type = PERF_TYPE_TRACEPOINT;
11082	attr.size = attr_sz;
11083	attr.config = tp_id;
11084
11085	pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11086		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11087	if (pfd < 0) {
11088		err = -errno;
11089		pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
11090			tp_category, tp_name,
11091			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11092		return err;
11093	}
11094	return pfd;
11095}
11096
11097struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11098						     const char *tp_category,
11099						     const char *tp_name,
11100						     const struct bpf_tracepoint_opts *opts)
11101{
11102	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11103	char errmsg[STRERR_BUFSIZE];
11104	struct bpf_link *link;
11105	int pfd, err;
11106
11107	if (!OPTS_VALID(opts, bpf_tracepoint_opts))
11108		return libbpf_err_ptr(-EINVAL);
11109
11110	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11111
11112	pfd = perf_event_open_tracepoint(tp_category, tp_name);
11113	if (pfd < 0) {
11114		pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11115			prog->name, tp_category, tp_name,
11116			libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11117		return libbpf_err_ptr(pfd);
11118	}
11119	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11120	err = libbpf_get_error(link);
11121	if (err) {
11122		close(pfd);
11123		pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11124			prog->name, tp_category, tp_name,
11125			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11126		return libbpf_err_ptr(err);
11127	}
11128	return link;
11129}
11130
11131struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11132						const char *tp_category,
11133						const char *tp_name)
11134{
11135	return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11136}
11137
11138static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11139{
11140	char *sec_name, *tp_cat, *tp_name;
11141
11142	*link = NULL;
11143
11144	/* no auto-attach for SEC("tp") or SEC("tracepoint") */
11145	if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11146		return 0;
11147
11148	sec_name = strdup(prog->sec_name);
11149	if (!sec_name)
11150		return -ENOMEM;
11151
11152	/* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
11153	if (str_has_pfx(prog->sec_name, "tp/"))
11154		tp_cat = sec_name + sizeof("tp/") - 1;
11155	else
11156		tp_cat = sec_name + sizeof("tracepoint/") - 1;
11157	tp_name = strchr(tp_cat, '/');
11158	if (!tp_name) {
11159		free(sec_name);
11160		return -EINVAL;
11161	}
11162	*tp_name = '\0';
11163	tp_name++;
11164
11165	*link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11166	free(sec_name);
11167	return libbpf_get_error(*link);
11168}
11169
11170struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11171						    const char *tp_name)
11172{
11173	char errmsg[STRERR_BUFSIZE];
11174	struct bpf_link *link;
11175	int prog_fd, pfd;
11176
11177	prog_fd = bpf_program__fd(prog);
11178	if (prog_fd < 0) {
11179		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11180		return libbpf_err_ptr(-EINVAL);
11181	}
11182
11183	link = calloc(1, sizeof(*link));
11184	if (!link)
11185		return libbpf_err_ptr(-ENOMEM);
11186	link->detach = &bpf_link__detach_fd;
11187
11188	pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
11189	if (pfd < 0) {
11190		pfd = -errno;
11191		free(link);
11192		pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11193			prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11194		return libbpf_err_ptr(pfd);
11195	}
11196	link->fd = pfd;
11197	return link;
11198}
11199
11200static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11201{
11202	static const char *const prefixes[] = {
11203		"raw_tp",
11204		"raw_tracepoint",
11205		"raw_tp.w",
11206		"raw_tracepoint.w",
11207	};
11208	size_t i;
11209	const char *tp_name = NULL;
11210
11211	*link = NULL;
11212
11213	for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
11214		size_t pfx_len;
11215
11216		if (!str_has_pfx(prog->sec_name, prefixes[i]))
11217			continue;
11218
11219		pfx_len = strlen(prefixes[i]);
11220		/* no auto-attach case of, e.g., SEC("raw_tp") */
11221		if (prog->sec_name[pfx_len] == '\0')
11222			return 0;
11223
11224		if (prog->sec_name[pfx_len] != '/')
11225			continue;
11226
11227		tp_name = prog->sec_name + pfx_len + 1;
11228		break;
11229	}
11230
11231	if (!tp_name) {
11232		pr_warn("prog '%s': invalid section name '%s'\n",
11233			prog->name, prog->sec_name);
11234		return -EINVAL;
11235	}
11236
11237	*link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11238	return libbpf_get_error(*link);
11239}
11240
11241/* Common logic for all BPF program types that attach to a btf_id */
11242static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11243						   const struct bpf_trace_opts *opts)
11244{
11245	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
11246	char errmsg[STRERR_BUFSIZE];
11247	struct bpf_link *link;
11248	int prog_fd, pfd;
11249
11250	if (!OPTS_VALID(opts, bpf_trace_opts))
11251		return libbpf_err_ptr(-EINVAL);
11252
11253	prog_fd = bpf_program__fd(prog);
11254	if (prog_fd < 0) {
11255		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11256		return libbpf_err_ptr(-EINVAL);
11257	}
11258
11259	link = calloc(1, sizeof(*link));
11260	if (!link)
11261		return libbpf_err_ptr(-ENOMEM);
11262	link->detach = &bpf_link__detach_fd;
11263
11264	/* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
11265	link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
11266	pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11267	if (pfd < 0) {
11268		pfd = -errno;
11269		free(link);
11270		pr_warn("prog '%s': failed to attach: %s\n",
11271			prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11272		return libbpf_err_ptr(pfd);
11273	}
11274	link->fd = pfd;
11275	return link;
11276}
11277
11278struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11279{
11280	return bpf_program__attach_btf_id(prog, NULL);
11281}
11282
11283struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11284						const struct bpf_trace_opts *opts)
11285{
11286	return bpf_program__attach_btf_id(prog, opts);
11287}
11288
11289struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11290{
11291	return bpf_program__attach_btf_id(prog, NULL);
11292}
11293
11294static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11295{
11296	*link = bpf_program__attach_trace(prog);
11297	return libbpf_get_error(*link);
11298}
11299
11300static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11301{
11302	*link = bpf_program__attach_lsm(prog);
11303	return libbpf_get_error(*link);
11304}
11305
11306static struct bpf_link *
11307bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
11308		       const char *target_name)
11309{
11310	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
11311			    .target_btf_id = btf_id);
11312	enum bpf_attach_type attach_type;
11313	char errmsg[STRERR_BUFSIZE];
11314	struct bpf_link *link;
11315	int prog_fd, link_fd;
11316
11317	prog_fd = bpf_program__fd(prog);
11318	if (prog_fd < 0) {
11319		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11320		return libbpf_err_ptr(-EINVAL);
11321	}
11322
11323	link = calloc(1, sizeof(*link));
11324	if (!link)
11325		return libbpf_err_ptr(-ENOMEM);
11326	link->detach = &bpf_link__detach_fd;
11327
11328	attach_type = bpf_program__expected_attach_type(prog);
11329	link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
11330	if (link_fd < 0) {
11331		link_fd = -errno;
11332		free(link);
11333		pr_warn("prog '%s': failed to attach to %s: %s\n",
11334			prog->name, target_name,
11335			libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11336		return libbpf_err_ptr(link_fd);
11337	}
11338	link->fd = link_fd;
11339	return link;
11340}
11341
11342struct bpf_link *
11343bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11344{
11345	return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
11346}
11347
11348struct bpf_link *
11349bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11350{
11351	return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
11352}
11353
11354struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11355{
11356	/* target_fd/target_ifindex use the same field in LINK_CREATE */
11357	return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
11358}
11359
11360struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
11361					      int target_fd,
11362					      const char *attach_func_name)
11363{
11364	int btf_id;
11365
11366	if (!!target_fd != !!attach_func_name) {
11367		pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
11368			prog->name);
11369		return libbpf_err_ptr(-EINVAL);
11370	}
11371
11372	if (prog->type != BPF_PROG_TYPE_EXT) {
11373		pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
11374			prog->name);
11375		return libbpf_err_ptr(-EINVAL);
11376	}
11377
11378	if (target_fd) {
11379		btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
11380		if (btf_id < 0)
11381			return libbpf_err_ptr(btf_id);
11382
11383		return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
11384	} else {
11385		/* no target, so use raw_tracepoint_open for compatibility
11386		 * with old kernels
11387		 */
11388		return bpf_program__attach_trace(prog);
11389	}
11390}
11391
11392struct bpf_link *
11393bpf_program__attach_iter(const struct bpf_program *prog,
11394			 const struct bpf_iter_attach_opts *opts)
11395{
11396	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
11397	char errmsg[STRERR_BUFSIZE];
11398	struct bpf_link *link;
11399	int prog_fd, link_fd;
11400	__u32 target_fd = 0;
11401
11402	if (!OPTS_VALID(opts, bpf_iter_attach_opts))
11403		return libbpf_err_ptr(-EINVAL);
11404
11405	link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
11406	link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
11407
11408	prog_fd = bpf_program__fd(prog);
11409	if (prog_fd < 0) {
11410		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11411		return libbpf_err_ptr(-EINVAL);
11412	}
11413
11414	link = calloc(1, sizeof(*link));
11415	if (!link)
11416		return libbpf_err_ptr(-ENOMEM);
11417	link->detach = &bpf_link__detach_fd;
11418
11419	link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
11420				  &link_create_opts);
11421	if (link_fd < 0) {
11422		link_fd = -errno;
11423		free(link);
11424		pr_warn("prog '%s': failed to attach to iterator: %s\n",
11425			prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11426		return libbpf_err_ptr(link_fd);
11427	}
11428	link->fd = link_fd;
11429	return link;
11430}
11431
11432static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11433{
11434	*link = bpf_program__attach_iter(prog, NULL);
11435	return libbpf_get_error(*link);
11436}
11437
11438struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
11439{
11440	struct bpf_link *link = NULL;
11441	int err;
11442
11443	if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11444		return libbpf_err_ptr(-EOPNOTSUPP);
11445
11446	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11447	if (err)
11448		return libbpf_err_ptr(err);
11449
11450	/* When calling bpf_program__attach() explicitly, auto-attach support
11451	 * is expected to work, so NULL returned link is considered an error.
11452	 * This is different for skeleton's attach, see comment in
11453	 * bpf_object__attach_skeleton().
11454	 */
11455	if (!link)
11456		return libbpf_err_ptr(-EOPNOTSUPP);
11457
11458	return link;
11459}
11460
11461static int bpf_link__detach_struct_ops(struct bpf_link *link)
11462{
11463	__u32 zero = 0;
11464
11465	if (bpf_map_delete_elem(link->fd, &zero))
11466		return -errno;
11467
11468	return 0;
11469}
11470
11471struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
11472{
11473	struct bpf_struct_ops *st_ops;
11474	struct bpf_link *link;
11475	__u32 i, zero = 0;
11476	int err;
11477
11478	if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11479		return libbpf_err_ptr(-EINVAL);
11480
11481	link = calloc(1, sizeof(*link));
11482	if (!link)
11483		return libbpf_err_ptr(-EINVAL);
11484
11485	st_ops = map->st_ops;
11486	for (i = 0; i < btf_vlen(st_ops->type); i++) {
11487		struct bpf_program *prog = st_ops->progs[i];
11488		void *kern_data;
11489		int prog_fd;
11490
11491		if (!prog)
11492			continue;
11493
11494		prog_fd = bpf_program__fd(prog);
11495		kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11496		*(unsigned long *)kern_data = prog_fd;
11497	}
11498
11499	err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11500	if (err) {
11501		err = -errno;
11502		free(link);
11503		return libbpf_err_ptr(err);
11504	}
11505
11506	link->detach = bpf_link__detach_struct_ops;
11507	link->fd = map->fd;
11508
11509	return link;
11510}
11511
11512typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
11513							  void *private_data);
11514
11515static enum bpf_perf_event_ret
11516perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
11517		       void **copy_mem, size_t *copy_size,
11518		       bpf_perf_event_print_t fn, void *private_data)
11519{
11520	struct perf_event_mmap_page *header = mmap_mem;
11521	__u64 data_head = ring_buffer_read_head(header);
11522	__u64 data_tail = header->data_tail;
11523	void *base = ((__u8 *)header) + page_size;
11524	int ret = LIBBPF_PERF_EVENT_CONT;
11525	struct perf_event_header *ehdr;
11526	size_t ehdr_size;
11527
11528	while (data_head != data_tail) {
11529		ehdr = base + (data_tail & (mmap_size - 1));
11530		ehdr_size = ehdr->size;
11531
11532		if (((void *)ehdr) + ehdr_size > base + mmap_size) {
11533			void *copy_start = ehdr;
11534			size_t len_first = base + mmap_size - copy_start;
11535			size_t len_secnd = ehdr_size - len_first;
11536
11537			if (*copy_size < ehdr_size) {
11538				free(*copy_mem);
11539				*copy_mem = malloc(ehdr_size);
11540				if (!*copy_mem) {
11541					*copy_size = 0;
11542					ret = LIBBPF_PERF_EVENT_ERROR;
11543					break;
11544				}
11545				*copy_size = ehdr_size;
11546			}
11547
11548			memcpy(*copy_mem, copy_start, len_first);
11549			memcpy(*copy_mem + len_first, base, len_secnd);
11550			ehdr = *copy_mem;
11551		}
11552
11553		ret = fn(ehdr, private_data);
11554		data_tail += ehdr_size;
11555		if (ret != LIBBPF_PERF_EVENT_CONT)
11556			break;
11557	}
11558
11559	ring_buffer_write_tail(header, data_tail);
11560	return libbpf_err(ret);
11561}
11562
11563struct perf_buffer;
11564
11565struct perf_buffer_params {
11566	struct perf_event_attr *attr;
11567	/* if event_cb is specified, it takes precendence */
11568	perf_buffer_event_fn event_cb;
11569	/* sample_cb and lost_cb are higher-level common-case callbacks */
11570	perf_buffer_sample_fn sample_cb;
11571	perf_buffer_lost_fn lost_cb;
11572	void *ctx;
11573	int cpu_cnt;
11574	int *cpus;
11575	int *map_keys;
11576};
11577
11578struct perf_cpu_buf {
11579	struct perf_buffer *pb;
11580	void *base; /* mmap()'ed memory */
11581	void *buf; /* for reconstructing segmented data */
11582	size_t buf_size;
11583	int fd;
11584	int cpu;
11585	int map_key;
11586};
11587
11588struct perf_buffer {
11589	perf_buffer_event_fn event_cb;
11590	perf_buffer_sample_fn sample_cb;
11591	perf_buffer_lost_fn lost_cb;
11592	void *ctx; /* passed into callbacks */
11593
11594	size_t page_size;
11595	size_t mmap_size;
11596	struct perf_cpu_buf **cpu_bufs;
11597	struct epoll_event *events;
11598	int cpu_cnt; /* number of allocated CPU buffers */
11599	int epoll_fd; /* perf event FD */
11600	int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
11601};
11602
11603static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
11604				      struct perf_cpu_buf *cpu_buf)
11605{
11606	if (!cpu_buf)
11607		return;
11608	if (cpu_buf->base &&
11609	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
11610		pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
11611	if (cpu_buf->fd >= 0) {
11612		ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
11613		close(cpu_buf->fd);
11614	}
11615	free(cpu_buf->buf);
11616	free(cpu_buf);
11617}
11618
11619void perf_buffer__free(struct perf_buffer *pb)
11620{
11621	int i;
11622
11623	if (IS_ERR_OR_NULL(pb))
11624		return;
11625	if (pb->cpu_bufs) {
11626		for (i = 0; i < pb->cpu_cnt; i++) {
11627			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11628
11629			if (!cpu_buf)
11630				continue;
11631
11632			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
11633			perf_buffer__free_cpu_buf(pb, cpu_buf);
11634		}
11635		free(pb->cpu_bufs);
11636	}
11637	if (pb->epoll_fd >= 0)
11638		close(pb->epoll_fd);
11639	free(pb->events);
11640	free(pb);
11641}
11642
11643static struct perf_cpu_buf *
11644perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
11645			  int cpu, int map_key)
11646{
11647	struct perf_cpu_buf *cpu_buf;
11648	char msg[STRERR_BUFSIZE];
11649	int err;
11650
11651	cpu_buf = calloc(1, sizeof(*cpu_buf));
11652	if (!cpu_buf)
11653		return ERR_PTR(-ENOMEM);
11654
11655	cpu_buf->pb = pb;
11656	cpu_buf->cpu = cpu;
11657	cpu_buf->map_key = map_key;
11658
11659	cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
11660			      -1, PERF_FLAG_FD_CLOEXEC);
11661	if (cpu_buf->fd < 0) {
11662		err = -errno;
11663		pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
11664			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11665		goto error;
11666	}
11667
11668	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
11669			     PROT_READ | PROT_WRITE, MAP_SHARED,
11670			     cpu_buf->fd, 0);
11671	if (cpu_buf->base == MAP_FAILED) {
11672		cpu_buf->base = NULL;
11673		err = -errno;
11674		pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
11675			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11676		goto error;
11677	}
11678
11679	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11680		err = -errno;
11681		pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
11682			cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11683		goto error;
11684	}
11685
11686	return cpu_buf;
11687
11688error:
11689	perf_buffer__free_cpu_buf(pb, cpu_buf);
11690	return (struct perf_cpu_buf *)ERR_PTR(err);
11691}
11692
11693static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11694					      struct perf_buffer_params *p);
11695
11696struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
11697				     perf_buffer_sample_fn sample_cb,
11698				     perf_buffer_lost_fn lost_cb,
11699				     void *ctx,
11700				     const struct perf_buffer_opts *opts)
11701{
11702	const size_t attr_sz = sizeof(struct perf_event_attr);
11703	struct perf_buffer_params p = {};
11704	struct perf_event_attr attr;
11705
11706	if (!OPTS_VALID(opts, perf_buffer_opts))
11707		return libbpf_err_ptr(-EINVAL);
11708
11709	memset(&attr, 0, attr_sz);
11710	attr.size = attr_sz;
11711	attr.config = PERF_COUNT_SW_BPF_OUTPUT;
11712	attr.type = PERF_TYPE_SOFTWARE;
11713	attr.sample_type = PERF_SAMPLE_RAW;
11714	attr.sample_period = 1;
11715	attr.wakeup_events = 1;
11716
11717	p.attr = &attr;
11718	p.sample_cb = sample_cb;
11719	p.lost_cb = lost_cb;
11720	p.ctx = ctx;
11721
11722	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11723}
11724
11725struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
11726					 struct perf_event_attr *attr,
11727					 perf_buffer_event_fn event_cb, void *ctx,
11728					 const struct perf_buffer_raw_opts *opts)
11729{
11730	struct perf_buffer_params p = {};
11731
11732	if (!attr)
11733		return libbpf_err_ptr(-EINVAL);
11734
11735	if (!OPTS_VALID(opts, perf_buffer_raw_opts))
11736		return libbpf_err_ptr(-EINVAL);
11737
11738	p.attr = attr;
11739	p.event_cb = event_cb;
11740	p.ctx = ctx;
11741	p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
11742	p.cpus = OPTS_GET(opts, cpus, NULL);
11743	p.map_keys = OPTS_GET(opts, map_keys, NULL);
11744
11745	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11746}
11747
11748static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11749					      struct perf_buffer_params *p)
11750{
11751	const char *online_cpus_file = "/sys/devices/system/cpu/online";
11752	struct bpf_map_info map;
11753	char msg[STRERR_BUFSIZE];
11754	struct perf_buffer *pb;
11755	bool *online = NULL;
11756	__u32 map_info_len;
11757	int err, i, j, n;
11758
11759	if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
11760		pr_warn("page count should be power of two, but is %zu\n",
11761			page_cnt);
11762		return ERR_PTR(-EINVAL);
11763	}
11764
11765	/* best-effort sanity checks */
11766	memset(&map, 0, sizeof(map));
11767	map_info_len = sizeof(map);
11768	err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
11769	if (err) {
11770		err = -errno;
11771		/* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
11772		 * -EBADFD, -EFAULT, or -E2BIG on real error
11773		 */
11774		if (err != -EINVAL) {
11775			pr_warn("failed to get map info for map FD %d: %s\n",
11776				map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
11777			return ERR_PTR(err);
11778		}
11779		pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
11780			 map_fd);
11781	} else {
11782		if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
11783			pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
11784				map.name);
11785			return ERR_PTR(-EINVAL);
11786		}
11787	}
11788
11789	pb = calloc(1, sizeof(*pb));
11790	if (!pb)
11791		return ERR_PTR(-ENOMEM);
11792
11793	pb->event_cb = p->event_cb;
11794	pb->sample_cb = p->sample_cb;
11795	pb->lost_cb = p->lost_cb;
11796	pb->ctx = p->ctx;
11797
11798	pb->page_size = getpagesize();
11799	pb->mmap_size = pb->page_size * page_cnt;
11800	pb->map_fd = map_fd;
11801
11802	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
11803	if (pb->epoll_fd < 0) {
11804		err = -errno;
11805		pr_warn("failed to create epoll instance: %s\n",
11806			libbpf_strerror_r(err, msg, sizeof(msg)));
11807		goto error;
11808	}
11809
11810	if (p->cpu_cnt > 0) {
11811		pb->cpu_cnt = p->cpu_cnt;
11812	} else {
11813		pb->cpu_cnt = libbpf_num_possible_cpus();
11814		if (pb->cpu_cnt < 0) {
11815			err = pb->cpu_cnt;
11816			goto error;
11817		}
11818		if (map.max_entries && map.max_entries < pb->cpu_cnt)
11819			pb->cpu_cnt = map.max_entries;
11820	}
11821
11822	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
11823	if (!pb->events) {
11824		err = -ENOMEM;
11825		pr_warn("failed to allocate events: out of memory\n");
11826		goto error;
11827	}
11828	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
11829	if (!pb->cpu_bufs) {
11830		err = -ENOMEM;
11831		pr_warn("failed to allocate buffers: out of memory\n");
11832		goto error;
11833	}
11834
11835	err = parse_cpu_mask_file(online_cpus_file, &online, &n);
11836	if (err) {
11837		pr_warn("failed to get online CPU mask: %d\n", err);
11838		goto error;
11839	}
11840
11841	for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
11842		struct perf_cpu_buf *cpu_buf;
11843		int cpu, map_key;
11844
11845		cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
11846		map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
11847
11848		/* in case user didn't explicitly requested particular CPUs to
11849		 * be attached to, skip offline/not present CPUs
11850		 */
11851		if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
11852			continue;
11853
11854		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
11855		if (IS_ERR(cpu_buf)) {
11856			err = PTR_ERR(cpu_buf);
11857			goto error;
11858		}
11859
11860		pb->cpu_bufs[j] = cpu_buf;
11861
11862		err = bpf_map_update_elem(pb->map_fd, &map_key,
11863					  &cpu_buf->fd, 0);
11864		if (err) {
11865			err = -errno;
11866			pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
11867				cpu, map_key, cpu_buf->fd,
11868				libbpf_strerror_r(err, msg, sizeof(msg)));
11869			goto error;
11870		}
11871
11872		pb->events[j].events = EPOLLIN;
11873		pb->events[j].data.ptr = cpu_buf;
11874		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
11875			      &pb->events[j]) < 0) {
11876			err = -errno;
11877			pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
11878				cpu, cpu_buf->fd,
11879				libbpf_strerror_r(err, msg, sizeof(msg)));
11880			goto error;
11881		}
11882		j++;
11883	}
11884	pb->cpu_cnt = j;
11885	free(online);
11886
11887	return pb;
11888
11889error:
11890	free(online);
11891	if (pb)
11892		perf_buffer__free(pb);
11893	return ERR_PTR(err);
11894}
11895
11896struct perf_sample_raw {
11897	struct perf_event_header header;
11898	uint32_t size;
11899	char data[];
11900};
11901
11902struct perf_sample_lost {
11903	struct perf_event_header header;
11904	uint64_t id;
11905	uint64_t lost;
11906	uint64_t sample_id;
11907};
11908
11909static enum bpf_perf_event_ret
11910perf_buffer__process_record(struct perf_event_header *e, void *ctx)
11911{
11912	struct perf_cpu_buf *cpu_buf = ctx;
11913	struct perf_buffer *pb = cpu_buf->pb;
11914	void *data = e;
11915
11916	/* user wants full control over parsing perf event */
11917	if (pb->event_cb)
11918		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
11919
11920	switch (e->type) {
11921	case PERF_RECORD_SAMPLE: {
11922		struct perf_sample_raw *s = data;
11923
11924		if (pb->sample_cb)
11925			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
11926		break;
11927	}
11928	case PERF_RECORD_LOST: {
11929		struct perf_sample_lost *s = data;
11930
11931		if (pb->lost_cb)
11932			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
11933		break;
11934	}
11935	default:
11936		pr_warn("unknown perf sample type %d\n", e->type);
11937		return LIBBPF_PERF_EVENT_ERROR;
11938	}
11939	return LIBBPF_PERF_EVENT_CONT;
11940}
11941
11942static int perf_buffer__process_records(struct perf_buffer *pb,
11943					struct perf_cpu_buf *cpu_buf)
11944{
11945	enum bpf_perf_event_ret ret;
11946
11947	ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
11948				     pb->page_size, &cpu_buf->buf,
11949				     &cpu_buf->buf_size,
11950				     perf_buffer__process_record, cpu_buf);
11951	if (ret != LIBBPF_PERF_EVENT_CONT)
11952		return ret;
11953	return 0;
11954}
11955
11956int perf_buffer__epoll_fd(const struct perf_buffer *pb)
11957{
11958	return pb->epoll_fd;
11959}
11960
11961int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
11962{
11963	int i, cnt, err;
11964
11965	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
11966	if (cnt < 0)
11967		return -errno;
11968
11969	for (i = 0; i < cnt; i++) {
11970		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
11971
11972		err = perf_buffer__process_records(pb, cpu_buf);
11973		if (err) {
11974			pr_warn("error while processing records: %d\n", err);
11975			return libbpf_err(err);
11976		}
11977	}
11978	return cnt;
11979}
11980
11981/* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
11982 * manager.
11983 */
11984size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
11985{
11986	return pb->cpu_cnt;
11987}
11988
11989/*
11990 * Return perf_event FD of a ring buffer in *buf_idx* slot of
11991 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
11992 * select()/poll()/epoll() Linux syscalls.
11993 */
11994int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
11995{
11996	struct perf_cpu_buf *cpu_buf;
11997
11998	if (buf_idx >= pb->cpu_cnt)
11999		return libbpf_err(-EINVAL);
12000
12001	cpu_buf = pb->cpu_bufs[buf_idx];
12002	if (!cpu_buf)
12003		return libbpf_err(-ENOENT);
12004
12005	return cpu_buf->fd;
12006}
12007
12008int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
12009{
12010	struct perf_cpu_buf *cpu_buf;
12011
12012	if (buf_idx >= pb->cpu_cnt)
12013		return libbpf_err(-EINVAL);
12014
12015	cpu_buf = pb->cpu_bufs[buf_idx];
12016	if (!cpu_buf)
12017		return libbpf_err(-ENOENT);
12018
12019	*buf = cpu_buf->base;
12020	*buf_size = pb->mmap_size;
12021	return 0;
12022}
12023
12024/*
12025 * Consume data from perf ring buffer corresponding to slot *buf_idx* in
12026 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
12027 * consume, do nothing and return success.
12028 * Returns:
12029 *   - 0 on success;
12030 *   - <0 on failure.
12031 */
12032int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
12033{
12034	struct perf_cpu_buf *cpu_buf;
12035
12036	if (buf_idx >= pb->cpu_cnt)
12037		return libbpf_err(-EINVAL);
12038
12039	cpu_buf = pb->cpu_bufs[buf_idx];
12040	if (!cpu_buf)
12041		return libbpf_err(-ENOENT);
12042
12043	return perf_buffer__process_records(pb, cpu_buf);
12044}
12045
12046int perf_buffer__consume(struct perf_buffer *pb)
12047{
12048	int i, err;
12049
12050	for (i = 0; i < pb->cpu_cnt; i++) {
12051		struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
12052
12053		if (!cpu_buf)
12054			continue;
12055
12056		err = perf_buffer__process_records(pb, cpu_buf);
12057		if (err) {
12058			pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
12059			return libbpf_err(err);
12060		}
12061	}
12062	return 0;
12063}
12064
12065int bpf_program__set_attach_target(struct bpf_program *prog,
12066				   int attach_prog_fd,
12067				   const char *attach_func_name)
12068{
12069	int btf_obj_fd = 0, btf_id = 0, err;
12070
12071	if (!prog || attach_prog_fd < 0)
12072		return libbpf_err(-EINVAL);
12073
12074	if (prog->obj->loaded)
12075		return libbpf_err(-EINVAL);
12076
12077	if (attach_prog_fd && !attach_func_name) {
12078		/* remember attach_prog_fd and let bpf_program__load() find
12079		 * BTF ID during the program load
12080		 */
12081		prog->attach_prog_fd = attach_prog_fd;
12082		return 0;
12083	}
12084
12085	if (attach_prog_fd) {
12086		btf_id = libbpf_find_prog_btf_id(attach_func_name,
12087						 attach_prog_fd);
12088		if (btf_id < 0)
12089			return libbpf_err(btf_id);
12090	} else {
12091		if (!attach_func_name)
12092			return libbpf_err(-EINVAL);
12093
12094		/* load btf_vmlinux, if not yet */
12095		err = bpf_object__load_vmlinux_btf(prog->obj, true);
12096		if (err)
12097			return libbpf_err(err);
12098		err = find_kernel_btf_id(prog->obj, attach_func_name,
12099					 prog->expected_attach_type,
12100					 &btf_obj_fd, &btf_id);
12101		if (err)
12102			return libbpf_err(err);
12103	}
12104
12105	prog->attach_btf_id = btf_id;
12106	prog->attach_btf_obj_fd = btf_obj_fd;
12107	prog->attach_prog_fd = attach_prog_fd;
12108	return 0;
12109}
12110
12111int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
12112{
12113	int err = 0, n, len, start, end = -1;
12114	bool *tmp;
12115
12116	*mask = NULL;
12117	*mask_sz = 0;
12118
12119	/* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12120	while (*s) {
12121		if (*s == ',' || *s == '\n') {
12122			s++;
12123			continue;
12124		}
12125		n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12126		if (n <= 0 || n > 2) {
12127			pr_warn("Failed to get CPU range %s: %d\n", s, n);
12128			err = -EINVAL;
12129			goto cleanup;
12130		} else if (n == 1) {
12131			end = start;
12132		}
12133		if (start < 0 || start > end) {
12134			pr_warn("Invalid CPU range [%d,%d] in %s\n",
12135				start, end, s);
12136			err = -EINVAL;
12137			goto cleanup;
12138		}
12139		tmp = realloc(*mask, end + 1);
12140		if (!tmp) {
12141			err = -ENOMEM;
12142			goto cleanup;
12143		}
12144		*mask = tmp;
12145		memset(tmp + *mask_sz, 0, start - *mask_sz);
12146		memset(tmp + start, 1, end - start + 1);
12147		*mask_sz = end + 1;
12148		s += len;
12149	}
12150	if (!*mask_sz) {
12151		pr_warn("Empty CPU range\n");
12152		return -EINVAL;
12153	}
12154	return 0;
12155cleanup:
12156	free(*mask);
12157	*mask = NULL;
12158	return err;
12159}
12160
12161int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
12162{
12163	int fd, err = 0, len;
12164	char buf[128];
12165
12166	fd = open(fcpu, O_RDONLY | O_CLOEXEC);
12167	if (fd < 0) {
12168		err = -errno;
12169		pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
12170		return err;
12171	}
12172	len = read(fd, buf, sizeof(buf));
12173	close(fd);
12174	if (len <= 0) {
12175		err = len ? -errno : -EINVAL;
12176		pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
12177		return err;
12178	}
12179	if (len >= sizeof(buf)) {
12180		pr_warn("CPU mask is too big in file %s\n", fcpu);
12181		return -E2BIG;
12182	}
12183	buf[len] = '\0';
12184
12185	return parse_cpu_mask_str(buf, mask, mask_sz);
12186}
12187
12188int libbpf_num_possible_cpus(void)
12189{
12190	static const char *fcpu = "/sys/devices/system/cpu/possible";
12191	static int cpus;
12192	int err, n, i, tmp_cpus;
12193	bool *mask;
12194
12195	tmp_cpus = READ_ONCE(cpus);
12196	if (tmp_cpus > 0)
12197		return tmp_cpus;
12198
12199	err = parse_cpu_mask_file(fcpu, &mask, &n);
12200	if (err)
12201		return libbpf_err(err);
12202
12203	tmp_cpus = 0;
12204	for (i = 0; i < n; i++) {
12205		if (mask[i])
12206			tmp_cpus++;
12207	}
12208	free(mask);
12209
12210	WRITE_ONCE(cpus, tmp_cpus);
12211	return tmp_cpus;
12212}
12213
12214static int populate_skeleton_maps(const struct bpf_object *obj,
12215				  struct bpf_map_skeleton *maps,
12216				  size_t map_cnt)
12217{
12218	int i;
12219
12220	for (i = 0; i < map_cnt; i++) {
12221		struct bpf_map **map = maps[i].map;
12222		const char *name = maps[i].name;
12223		void **mmaped = maps[i].mmaped;
12224
12225		*map = bpf_object__find_map_by_name(obj, name);
12226		if (!*map) {
12227			pr_warn("failed to find skeleton map '%s'\n", name);
12228			return -ESRCH;
12229		}
12230
12231		/* externs shouldn't be pre-setup from user code */
12232		if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12233			*mmaped = (*map)->mmaped;
12234	}
12235	return 0;
12236}
12237
12238static int populate_skeleton_progs(const struct bpf_object *obj,
12239				   struct bpf_prog_skeleton *progs,
12240				   size_t prog_cnt)
12241{
12242	int i;
12243
12244	for (i = 0; i < prog_cnt; i++) {
12245		struct bpf_program **prog = progs[i].prog;
12246		const char *name = progs[i].name;
12247
12248		*prog = bpf_object__find_program_by_name(obj, name);
12249		if (!*prog) {
12250			pr_warn("failed to find skeleton program '%s'\n", name);
12251			return -ESRCH;
12252		}
12253	}
12254	return 0;
12255}
12256
12257int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
12258			      const struct bpf_object_open_opts *opts)
12259{
12260	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
12261		.object_name = s->name,
12262	);
12263	struct bpf_object *obj;
12264	int err;
12265
12266	/* Attempt to preserve opts->object_name, unless overriden by user
12267	 * explicitly. Overwriting object name for skeletons is discouraged,
12268	 * as it breaks global data maps, because they contain object name
12269	 * prefix as their own map name prefix. When skeleton is generated,
12270	 * bpftool is making an assumption that this name will stay the same.
12271	 */
12272	if (opts) {
12273		memcpy(&skel_opts, opts, sizeof(*opts));
12274		if (!opts->object_name)
12275			skel_opts.object_name = s->name;
12276	}
12277
12278	obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12279	err = libbpf_get_error(obj);
12280	if (err) {
12281		pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
12282			s->name, err);
12283		return libbpf_err(err);
12284	}
12285
12286	*s->obj = obj;
12287	err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12288	if (err) {
12289		pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12290		return libbpf_err(err);
12291	}
12292
12293	err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12294	if (err) {
12295		pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12296		return libbpf_err(err);
12297	}
12298
12299	return 0;
12300}
12301
12302int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
12303{
12304	int err, len, var_idx, i;
12305	const char *var_name;
12306	const struct bpf_map *map;
12307	struct btf *btf;
12308	__u32 map_type_id;
12309	const struct btf_type *map_type, *var_type;
12310	const struct bpf_var_skeleton *var_skel;
12311	struct btf_var_secinfo *var;
12312
12313	if (!s->obj)
12314		return libbpf_err(-EINVAL);
12315
12316	btf = bpf_object__btf(s->obj);
12317	if (!btf) {
12318		pr_warn("subskeletons require BTF at runtime (object %s)\n",
12319			bpf_object__name(s->obj));
12320		return libbpf_err(-errno);
12321	}
12322
12323	err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12324	if (err) {
12325		pr_warn("failed to populate subskeleton maps: %d\n", err);
12326		return libbpf_err(err);
12327	}
12328
12329	err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12330	if (err) {
12331		pr_warn("failed to populate subskeleton maps: %d\n", err);
12332		return libbpf_err(err);
12333	}
12334
12335	for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12336		var_skel = &s->vars[var_idx];
12337		map = *var_skel->map;
12338		map_type_id = bpf_map__btf_value_type_id(map);
12339		map_type = btf__type_by_id(btf, map_type_id);
12340
12341		if (!btf_is_datasec(map_type)) {
12342			pr_warn("type for map '%1$s' is not a datasec: %2$s",
12343				bpf_map__name(map),
12344				__btf_kind_str(btf_kind(map_type)));
12345			return libbpf_err(-EINVAL);
12346		}
12347
12348		len = btf_vlen(map_type);
12349		var = btf_var_secinfos(map_type);
12350		for (i = 0; i < len; i++, var++) {
12351			var_type = btf__type_by_id(btf, var->type);
12352			var_name = btf__name_by_offset(btf, var_type->name_off);
12353			if (strcmp(var_name, var_skel->name) == 0) {
12354				*var_skel->addr = map->mmaped + var->offset;
12355				break;
12356			}
12357		}
12358	}
12359	return 0;
12360}
12361
12362void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
12363{
12364	if (!s)
12365		return;
12366	free(s->maps);
12367	free(s->progs);
12368	free(s->vars);
12369	free(s);
12370}
12371
12372int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
12373{
12374	int i, err;
12375
12376	err = bpf_object__load(*s->obj);
12377	if (err) {
12378		pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12379		return libbpf_err(err);
12380	}
12381
12382	for (i = 0; i < s->map_cnt; i++) {
12383		struct bpf_map *map = *s->maps[i].map;
12384		size_t mmap_sz = bpf_map_mmap_sz(map);
12385		int prot, map_fd = bpf_map__fd(map);
12386		void **mmaped = s->maps[i].mmaped;
12387
12388		if (!mmaped)
12389			continue;
12390
12391		if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12392			*mmaped = NULL;
12393			continue;
12394		}
12395
12396		if (map->def.map_flags & BPF_F_RDONLY_PROG)
12397			prot = PROT_READ;
12398		else
12399			prot = PROT_READ | PROT_WRITE;
12400
12401		/* Remap anonymous mmap()-ed "map initialization image" as
12402		 * a BPF map-backed mmap()-ed memory, but preserving the same
12403		 * memory address. This will cause kernel to change process'
12404		 * page table to point to a different piece of kernel memory,
12405		 * but from userspace point of view memory address (and its
12406		 * contents, being identical at this point) will stay the
12407		 * same. This mapping will be released by bpf_object__close()
12408		 * as per normal clean up procedure, so we don't need to worry
12409		 * about it from skeleton's clean up perspective.
12410		 */
12411		*mmaped = mmap(map->mmaped, mmap_sz, prot,
12412				MAP_SHARED | MAP_FIXED, map_fd, 0);
12413		if (*mmaped == MAP_FAILED) {
12414			err = -errno;
12415			*mmaped = NULL;
12416			pr_warn("failed to re-mmap() map '%s': %d\n",
12417				 bpf_map__name(map), err);
12418			return libbpf_err(err);
12419		}
12420	}
12421
12422	return 0;
12423}
12424
12425int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
12426{
12427	int i, err;
12428
12429	for (i = 0; i < s->prog_cnt; i++) {
12430		struct bpf_program *prog = *s->progs[i].prog;
12431		struct bpf_link **link = s->progs[i].link;
12432
12433		if (!prog->autoload || !prog->autoattach)
12434			continue;
12435
12436		/* auto-attaching not supported for this program */
12437		if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12438			continue;
12439
12440		/* if user already set the link manually, don't attempt auto-attach */
12441		if (*link)
12442			continue;
12443
12444		err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12445		if (err) {
12446			pr_warn("prog '%s': failed to auto-attach: %d\n",
12447				bpf_program__name(prog), err);
12448			return libbpf_err(err);
12449		}
12450
12451		/* It's possible that for some SEC() definitions auto-attach
12452		 * is supported in some cases (e.g., if definition completely
12453		 * specifies target information), but is not in other cases.
12454		 * SEC("uprobe") is one such case. If user specified target
12455		 * binary and function name, such BPF program can be
12456		 * auto-attached. But if not, it shouldn't trigger skeleton's
12457		 * attach to fail. It should just be skipped.
12458		 * attach_fn signals such case with returning 0 (no error) and
12459		 * setting link to NULL.
12460		 */
12461	}
12462
12463	return 0;
12464}
12465
12466void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
12467{
12468	int i;
12469
12470	for (i = 0; i < s->prog_cnt; i++) {
12471		struct bpf_link **link = s->progs[i].link;
12472
12473		bpf_link__destroy(*link);
12474		*link = NULL;
12475	}
12476}
12477
12478void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
12479{
12480	if (!s)
12481		return;
12482
12483	if (s->progs)
12484		bpf_object__detach_skeleton(s);
12485	if (s->obj)
12486		bpf_object__close(*s->obj);
12487	free(s->maps);
12488	free(s->progs);
12489	free(s);
12490}