1// SPDX-License-Identifier: LGPL-2.1
   2
   3/*
   4 * Common eBPF ELF object loading operations.
   5 *
   6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8 * Copyright (C) 2015 Huawei Inc.
   9 * Copyright (C) 2017 Nicira, Inc.
  10 *
  11 * This program is free software; you can redistribute it and/or
  12 * modify it under the terms of the GNU Lesser General Public
  13 * License as published by the Free Software Foundation;
  14 * version 2.1 of the License (not later!)
  15 *
  16 * This program is distributed in the hope that it will be useful,
  17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 * GNU Lesser General Public License for more details.
  20 *
  21 * You should have received a copy of the GNU Lesser General Public
  22 * License along with this program; if not,  see <http://www.gnu.org/licenses>
  23 */
  24
  25#include <stdlib.h>
  26#include <stdio.h>
  27#include <stdarg.h>
  28#include <libgen.h>
  29#include <inttypes.h>
  30#include <string.h>
  31#include <unistd.h>
  32#include <fcntl.h>
  33#include <errno.h>
  34#include <asm/unistd.h>
  35#include <linux/err.h>
  36#include <linux/kernel.h>
  37#include <linux/bpf.h>
  38#include <linux/list.h>
  39#include <linux/limits.h>
  40#include <sys/stat.h>
  41#include <sys/types.h>
  42#include <sys/vfs.h>
  43#include <libelf.h>
  44#include <gelf.h>
  45
  46#include "libbpf.h"
  47#include "bpf.h"
  48
  49#ifndef EM_BPF
  50#define EM_BPF 247
  51#endif
  52
  53#ifndef BPF_FS_MAGIC
  54#define BPF_FS_MAGIC		0xcafe4a11
  55#endif
  56
  57#define __printf(a, b)	__attribute__((format(printf, a, b)))
  58
  59__printf(1, 2)
  60static int __base_pr(const char *format, ...)
  61{
  62	va_list args;
  63	int err;
  64
  65	va_start(args, format);
  66	err = vfprintf(stderr, format, args);
  67	va_end(args);
  68	return err;
  69}
  70
  71static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  72static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  73static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  74
  75#define __pr(func, fmt, ...)	\
  76do {				\
  77	if ((func))		\
  78		(func)("libbpf: " fmt, ##__VA_ARGS__); \
  79} while (0)
  80
  81#define pr_warning(fmt, ...)	__pr(__pr_warning, fmt, ##__VA_ARGS__)
  82#define pr_info(fmt, ...)	__pr(__pr_info, fmt, ##__VA_ARGS__)
  83#define pr_debug(fmt, ...)	__pr(__pr_debug, fmt, ##__VA_ARGS__)
  84
  85void libbpf_set_print(libbpf_print_fn_t warn,
  86		      libbpf_print_fn_t info,
  87		      libbpf_print_fn_t debug)
  88{
  89	__pr_warning = warn;
  90	__pr_info = info;
  91	__pr_debug = debug;
  92}
  93
  94#define STRERR_BUFSIZE  128
  95
  96#define ERRNO_OFFSET(e)		((e) - __LIBBPF_ERRNO__START)
  97#define ERRCODE_OFFSET(c)	ERRNO_OFFSET(LIBBPF_ERRNO__##c)
  98#define NR_ERRNO	(__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
  99
 100static const char *libbpf_strerror_table[NR_ERRNO] = {
 101	[ERRCODE_OFFSET(LIBELF)]	= "Something wrong in libelf",
 102	[ERRCODE_OFFSET(FORMAT)]	= "BPF object format invalid",
 103	[ERRCODE_OFFSET(KVERSION)]	= "'version' section incorrect or lost",
 104	[ERRCODE_OFFSET(ENDIAN)]	= "Endian mismatch",
 105	[ERRCODE_OFFSET(INTERNAL)]	= "Internal error in libbpf",
 106	[ERRCODE_OFFSET(RELOC)]		= "Relocation failed",
 107	[ERRCODE_OFFSET(VERIFY)]	= "Kernel verifier blocks program loading",
 108	[ERRCODE_OFFSET(PROG2BIG)]	= "Program too big",
 109	[ERRCODE_OFFSET(KVER)]		= "Incorrect kernel version",
 110	[ERRCODE_OFFSET(PROGTYPE)]	= "Kernel doesn't support this program type",
 111	[ERRCODE_OFFSET(WRNGPID)]	= "Wrong pid in netlink message",
 112	[ERRCODE_OFFSET(INVSEQ)]	= "Invalid netlink sequence",
 113};
 114
 115int libbpf_strerror(int err, char *buf, size_t size)
 116{
 117	if (!buf || !size)
 118		return -1;
 119
 120	err = err > 0 ? err : -err;
 121
 122	if (err < __LIBBPF_ERRNO__START) {
 123		int ret;
 124
 125		ret = strerror_r(err, buf, size);
 126		buf[size - 1] = '\0';
 127		return ret;
 128	}
 129
 130	if (err < __LIBBPF_ERRNO__END) {
 131		const char *msg;
 132
 133		msg = libbpf_strerror_table[ERRNO_OFFSET(err)];
 134		snprintf(buf, size, "%s", msg);
 135		buf[size - 1] = '\0';
 136		return 0;
 137	}
 138
 139	snprintf(buf, size, "Unknown libbpf error %d", err);
 140	buf[size - 1] = '\0';
 141	return -1;
 142}
 143
 144#define CHECK_ERR(action, err, out) do {	\
 145	err = action;			\
 146	if (err)			\
 147		goto out;		\
 148} while(0)
 149
 150
 151/* Copied from tools/perf/util/util.h */
 152#ifndef zfree
 153# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 154#endif
 155
 156#ifndef zclose
 157# define zclose(fd) ({			\
 158	int ___err = 0;			\
 159	if ((fd) >= 0)			\
 160		___err = close((fd));	\
 161	fd = -1;			\
 162	___err; })
 163#endif
 164
 165#ifdef HAVE_LIBELF_MMAP_SUPPORT
 166# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 167#else
 168# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 169#endif
 170
 171/*
 172 * bpf_prog should be a better name but it has been used in
 173 * linux/filter.h.
 174 */
 175struct bpf_program {
 176	/* Index in elf obj file, for relocation use. */
 177	int idx;
 178	char *name;
 179	char *section_name;
 180	struct bpf_insn *insns;
 181	size_t insns_cnt, main_prog_cnt;
 182	enum bpf_prog_type type;
 183
 184	struct reloc_desc {
 185		enum {
 186			RELO_LD64,
 187			RELO_CALL,
 188		} type;
 189		int insn_idx;
 190		union {
 191			int map_idx;
 192			int text_off;
 193		};
 194	} *reloc_desc;
 195	int nr_reloc;
 196
 197	struct {
 198		int nr;
 199		int *fds;
 200	} instances;
 201	bpf_program_prep_t preprocessor;
 202
 203	struct bpf_object *obj;
 204	void *priv;
 205	bpf_program_clear_priv_t clear_priv;
 206
 207	enum bpf_attach_type expected_attach_type;
 208};
 209
 210struct bpf_map {
 211	int fd;
 212	char *name;
 213	size_t offset;
 214	struct bpf_map_def def;
 215	void *priv;
 216	bpf_map_clear_priv_t clear_priv;
 217};
 218
 219static LIST_HEAD(bpf_objects_list);
 220
 221struct bpf_object {
 222	char license[64];
 223	u32 kern_version;
 224
 225	struct bpf_program *programs;
 226	size_t nr_programs;
 227	struct bpf_map *maps;
 228	size_t nr_maps;
 229
 230	bool loaded;
 231
 232	/*
 233	 * Information when doing elf related work. Only valid if fd
 234	 * is valid.
 235	 */
 236	struct {
 237		int fd;
 238		void *obj_buf;
 239		size_t obj_buf_sz;
 240		Elf *elf;
 241		GElf_Ehdr ehdr;
 242		Elf_Data *symbols;
 243		size_t strtabidx;
 244		struct {
 245			GElf_Shdr shdr;
 246			Elf_Data *data;
 247		} *reloc;
 248		int nr_reloc;
 249		int maps_shndx;
 250		int text_shndx;
 251	} efile;
 252	/*
 253	 * All loaded bpf_object is linked in a list, which is
 254	 * hidden to caller. bpf_objects__<func> handlers deal with
 255	 * all objects.
 256	 */
 257	struct list_head list;
 258
 259	void *priv;
 260	bpf_object_clear_priv_t clear_priv;
 261
 262	char path[];
 263};
 264#define obj_elf_valid(o)	((o)->efile.elf)
 265
 266static void bpf_program__unload(struct bpf_program *prog)
 267{
 268	int i;
 269
 270	if (!prog)
 271		return;
 272
 273	/*
 274	 * If the object is opened but the program was never loaded,
 275	 * it is possible that prog->instances.nr == -1.
 276	 */
 277	if (prog->instances.nr > 0) {
 278		for (i = 0; i < prog->instances.nr; i++)
 279			zclose(prog->instances.fds[i]);
 280	} else if (prog->instances.nr != -1) {
 281		pr_warning("Internal error: instances.nr is %d\n",
 282			   prog->instances.nr);
 283	}
 284
 285	prog->instances.nr = -1;
 286	zfree(&prog->instances.fds);
 287}
 288
 289static void bpf_program__exit(struct bpf_program *prog)
 290{
 291	if (!prog)
 292		return;
 293
 294	if (prog->clear_priv)
 295		prog->clear_priv(prog, prog->priv);
 296
 297	prog->priv = NULL;
 298	prog->clear_priv = NULL;
 299
 300	bpf_program__unload(prog);
 301	zfree(&prog->name);
 302	zfree(&prog->section_name);
 303	zfree(&prog->insns);
 304	zfree(&prog->reloc_desc);
 305
 306	prog->nr_reloc = 0;
 307	prog->insns_cnt = 0;
 308	prog->idx = -1;
 309}
 310
 311static int
 312bpf_program__init(void *data, size_t size, char *section_name, int idx,
 313		  struct bpf_program *prog)
 314{
 315	if (size < sizeof(struct bpf_insn)) {
 316		pr_warning("corrupted section '%s'\n", section_name);
 317		return -EINVAL;
 318	}
 319
 320	bzero(prog, sizeof(*prog));
 321
 322	prog->section_name = strdup(section_name);
 323	if (!prog->section_name) {
 324		pr_warning("failed to alloc name for prog under section(%d) %s\n",
 325			   idx, section_name);
 326		goto errout;
 327	}
 328
 329	prog->insns = malloc(size);
 330	if (!prog->insns) {
 331		pr_warning("failed to alloc insns for prog under section %s\n",
 332			   section_name);
 333		goto errout;
 334	}
 335	prog->insns_cnt = size / sizeof(struct bpf_insn);
 336	memcpy(prog->insns, data,
 337	       prog->insns_cnt * sizeof(struct bpf_insn));
 338	prog->idx = idx;
 339	prog->instances.fds = NULL;
 340	prog->instances.nr = -1;
 341	prog->type = BPF_PROG_TYPE_KPROBE;
 342
 343	return 0;
 344errout:
 345	bpf_program__exit(prog);
 346	return -ENOMEM;
 347}
 348
 349static int
 350bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 351			char *section_name, int idx)
 352{
 353	struct bpf_program prog, *progs;
 354	int nr_progs, err;
 355
 356	err = bpf_program__init(data, size, section_name, idx, &prog);
 357	if (err)
 358		return err;
 359
 360	progs = obj->programs;
 361	nr_progs = obj->nr_programs;
 362
 363	progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
 364	if (!progs) {
 365		/*
 366		 * In this case the original obj->programs
 367		 * is still valid, so don't need special treat for
 368		 * bpf_close_object().
 369		 */
 370		pr_warning("failed to alloc a new program under section '%s'\n",
 371			   section_name);
 372		bpf_program__exit(&prog);
 373		return -ENOMEM;
 374	}
 375
 376	pr_debug("found program %s\n", prog.section_name);
 377	obj->programs = progs;
 378	obj->nr_programs = nr_progs + 1;
 379	prog.obj = obj;
 380	progs[nr_progs] = prog;
 381	return 0;
 382}
 383
 384static int
 385bpf_object__init_prog_names(struct bpf_object *obj)
 386{
 387	Elf_Data *symbols = obj->efile.symbols;
 388	struct bpf_program *prog;
 389	size_t pi, si;
 390
 391	for (pi = 0; pi < obj->nr_programs; pi++) {
 392		const char *name = NULL;
 393
 394		prog = &obj->programs[pi];
 395		if (prog->idx == obj->efile.text_shndx) {
 396			name = ".text";
 397			goto skip_search;
 398		}
 399
 400		for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 401		     si++) {
 402			GElf_Sym sym;
 403
 404			if (!gelf_getsym(symbols, si, &sym))
 405				continue;
 406			if (sym.st_shndx != prog->idx)
 407				continue;
 408			if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 409				continue;
 410
 411			name = elf_strptr(obj->efile.elf,
 412					  obj->efile.strtabidx,
 413					  sym.st_name);
 414			if (!name) {
 415				pr_warning("failed to get sym name string for prog %s\n",
 416					   prog->section_name);
 417				return -LIBBPF_ERRNO__LIBELF;
 418			}
 419		}
 420
 421		if (!name) {
 422			pr_warning("failed to find sym for prog %s\n",
 423				   prog->section_name);
 424			return -EINVAL;
 425		}
 426skip_search:
 427		prog->name = strdup(name);
 428		if (!prog->name) {
 429			pr_warning("failed to allocate memory for prog sym %s\n",
 430				   name);
 431			return -ENOMEM;
 432		}
 433	}
 434
 435	return 0;
 436}
 437
 438static struct bpf_object *bpf_object__new(const char *path,
 439					  void *obj_buf,
 440					  size_t obj_buf_sz)
 441{
 442	struct bpf_object *obj;
 443
 444	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 445	if (!obj) {
 446		pr_warning("alloc memory failed for %s\n", path);
 447		return ERR_PTR(-ENOMEM);
 448	}
 449
 450	strcpy(obj->path, path);
 451	obj->efile.fd = -1;
 452
 453	/*
 454	 * Caller of this function should also calls
 455	 * bpf_object__elf_finish() after data collection to return
 456	 * obj_buf to user. If not, we should duplicate the buffer to
 457	 * avoid user freeing them before elf finish.
 458	 */
 459	obj->efile.obj_buf = obj_buf;
 460	obj->efile.obj_buf_sz = obj_buf_sz;
 461	obj->efile.maps_shndx = -1;
 462
 463	obj->loaded = false;
 464
 465	INIT_LIST_HEAD(&obj->list);
 466	list_add(&obj->list, &bpf_objects_list);
 467	return obj;
 468}
 469
 470static void bpf_object__elf_finish(struct bpf_object *obj)
 471{
 472	if (!obj_elf_valid(obj))
 473		return;
 474
 475	if (obj->efile.elf) {
 476		elf_end(obj->efile.elf);
 477		obj->efile.elf = NULL;
 478	}
 479	obj->efile.symbols = NULL;
 480
 481	zfree(&obj->efile.reloc);
 482	obj->efile.nr_reloc = 0;
 483	zclose(obj->efile.fd);
 484	obj->efile.obj_buf = NULL;
 485	obj->efile.obj_buf_sz = 0;
 486}
 487
 488static int bpf_object__elf_init(struct bpf_object *obj)
 489{
 490	int err = 0;
 491	GElf_Ehdr *ep;
 492
 493	if (obj_elf_valid(obj)) {
 494		pr_warning("elf init: internal error\n");
 495		return -LIBBPF_ERRNO__LIBELF;
 496	}
 497
 498	if (obj->efile.obj_buf_sz > 0) {
 499		/*
 500		 * obj_buf should have been validated by
 501		 * bpf_object__open_buffer().
 502		 */
 503		obj->efile.elf = elf_memory(obj->efile.obj_buf,
 504					    obj->efile.obj_buf_sz);
 505	} else {
 506		obj->efile.fd = open(obj->path, O_RDONLY);
 507		if (obj->efile.fd < 0) {
 508			pr_warning("failed to open %s: %s\n", obj->path,
 509					strerror(errno));
 510			return -errno;
 511		}
 512
 513		obj->efile.elf = elf_begin(obj->efile.fd,
 514				LIBBPF_ELF_C_READ_MMAP,
 515				NULL);
 516	}
 517
 518	if (!obj->efile.elf) {
 519		pr_warning("failed to open %s as ELF file\n",
 520				obj->path);
 521		err = -LIBBPF_ERRNO__LIBELF;
 522		goto errout;
 523	}
 524
 525	if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 526		pr_warning("failed to get EHDR from %s\n",
 527				obj->path);
 528		err = -LIBBPF_ERRNO__FORMAT;
 529		goto errout;
 530	}
 531	ep = &obj->efile.ehdr;
 532
 533	/* Old LLVM set e_machine to EM_NONE */
 534	if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
 535		pr_warning("%s is not an eBPF object file\n",
 536			obj->path);
 537		err = -LIBBPF_ERRNO__FORMAT;
 538		goto errout;
 539	}
 540
 541	return 0;
 542errout:
 543	bpf_object__elf_finish(obj);
 544	return err;
 545}
 546
 547static int
 548bpf_object__check_endianness(struct bpf_object *obj)
 549{
 550	static unsigned int const endian = 1;
 551
 552	switch (obj->efile.ehdr.e_ident[EI_DATA]) {
 553	case ELFDATA2LSB:
 554		/* We are big endian, BPF obj is little endian. */
 555		if (*(unsigned char const *)&endian != 1)
 556			goto mismatch;
 557		break;
 558
 559	case ELFDATA2MSB:
 560		/* We are little endian, BPF obj is big endian. */
 561		if (*(unsigned char const *)&endian != 0)
 562			goto mismatch;
 563		break;
 564	default:
 565		return -LIBBPF_ERRNO__ENDIAN;
 566	}
 567
 568	return 0;
 569
 570mismatch:
 571	pr_warning("Error: endianness mismatch.\n");
 572	return -LIBBPF_ERRNO__ENDIAN;
 573}
 574
 575static int
 576bpf_object__init_license(struct bpf_object *obj,
 577			 void *data, size_t size)
 578{
 579	memcpy(obj->license, data,
 580	       min(size, sizeof(obj->license) - 1));
 581	pr_debug("license of %s is %s\n", obj->path, obj->license);
 582	return 0;
 583}
 584
 585static int
 586bpf_object__init_kversion(struct bpf_object *obj,
 587			  void *data, size_t size)
 588{
 589	u32 kver;
 590
 591	if (size != sizeof(kver)) {
 592		pr_warning("invalid kver section in %s\n", obj->path);
 593		return -LIBBPF_ERRNO__FORMAT;
 594	}
 595	memcpy(&kver, data, sizeof(kver));
 596	obj->kern_version = kver;
 597	pr_debug("kernel version of %s is %x\n", obj->path,
 598		 obj->kern_version);
 599	return 0;
 600}
 601
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 602static int compare_bpf_map(const void *_a, const void *_b)
 603{
 604	const struct bpf_map *a = _a;
 605	const struct bpf_map *b = _b;
 606
 607	return a->offset - b->offset;
 608}
 609
 610static int
 611bpf_object__init_maps(struct bpf_object *obj)
 612{
 613	int i, map_idx, map_def_sz, nr_maps = 0;
 614	Elf_Scn *scn;
 615	Elf_Data *data;
 616	Elf_Data *symbols = obj->efile.symbols;
 617
 618	if (obj->efile.maps_shndx < 0)
 619		return -EINVAL;
 620	if (!symbols)
 621		return -EINVAL;
 622
 623	scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 624	if (scn)
 625		data = elf_getdata(scn, NULL);
 626	if (!scn || !data) {
 627		pr_warning("failed to get Elf_Data from map section %d\n",
 628			   obj->efile.maps_shndx);
 629		return -EINVAL;
 630	}
 631
 632	/*
 633	 * Count number of maps. Each map has a name.
 634	 * Array of maps is not supported: only the first element is
 635	 * considered.
 636	 *
 637	 * TODO: Detect array of map and report error.
 638	 */
 639	for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 640		GElf_Sym sym;
 641
 642		if (!gelf_getsym(symbols, i, &sym))
 643			continue;
 644		if (sym.st_shndx != obj->efile.maps_shndx)
 645			continue;
 646		nr_maps++;
 647	}
 648
 649	/* Alloc obj->maps and fill nr_maps. */
 650	pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
 651		 nr_maps, data->d_size);
 652
 653	if (!nr_maps)
 654		return 0;
 655
 656	/* Assume equally sized map definitions */
 657	map_def_sz = data->d_size / nr_maps;
 658	if (!data->d_size || (data->d_size % nr_maps) != 0) {
 659		pr_warning("unable to determine map definition size "
 660			   "section %s, %d maps in %zd bytes\n",
 661			   obj->path, nr_maps, data->d_size);
 662		return -EINVAL;
 663	}
 664
 665	obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
 666	if (!obj->maps) {
 667		pr_warning("alloc maps for object failed\n");
 668		return -ENOMEM;
 669	}
 670	obj->nr_maps = nr_maps;
 671
 672	/*
 673	 * fill all fd with -1 so won't close incorrect
 674	 * fd (fd=0 is stdin) when failure (zclose won't close
 675	 * negative fd)).
 676	 */
 677	for (i = 0; i < nr_maps; i++)
 678		obj->maps[i].fd = -1;
 679
 680	/*
 681	 * Fill obj->maps using data in "maps" section.
 682	 */
 683	for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 684		GElf_Sym sym;
 685		const char *map_name;
 686		struct bpf_map_def *def;
 687
 688		if (!gelf_getsym(symbols, i, &sym))
 689			continue;
 690		if (sym.st_shndx != obj->efile.maps_shndx)
 691			continue;
 692
 693		map_name = elf_strptr(obj->efile.elf,
 694				      obj->efile.strtabidx,
 695				      sym.st_name);
 696		obj->maps[map_idx].offset = sym.st_value;
 697		if (sym.st_value + map_def_sz > data->d_size) {
 698			pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
 699				   obj->path, map_name);
 700			return -EINVAL;
 701		}
 702
 703		obj->maps[map_idx].name = strdup(map_name);
 704		if (!obj->maps[map_idx].name) {
 705			pr_warning("failed to alloc map name\n");
 706			return -ENOMEM;
 707		}
 708		pr_debug("map %d is \"%s\"\n", map_idx,
 709			 obj->maps[map_idx].name);
 710		def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
 711		/*
 712		 * If the definition of the map in the object file fits in
 713		 * bpf_map_def, copy it.  Any extra fields in our version
 714		 * of bpf_map_def will default to zero as a result of the
 715		 * calloc above.
 716		 */
 717		if (map_def_sz <= sizeof(struct bpf_map_def)) {
 718			memcpy(&obj->maps[map_idx].def, def, map_def_sz);
 719		} else {
 720			/*
 721			 * Here the map structure being read is bigger than what
 722			 * we expect, truncate if the excess bits are all zero.
 723			 * If they are not zero, reject this map as
 724			 * incompatible.
 725			 */
 726			char *b;
 727			for (b = ((char *)def) + sizeof(struct bpf_map_def);
 728			     b < ((char *)def) + map_def_sz; b++) {
 729				if (*b != 0) {
 730					pr_warning("maps section in %s: \"%s\" "
 731						   "has unrecognized, non-zero "
 732						   "options\n",
 733						   obj->path, map_name);
 734					return -EINVAL;
 735				}
 736			}
 737			memcpy(&obj->maps[map_idx].def, def,
 738			       sizeof(struct bpf_map_def));
 739		}
 740		map_idx++;
 741	}
 742
 743	qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
 744	return 0;
 745}
 746
 747static bool section_have_execinstr(struct bpf_object *obj, int idx)
 748{
 749	Elf_Scn *scn;
 750	GElf_Shdr sh;
 751
 752	scn = elf_getscn(obj->efile.elf, idx);
 753	if (!scn)
 754		return false;
 755
 756	if (gelf_getshdr(scn, &sh) != &sh)
 757		return false;
 758
 759	if (sh.sh_flags & SHF_EXECINSTR)
 760		return true;
 761
 762	return false;
 763}
 764
 765static int bpf_object__elf_collect(struct bpf_object *obj)
 766{
 767	Elf *elf = obj->efile.elf;
 768	GElf_Ehdr *ep = &obj->efile.ehdr;
 769	Elf_Scn *scn = NULL;
 770	int idx = 0, err = 0;
 771
 772	/* Elf is corrupted/truncated, avoid calling elf_strptr. */
 773	if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
 774		pr_warning("failed to get e_shstrndx from %s\n",
 775			   obj->path);
 776		return -LIBBPF_ERRNO__FORMAT;
 777	}
 778
 779	while ((scn = elf_nextscn(elf, scn)) != NULL) {
 780		char *name;
 781		GElf_Shdr sh;
 782		Elf_Data *data;
 783
 784		idx++;
 785		if (gelf_getshdr(scn, &sh) != &sh) {
 786			pr_warning("failed to get section(%d) header from %s\n",
 787				   idx, obj->path);
 788			err = -LIBBPF_ERRNO__FORMAT;
 789			goto out;
 790		}
 791
 792		name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 793		if (!name) {
 794			pr_warning("failed to get section(%d) name from %s\n",
 795				   idx, obj->path);
 796			err = -LIBBPF_ERRNO__FORMAT;
 797			goto out;
 798		}
 799
 800		data = elf_getdata(scn, 0);
 801		if (!data) {
 802			pr_warning("failed to get section(%d) data from %s(%s)\n",
 803				   idx, name, obj->path);
 804			err = -LIBBPF_ERRNO__FORMAT;
 805			goto out;
 806		}
 807		pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 808			 idx, name, (unsigned long)data->d_size,
 809			 (int)sh.sh_link, (unsigned long)sh.sh_flags,
 810			 (int)sh.sh_type);
 811
 812		if (strcmp(name, "license") == 0)
 813			err = bpf_object__init_license(obj,
 814						       data->d_buf,
 815						       data->d_size);
 816		else if (strcmp(name, "version") == 0)
 817			err = bpf_object__init_kversion(obj,
 818							data->d_buf,
 819							data->d_size);
 820		else if (strcmp(name, "maps") == 0)
 821			obj->efile.maps_shndx = idx;
 822		else if (sh.sh_type == SHT_SYMTAB) {
 823			if (obj->efile.symbols) {
 824				pr_warning("bpf: multiple SYMTAB in %s\n",
 825					   obj->path);
 826				err = -LIBBPF_ERRNO__FORMAT;
 827			} else {
 828				obj->efile.symbols = data;
 829				obj->efile.strtabidx = sh.sh_link;
 830			}
 831		} else if ((sh.sh_type == SHT_PROGBITS) &&
 832			   (sh.sh_flags & SHF_EXECINSTR) &&
 833			   (data->d_size > 0)) {
 834			if (strcmp(name, ".text") == 0)
 835				obj->efile.text_shndx = idx;
 836			err = bpf_object__add_program(obj, data->d_buf,
 837						      data->d_size, name, idx);
 838			if (err) {
 839				char errmsg[STRERR_BUFSIZE];
 840
 841				strerror_r(-err, errmsg, sizeof(errmsg));
 842				pr_warning("failed to alloc program %s (%s): %s",
 843					   name, obj->path, errmsg);
 844			}
 845		} else if (sh.sh_type == SHT_REL) {
 846			void *reloc = obj->efile.reloc;
 847			int nr_reloc = obj->efile.nr_reloc + 1;
 848			int sec = sh.sh_info; /* points to other section */
 849
 850			/* Only do relo for section with exec instructions */
 851			if (!section_have_execinstr(obj, sec)) {
 852				pr_debug("skip relo %s(%d) for section(%d)\n",
 853					 name, idx, sec);
 854				continue;
 855			}
 856
 857			reloc = realloc(reloc,
 858					sizeof(*obj->efile.reloc) * nr_reloc);
 859			if (!reloc) {
 860				pr_warning("realloc failed\n");
 861				err = -ENOMEM;
 862			} else {
 863				int n = nr_reloc - 1;
 864
 865				obj->efile.reloc = reloc;
 866				obj->efile.nr_reloc = nr_reloc;
 867
 868				obj->efile.reloc[n].shdr = sh;
 869				obj->efile.reloc[n].data = data;
 870			}
 871		} else {
 872			pr_debug("skip section(%d) %s\n", idx, name);
 873		}
 874		if (err)
 875			goto out;
 876	}
 877
 878	if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
 879		pr_warning("Corrupted ELF file: index of strtab invalid\n");
 880		return LIBBPF_ERRNO__FORMAT;
 881	}
 882	if (obj->efile.maps_shndx >= 0) {
 883		err = bpf_object__init_maps(obj);
 884		if (err)
 885			goto out;
 886	}
 887	err = bpf_object__init_prog_names(obj);
 888out:
 889	return err;
 890}
 891
 892static struct bpf_program *
 893bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
 894{
 895	struct bpf_program *prog;
 896	size_t i;
 897
 898	for (i = 0; i < obj->nr_programs; i++) {
 899		prog = &obj->programs[i];
 900		if (prog->idx == idx)
 901			return prog;
 902	}
 903	return NULL;
 904}
 905
 906static int
 907bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
 908			   Elf_Data *data, struct bpf_object *obj)
 
 
 909{
 910	Elf_Data *symbols = obj->efile.symbols;
 911	int text_shndx = obj->efile.text_shndx;
 912	int maps_shndx = obj->efile.maps_shndx;
 913	struct bpf_map *maps = obj->maps;
 914	size_t nr_maps = obj->nr_maps;
 915	int i, nrels;
 916
 917	pr_debug("collecting relocating info for: '%s'\n",
 918		 prog->section_name);
 919	nrels = shdr->sh_size / shdr->sh_entsize;
 920
 921	prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
 922	if (!prog->reloc_desc) {
 923		pr_warning("failed to alloc memory in relocation\n");
 924		return -ENOMEM;
 925	}
 926	prog->nr_reloc = nrels;
 927
 928	for (i = 0; i < nrels; i++) {
 929		GElf_Sym sym;
 930		GElf_Rel rel;
 931		unsigned int insn_idx;
 932		struct bpf_insn *insns = prog->insns;
 933		size_t map_idx;
 934
 935		if (!gelf_getrel(data, i, &rel)) {
 936			pr_warning("relocation: failed to get %d reloc\n", i);
 937			return -LIBBPF_ERRNO__FORMAT;
 938		}
 939
 940		if (!gelf_getsym(symbols,
 941				 GELF_R_SYM(rel.r_info),
 942				 &sym)) {
 943			pr_warning("relocation: symbol %"PRIx64" not found\n",
 944				   GELF_R_SYM(rel.r_info));
 945			return -LIBBPF_ERRNO__FORMAT;
 946		}
 947		pr_debug("relo for %lld value %lld name %d\n",
 948			 (long long) (rel.r_info >> 32),
 949			 (long long) sym.st_value, sym.st_name);
 950
 951		if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
 952			pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
 953				   prog->section_name, sym.st_shndx);
 954			return -LIBBPF_ERRNO__RELOC;
 955		}
 956
 957		insn_idx = rel.r_offset / sizeof(struct bpf_insn);
 958		pr_debug("relocation: insn_idx=%u\n", insn_idx);
 959
 960		if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
 961			if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
 962				pr_warning("incorrect bpf_call opcode\n");
 963				return -LIBBPF_ERRNO__RELOC;
 964			}
 965			prog->reloc_desc[i].type = RELO_CALL;
 966			prog->reloc_desc[i].insn_idx = insn_idx;
 967			prog->reloc_desc[i].text_off = sym.st_value;
 968			continue;
 969		}
 970
 971		if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
 972			pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
 973				   insn_idx, insns[insn_idx].code);
 974			return -LIBBPF_ERRNO__RELOC;
 975		}
 976
 977		/* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
 978		for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 979			if (maps[map_idx].offset == sym.st_value) {
 980				pr_debug("relocation: find map %zd (%s) for insn %u\n",
 981					 map_idx, maps[map_idx].name, insn_idx);
 982				break;
 983			}
 984		}
 985
 986		if (map_idx >= nr_maps) {
 987			pr_warning("bpf relocation: map_idx %d large than %d\n",
 988				   (int)map_idx, (int)nr_maps - 1);
 989			return -LIBBPF_ERRNO__RELOC;
 990		}
 991
 992		prog->reloc_desc[i].type = RELO_LD64;
 993		prog->reloc_desc[i].insn_idx = insn_idx;
 994		prog->reloc_desc[i].map_idx = map_idx;
 995	}
 996	return 0;
 997}
 998
 999static int
1000bpf_object__create_maps(struct bpf_object *obj)
1001{
1002	unsigned int i;
1003
1004	for (i = 0; i < obj->nr_maps; i++) {
1005		struct bpf_map_def *def = &obj->maps[i].def;
1006		int *pfd = &obj->maps[i].fd;
1007
1008		*pfd = bpf_create_map_name(def->type,
1009					   obj->maps[i].name,
1010					   def->key_size,
1011					   def->value_size,
1012					   def->max_entries,
1013					   def->map_flags);
1014		if (*pfd < 0) {
1015			size_t j;
1016			int err = *pfd;
1017
1018			pr_warning("failed to create map (name: '%s'): %s\n",
1019				   obj->maps[i].name,
1020				   strerror(errno));
1021			for (j = 0; j < i; j++)
1022				zclose(obj->maps[j].fd);
1023			return err;
1024		}
1025		pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd);
1026	}
1027
1028	return 0;
1029}
1030
1031static int
1032bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1033			struct reloc_desc *relo)
1034{
1035	struct bpf_insn *insn, *new_insn;
1036	struct bpf_program *text;
1037	size_t new_cnt;
1038
1039	if (relo->type != RELO_CALL)
1040		return -LIBBPF_ERRNO__RELOC;
1041
1042	if (prog->idx == obj->efile.text_shndx) {
1043		pr_warning("relo in .text insn %d into off %d\n",
1044			   relo->insn_idx, relo->text_off);
1045		return -LIBBPF_ERRNO__RELOC;
1046	}
1047
1048	if (prog->main_prog_cnt == 0) {
1049		text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1050		if (!text) {
1051			pr_warning("no .text section found yet relo into text exist\n");
1052			return -LIBBPF_ERRNO__RELOC;
1053		}
1054		new_cnt = prog->insns_cnt + text->insns_cnt;
1055		new_insn = realloc(prog->insns, new_cnt * sizeof(*insn));
1056		if (!new_insn) {
1057			pr_warning("oom in prog realloc\n");
1058			return -ENOMEM;
1059		}
1060		memcpy(new_insn + prog->insns_cnt, text->insns,
1061		       text->insns_cnt * sizeof(*insn));
1062		prog->insns = new_insn;
1063		prog->main_prog_cnt = prog->insns_cnt;
1064		prog->insns_cnt = new_cnt;
1065		pr_debug("added %zd insn from %s to prog %s\n",
1066			 text->insns_cnt, text->section_name,
1067			 prog->section_name);
1068	}
1069	insn = &prog->insns[relo->insn_idx];
1070	insn->imm += prog->main_prog_cnt - relo->insn_idx;
1071	return 0;
1072}
1073
1074static int
1075bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1076{
1077	int i, err;
1078
1079	if (!prog || !prog->reloc_desc)
1080		return 0;
1081
1082	for (i = 0; i < prog->nr_reloc; i++) {
1083		if (prog->reloc_desc[i].type == RELO_LD64) {
1084			struct bpf_insn *insns = prog->insns;
1085			int insn_idx, map_idx;
1086
1087			insn_idx = prog->reloc_desc[i].insn_idx;
1088			map_idx = prog->reloc_desc[i].map_idx;
1089
1090			if (insn_idx >= (int)prog->insns_cnt) {
1091				pr_warning("relocation out of range: '%s'\n",
1092					   prog->section_name);
1093				return -LIBBPF_ERRNO__RELOC;
1094			}
1095			insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1096			insns[insn_idx].imm = obj->maps[map_idx].fd;
1097		} else {
1098			err = bpf_program__reloc_text(prog, obj,
1099						      &prog->reloc_desc[i]);
1100			if (err)
1101				return err;
1102		}
 
 
1103	}
1104
1105	zfree(&prog->reloc_desc);
1106	prog->nr_reloc = 0;
1107	return 0;
1108}
1109
1110
1111static int
1112bpf_object__relocate(struct bpf_object *obj)
1113{
1114	struct bpf_program *prog;
1115	size_t i;
1116	int err;
1117
1118	for (i = 0; i < obj->nr_programs; i++) {
1119		prog = &obj->programs[i];
1120
1121		err = bpf_program__relocate(prog, obj);
1122		if (err) {
1123			pr_warning("failed to relocate '%s'\n",
1124				   prog->section_name);
1125			return err;
1126		}
1127	}
1128	return 0;
1129}
1130
1131static int bpf_object__collect_reloc(struct bpf_object *obj)
1132{
1133	int i, err;
1134
1135	if (!obj_elf_valid(obj)) {
1136		pr_warning("Internal error: elf object is closed\n");
1137		return -LIBBPF_ERRNO__INTERNAL;
1138	}
1139
1140	for (i = 0; i < obj->efile.nr_reloc; i++) {
1141		GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1142		Elf_Data *data = obj->efile.reloc[i].data;
1143		int idx = shdr->sh_info;
1144		struct bpf_program *prog;
 
1145
1146		if (shdr->sh_type != SHT_REL) {
1147			pr_warning("internal error at %d\n", __LINE__);
1148			return -LIBBPF_ERRNO__INTERNAL;
1149		}
1150
1151		prog = bpf_object__find_prog_by_idx(obj, idx);
1152		if (!prog) {
1153			pr_warning("relocation failed: no section(%d)\n", idx);
 
1154			return -LIBBPF_ERRNO__RELOC;
1155		}
1156
1157		err = bpf_program__collect_reloc(prog,
1158						 shdr, data,
1159						 obj);
 
1160		if (err)
1161			return err;
1162	}
1163	return 0;
1164}
1165
1166static int
1167load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1168	     const char *name, struct bpf_insn *insns, int insns_cnt,
1169	     char *license, u32 kern_version, int *pfd)
1170{
1171	struct bpf_load_program_attr load_attr;
1172	char *log_buf;
1173	int ret;
 
1174
1175	memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1176	load_attr.prog_type = type;
1177	load_attr.expected_attach_type = expected_attach_type;
1178	load_attr.name = name;
1179	load_attr.insns = insns;
1180	load_attr.insns_cnt = insns_cnt;
1181	load_attr.license = license;
1182	load_attr.kern_version = kern_version;
1183
1184	if (!load_attr.insns || !load_attr.insns_cnt)
1185		return -EINVAL;
1186
1187	log_buf = malloc(BPF_LOG_BUF_SIZE);
1188	if (!log_buf)
1189		pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1190
1191	ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
 
1192
1193	if (ret >= 0) {
1194		*pfd = ret;
1195		ret = 0;
1196		goto out;
1197	}
1198
1199	ret = -LIBBPF_ERRNO__LOAD;
1200	pr_warning("load bpf program failed: %s\n", strerror(errno));
1201
1202	if (log_buf && log_buf[0] != '\0') {
1203		ret = -LIBBPF_ERRNO__VERIFY;
1204		pr_warning("-- BEGIN DUMP LOG ---\n");
1205		pr_warning("\n%s\n", log_buf);
1206		pr_warning("-- END LOG --\n");
1207	} else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1208		pr_warning("Program too large (%zu insns), at most %d insns\n",
1209			   load_attr.insns_cnt, BPF_MAXINSNS);
1210		ret = -LIBBPF_ERRNO__PROG2BIG;
1211	} else {
1212		/* Wrong program type? */
1213		if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1214			int fd;
1215
1216			load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1217			load_attr.expected_attach_type = 0;
1218			fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1219			if (fd >= 0) {
1220				close(fd);
1221				ret = -LIBBPF_ERRNO__PROGTYPE;
1222				goto out;
1223			}
1224		}
1225
1226		if (log_buf)
1227			ret = -LIBBPF_ERRNO__KVER;
1228	}
1229
1230out:
1231	free(log_buf);
1232	return ret;
1233}
1234
1235static int
1236bpf_program__load(struct bpf_program *prog,
1237		  char *license, u32 kern_version)
1238{
1239	int err = 0, fd, i;
1240
1241	if (prog->instances.nr < 0 || !prog->instances.fds) {
1242		if (prog->preprocessor) {
1243			pr_warning("Internal error: can't load program '%s'\n",
1244				   prog->section_name);
1245			return -LIBBPF_ERRNO__INTERNAL;
1246		}
1247
1248		prog->instances.fds = malloc(sizeof(int));
1249		if (!prog->instances.fds) {
1250			pr_warning("Not enough memory for BPF fds\n");
1251			return -ENOMEM;
1252		}
1253		prog->instances.nr = 1;
1254		prog->instances.fds[0] = -1;
1255	}
1256
1257	if (!prog->preprocessor) {
1258		if (prog->instances.nr != 1) {
1259			pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1260				   prog->section_name, prog->instances.nr);
1261		}
1262		err = load_program(prog->type, prog->expected_attach_type,
1263				   prog->name, prog->insns, prog->insns_cnt,
1264				   license, kern_version, &fd);
1265		if (!err)
1266			prog->instances.fds[0] = fd;
1267		goto out;
1268	}
1269
1270	for (i = 0; i < prog->instances.nr; i++) {
1271		struct bpf_prog_prep_result result;
1272		bpf_program_prep_t preprocessor = prog->preprocessor;
1273
1274		bzero(&result, sizeof(result));
1275		err = preprocessor(prog, i, prog->insns,
1276				   prog->insns_cnt, &result);
1277		if (err) {
1278			pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1279				   i, prog->section_name);
1280			goto out;
1281		}
1282
1283		if (!result.new_insn_ptr || !result.new_insn_cnt) {
1284			pr_debug("Skip loading the %dth instance of program '%s'\n",
1285				 i, prog->section_name);
1286			prog->instances.fds[i] = -1;
1287			if (result.pfd)
1288				*result.pfd = -1;
1289			continue;
1290		}
1291
1292		err = load_program(prog->type, prog->expected_attach_type,
1293				   prog->name, result.new_insn_ptr,
1294				   result.new_insn_cnt,
1295				   license, kern_version, &fd);
1296
1297		if (err) {
1298			pr_warning("Loading the %dth instance of program '%s' failed\n",
1299					i, prog->section_name);
1300			goto out;
1301		}
1302
1303		if (result.pfd)
1304			*result.pfd = fd;
1305		prog->instances.fds[i] = fd;
1306	}
1307out:
1308	if (err)
1309		pr_warning("failed to load program '%s'\n",
1310			   prog->section_name);
1311	zfree(&prog->insns);
1312	prog->insns_cnt = 0;
1313	return err;
1314}
1315
1316static int
1317bpf_object__load_progs(struct bpf_object *obj)
1318{
1319	size_t i;
1320	int err;
1321
1322	for (i = 0; i < obj->nr_programs; i++) {
1323		if (obj->programs[i].idx == obj->efile.text_shndx)
1324			continue;
1325		err = bpf_program__load(&obj->programs[i],
1326					obj->license,
1327					obj->kern_version);
1328		if (err)
1329			return err;
1330	}
1331	return 0;
1332}
1333
1334static int bpf_object__validate(struct bpf_object *obj)
1335{
1336	if (obj->kern_version == 0) {
1337		pr_warning("%s doesn't provide kernel version\n",
1338			   obj->path);
1339		return -LIBBPF_ERRNO__KVERSION;
1340	}
1341	return 0;
1342}
1343
1344static struct bpf_object *
1345__bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz)
1346{
1347	struct bpf_object *obj;
1348	int err;
1349
1350	if (elf_version(EV_CURRENT) == EV_NONE) {
1351		pr_warning("failed to init libelf for %s\n", path);
1352		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1353	}
1354
1355	obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1356	if (IS_ERR(obj))
1357		return obj;
1358
1359	CHECK_ERR(bpf_object__elf_init(obj), err, out);
1360	CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1361	CHECK_ERR(bpf_object__elf_collect(obj), err, out);
1362	CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1363	CHECK_ERR(bpf_object__validate(obj), err, out);
1364
1365	bpf_object__elf_finish(obj);
1366	return obj;
1367out:
1368	bpf_object__close(obj);
1369	return ERR_PTR(err);
1370}
1371
1372struct bpf_object *bpf_object__open(const char *path)
1373{
1374	/* param validation */
1375	if (!path)
1376		return NULL;
1377
1378	pr_debug("loading %s\n", path);
1379
1380	return __bpf_object__open(path, NULL, 0);
1381}
1382
1383struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1384					   size_t obj_buf_sz,
1385					   const char *name)
1386{
1387	char tmp_name[64];
1388
1389	/* param validation */
1390	if (!obj_buf || obj_buf_sz <= 0)
1391		return NULL;
1392
1393	if (!name) {
1394		snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1395			 (unsigned long)obj_buf,
1396			 (unsigned long)obj_buf_sz);
1397		tmp_name[sizeof(tmp_name) - 1] = '\0';
1398		name = tmp_name;
1399	}
1400	pr_debug("loading object '%s' from buffer\n",
1401		 name);
1402
1403	return __bpf_object__open(name, obj_buf, obj_buf_sz);
1404}
1405
1406int bpf_object__unload(struct bpf_object *obj)
1407{
1408	size_t i;
1409
1410	if (!obj)
1411		return -EINVAL;
1412
1413	for (i = 0; i < obj->nr_maps; i++)
1414		zclose(obj->maps[i].fd);
1415
1416	for (i = 0; i < obj->nr_programs; i++)
1417		bpf_program__unload(&obj->programs[i]);
1418
1419	return 0;
1420}
1421
1422int bpf_object__load(struct bpf_object *obj)
1423{
1424	int err;
1425
1426	if (!obj)
1427		return -EINVAL;
1428
1429	if (obj->loaded) {
1430		pr_warning("object should not be loaded twice\n");
1431		return -EINVAL;
1432	}
1433
1434	obj->loaded = true;
1435
1436	CHECK_ERR(bpf_object__create_maps(obj), err, out);
1437	CHECK_ERR(bpf_object__relocate(obj), err, out);
1438	CHECK_ERR(bpf_object__load_progs(obj), err, out);
1439
1440	return 0;
1441out:
1442	bpf_object__unload(obj);
1443	pr_warning("failed to load object '%s'\n", obj->path);
1444	return err;
1445}
1446
1447static int check_path(const char *path)
1448{
1449	struct statfs st_fs;
1450	char *dname, *dir;
1451	int err = 0;
1452
1453	if (path == NULL)
1454		return -EINVAL;
1455
1456	dname = strdup(path);
1457	if (dname == NULL)
1458		return -ENOMEM;
1459
1460	dir = dirname(dname);
1461	if (statfs(dir, &st_fs)) {
1462		pr_warning("failed to statfs %s: %s\n", dir, strerror(errno));
1463		err = -errno;
1464	}
1465	free(dname);
1466
1467	if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1468		pr_warning("specified path %s is not on BPF FS\n", path);
1469		err = -EINVAL;
1470	}
1471
1472	return err;
1473}
1474
1475int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1476			      int instance)
1477{
1478	int err;
1479
1480	err = check_path(path);
1481	if (err)
1482		return err;
1483
1484	if (prog == NULL) {
1485		pr_warning("invalid program pointer\n");
1486		return -EINVAL;
1487	}
1488
1489	if (instance < 0 || instance >= prog->instances.nr) {
1490		pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1491			   instance, prog->section_name, prog->instances.nr);
1492		return -EINVAL;
1493	}
1494
1495	if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1496		pr_warning("failed to pin program: %s\n", strerror(errno));
1497		return -errno;
1498	}
1499	pr_debug("pinned program '%s'\n", path);
1500
1501	return 0;
1502}
1503
1504static int make_dir(const char *path)
1505{
1506	int err = 0;
1507
1508	if (mkdir(path, 0700) && errno != EEXIST)
1509		err = -errno;
1510
1511	if (err)
1512		pr_warning("failed to mkdir %s: %s\n", path, strerror(-err));
1513	return err;
1514}
1515
1516int bpf_program__pin(struct bpf_program *prog, const char *path)
1517{
1518	int i, err;
1519
1520	err = check_path(path);
1521	if (err)
1522		return err;
1523
1524	if (prog == NULL) {
1525		pr_warning("invalid program pointer\n");
1526		return -EINVAL;
1527	}
1528
1529	if (prog->instances.nr <= 0) {
1530		pr_warning("no instances of prog %s to pin\n",
1531			   prog->section_name);
1532		return -EINVAL;
1533	}
1534
1535	err = make_dir(path);
1536	if (err)
1537		return err;
1538
1539	for (i = 0; i < prog->instances.nr; i++) {
1540		char buf[PATH_MAX];
1541		int len;
1542
1543		len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1544		if (len < 0)
1545			return -EINVAL;
1546		else if (len >= PATH_MAX)
1547			return -ENAMETOOLONG;
1548
1549		err = bpf_program__pin_instance(prog, buf, i);
1550		if (err)
1551			return err;
1552	}
1553
1554	return 0;
1555}
1556
1557int bpf_map__pin(struct bpf_map *map, const char *path)
1558{
1559	int err;
1560
1561	err = check_path(path);
1562	if (err)
1563		return err;
1564
1565	if (map == NULL) {
1566		pr_warning("invalid map pointer\n");
1567		return -EINVAL;
1568	}
1569
1570	if (bpf_obj_pin(map->fd, path)) {
1571		pr_warning("failed to pin map: %s\n", strerror(errno));
1572		return -errno;
1573	}
1574
1575	pr_debug("pinned map '%s'\n", path);
1576	return 0;
1577}
1578
1579int bpf_object__pin(struct bpf_object *obj, const char *path)
1580{
1581	struct bpf_program *prog;
1582	struct bpf_map *map;
1583	int err;
1584
1585	if (!obj)
1586		return -ENOENT;
1587
1588	if (!obj->loaded) {
1589		pr_warning("object not yet loaded; load it first\n");
1590		return -ENOENT;
1591	}
1592
1593	err = make_dir(path);
1594	if (err)
1595		return err;
1596
1597	bpf_map__for_each(map, obj) {
1598		char buf[PATH_MAX];
1599		int len;
1600
1601		len = snprintf(buf, PATH_MAX, "%s/%s", path,
1602			       bpf_map__name(map));
1603		if (len < 0)
1604			return -EINVAL;
1605		else if (len >= PATH_MAX)
1606			return -ENAMETOOLONG;
1607
1608		err = bpf_map__pin(map, buf);
1609		if (err)
1610			return err;
1611	}
1612
1613	bpf_object__for_each_program(prog, obj) {
1614		char buf[PATH_MAX];
1615		int len;
1616
1617		len = snprintf(buf, PATH_MAX, "%s/%s", path,
1618			       prog->section_name);
1619		if (len < 0)
1620			return -EINVAL;
1621		else if (len >= PATH_MAX)
1622			return -ENAMETOOLONG;
1623
1624		err = bpf_program__pin(prog, buf);
1625		if (err)
1626			return err;
1627	}
1628
1629	return 0;
1630}
1631
1632void bpf_object__close(struct bpf_object *obj)
1633{
1634	size_t i;
1635
1636	if (!obj)
1637		return;
1638
1639	if (obj->clear_priv)
1640		obj->clear_priv(obj, obj->priv);
1641
1642	bpf_object__elf_finish(obj);
1643	bpf_object__unload(obj);
1644
1645	for (i = 0; i < obj->nr_maps; i++) {
1646		zfree(&obj->maps[i].name);
1647		if (obj->maps[i].clear_priv)
1648			obj->maps[i].clear_priv(&obj->maps[i],
1649						obj->maps[i].priv);
1650		obj->maps[i].priv = NULL;
1651		obj->maps[i].clear_priv = NULL;
1652	}
1653	zfree(&obj->maps);
1654	obj->nr_maps = 0;
1655
1656	if (obj->programs && obj->nr_programs) {
1657		for (i = 0; i < obj->nr_programs; i++)
1658			bpf_program__exit(&obj->programs[i]);
1659	}
1660	zfree(&obj->programs);
1661
1662	list_del(&obj->list);
1663	free(obj);
1664}
1665
1666struct bpf_object *
1667bpf_object__next(struct bpf_object *prev)
1668{
1669	struct bpf_object *next;
1670
1671	if (!prev)
1672		next = list_first_entry(&bpf_objects_list,
1673					struct bpf_object,
1674					list);
1675	else
1676		next = list_next_entry(prev, list);
1677
1678	/* Empty list is noticed here so don't need checking on entry. */
1679	if (&next->list == &bpf_objects_list)
1680		return NULL;
1681
1682	return next;
1683}
1684
1685const char *bpf_object__name(struct bpf_object *obj)
1686{
1687	return obj ? obj->path : ERR_PTR(-EINVAL);
1688}
1689
1690unsigned int bpf_object__kversion(struct bpf_object *obj)
1691{
1692	return obj ? obj->kern_version : 0;
1693}
1694
1695int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1696			 bpf_object_clear_priv_t clear_priv)
1697{
1698	if (obj->priv && obj->clear_priv)
1699		obj->clear_priv(obj, obj->priv);
1700
1701	obj->priv = priv;
1702	obj->clear_priv = clear_priv;
1703	return 0;
1704}
1705
1706void *bpf_object__priv(struct bpf_object *obj)
1707{
1708	return obj ? obj->priv : ERR_PTR(-EINVAL);
1709}
1710
1711struct bpf_program *
1712bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1713{
1714	size_t idx;
1715
1716	if (!obj->programs)
1717		return NULL;
1718	/* First handler */
1719	if (prev == NULL)
1720		return &obj->programs[0];
1721
1722	if (prev->obj != obj) {
1723		pr_warning("error: program handler doesn't match object\n");
1724		return NULL;
1725	}
1726
1727	idx = (prev - obj->programs) + 1;
1728	if (idx >= obj->nr_programs)
1729		return NULL;
1730	return &obj->programs[idx];
1731}
1732
1733int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1734			  bpf_program_clear_priv_t clear_priv)
1735{
1736	if (prog->priv && prog->clear_priv)
1737		prog->clear_priv(prog, prog->priv);
1738
1739	prog->priv = priv;
1740	prog->clear_priv = clear_priv;
1741	return 0;
1742}
1743
1744void *bpf_program__priv(struct bpf_program *prog)
1745{
1746	return prog ? prog->priv : ERR_PTR(-EINVAL);
1747}
1748
1749const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1750{
1751	const char *title;
1752
1753	title = prog->section_name;
1754	if (needs_copy) {
1755		title = strdup(title);
1756		if (!title) {
1757			pr_warning("failed to strdup program title\n");
1758			return ERR_PTR(-ENOMEM);
1759		}
1760	}
1761
1762	return title;
1763}
1764
1765int bpf_program__fd(struct bpf_program *prog)
1766{
1767	return bpf_program__nth_fd(prog, 0);
1768}
1769
1770int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1771			  bpf_program_prep_t prep)
1772{
1773	int *instances_fds;
1774
1775	if (nr_instances <= 0 || !prep)
1776		return -EINVAL;
1777
1778	if (prog->instances.nr > 0 || prog->instances.fds) {
1779		pr_warning("Can't set pre-processor after loading\n");
1780		return -EINVAL;
1781	}
1782
1783	instances_fds = malloc(sizeof(int) * nr_instances);
1784	if (!instances_fds) {
1785		pr_warning("alloc memory failed for fds\n");
1786		return -ENOMEM;
1787	}
1788
1789	/* fill all fd with -1 */
1790	memset(instances_fds, -1, sizeof(int) * nr_instances);
1791
1792	prog->instances.nr = nr_instances;
1793	prog->instances.fds = instances_fds;
1794	prog->preprocessor = prep;
1795	return 0;
1796}
1797
1798int bpf_program__nth_fd(struct bpf_program *prog, int n)
1799{
1800	int fd;
1801
1802	if (n >= prog->instances.nr || n < 0) {
1803		pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
1804			   n, prog->section_name, prog->instances.nr);
1805		return -EINVAL;
1806	}
1807
1808	fd = prog->instances.fds[n];
1809	if (fd < 0) {
1810		pr_warning("%dth instance of program '%s' is invalid\n",
1811			   n, prog->section_name);
1812		return -ENOENT;
1813	}
1814
1815	return fd;
1816}
1817
1818void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
 
1819{
1820	prog->type = type;
1821}
1822
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1823static bool bpf_program__is_type(struct bpf_program *prog,
1824				 enum bpf_prog_type type)
1825{
1826	return prog ? (prog->type == type) : false;
1827}
1828
1829#define BPF_PROG_TYPE_FNS(NAME, TYPE)			\
1830int bpf_program__set_##NAME(struct bpf_program *prog)	\
1831{							\
1832	if (!prog)					\
1833		return -EINVAL;				\
1834	bpf_program__set_type(prog, TYPE);		\
1835	return 0;					\
1836}							\
1837							\
1838bool bpf_program__is_##NAME(struct bpf_program *prog)	\
1839{							\
1840	return bpf_program__is_type(prog, TYPE);	\
1841}							\
1842
1843BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
1844BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
1845BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
1846BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
1847BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
1848BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
1849BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
1850
1851static void bpf_program__set_expected_attach_type(struct bpf_program *prog,
1852						 enum bpf_attach_type type)
1853{
1854	prog->expected_attach_type = type;
1855}
1856
1857#define BPF_PROG_SEC_FULL(string, ptype, atype) \
1858	{ string, sizeof(string) - 1, ptype, atype }
1859
1860#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0)
1861
1862#define BPF_SA_PROG_SEC(string, ptype) \
1863	BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype)
1864
1865static const struct {
1866	const char *sec;
1867	size_t len;
1868	enum bpf_prog_type prog_type;
1869	enum bpf_attach_type expected_attach_type;
1870} section_names[] = {
1871	BPF_PROG_SEC("socket",		BPF_PROG_TYPE_SOCKET_FILTER),
1872	BPF_PROG_SEC("kprobe/",		BPF_PROG_TYPE_KPROBE),
1873	BPF_PROG_SEC("kretprobe/",	BPF_PROG_TYPE_KPROBE),
1874	BPF_PROG_SEC("classifier",	BPF_PROG_TYPE_SCHED_CLS),
1875	BPF_PROG_SEC("action",		BPF_PROG_TYPE_SCHED_ACT),
1876	BPF_PROG_SEC("tracepoint/",	BPF_PROG_TYPE_TRACEPOINT),
1877	BPF_PROG_SEC("xdp",		BPF_PROG_TYPE_XDP),
1878	BPF_PROG_SEC("perf_event",	BPF_PROG_TYPE_PERF_EVENT),
1879	BPF_PROG_SEC("cgroup/skb",	BPF_PROG_TYPE_CGROUP_SKB),
1880	BPF_PROG_SEC("cgroup/sock",	BPF_PROG_TYPE_CGROUP_SOCK),
1881	BPF_PROG_SEC("cgroup/dev",	BPF_PROG_TYPE_CGROUP_DEVICE),
1882	BPF_PROG_SEC("lwt_in",		BPF_PROG_TYPE_LWT_IN),
1883	BPF_PROG_SEC("lwt_out",		BPF_PROG_TYPE_LWT_OUT),
1884	BPF_PROG_SEC("lwt_xmit",	BPF_PROG_TYPE_LWT_XMIT),
1885	BPF_PROG_SEC("sockops",		BPF_PROG_TYPE_SOCK_OPS),
1886	BPF_PROG_SEC("sk_skb",		BPF_PROG_TYPE_SK_SKB),
1887	BPF_PROG_SEC("sk_msg",		BPF_PROG_TYPE_SK_MSG),
1888	BPF_SA_PROG_SEC("cgroup/bind4",	BPF_CGROUP_INET4_BIND),
1889	BPF_SA_PROG_SEC("cgroup/bind6",	BPF_CGROUP_INET6_BIND),
1890	BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT),
1891	BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT),
1892};
1893
1894#undef BPF_PROG_SEC
1895#undef BPF_PROG_SEC_FULL
1896#undef BPF_SA_PROG_SEC
1897
1898static int bpf_program__identify_section(struct bpf_program *prog)
1899{
1900	int i;
1901
1902	if (!prog->section_name)
1903		goto err;
1904
1905	for (i = 0; i < ARRAY_SIZE(section_names); i++)
1906		if (strncmp(prog->section_name, section_names[i].sec,
1907			    section_names[i].len) == 0)
1908			return i;
1909
1910err:
1911	pr_warning("failed to guess program type based on section name %s\n",
1912		   prog->section_name);
1913
1914	return -1;
 
 
1915}
1916
1917int bpf_map__fd(struct bpf_map *map)
1918{
1919	return map ? map->fd : -EINVAL;
1920}
1921
1922const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
1923{
1924	return map ? &map->def : ERR_PTR(-EINVAL);
1925}
1926
1927const char *bpf_map__name(struct bpf_map *map)
1928{
1929	return map ? map->name : NULL;
1930}
1931
1932int bpf_map__set_priv(struct bpf_map *map, void *priv,
1933		     bpf_map_clear_priv_t clear_priv)
1934{
1935	if (!map)
1936		return -EINVAL;
1937
1938	if (map->priv) {
1939		if (map->clear_priv)
1940			map->clear_priv(map, map->priv);
1941	}
1942
1943	map->priv = priv;
1944	map->clear_priv = clear_priv;
1945	return 0;
1946}
1947
1948void *bpf_map__priv(struct bpf_map *map)
1949{
1950	return map ? map->priv : ERR_PTR(-EINVAL);
1951}
1952
1953struct bpf_map *
1954bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
1955{
1956	size_t idx;
1957	struct bpf_map *s, *e;
1958
1959	if (!obj || !obj->maps)
1960		return NULL;
1961
1962	s = obj->maps;
1963	e = obj->maps + obj->nr_maps;
1964
1965	if (prev == NULL)
1966		return s;
1967
1968	if ((prev < s) || (prev >= e)) {
1969		pr_warning("error in %s: map handler doesn't belong to object\n",
1970			   __func__);
1971		return NULL;
1972	}
1973
1974	idx = (prev - obj->maps) + 1;
1975	if (idx >= obj->nr_maps)
1976		return NULL;
1977	return &obj->maps[idx];
1978}
1979
1980struct bpf_map *
1981bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
1982{
1983	struct bpf_map *pos;
1984
1985	bpf_map__for_each(pos, obj) {
1986		if (pos->name && !strcmp(pos->name, name))
1987			return pos;
1988	}
1989	return NULL;
1990}
1991
1992struct bpf_map *
1993bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
1994{
1995	int i;
1996
1997	for (i = 0; i < obj->nr_maps; i++) {
1998		if (obj->maps[i].offset == offset)
1999			return &obj->maps[i];
2000	}
2001	return ERR_PTR(-ENOENT);
2002}
2003
2004long libbpf_get_error(const void *ptr)
2005{
2006	if (IS_ERR(ptr))
2007		return PTR_ERR(ptr);
2008	return 0;
2009}
2010
2011int bpf_prog_load(const char *file, enum bpf_prog_type type,
2012		  struct bpf_object **pobj, int *prog_fd)
2013{
2014	struct bpf_prog_load_attr attr;
2015
2016	memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2017	attr.file = file;
2018	attr.prog_type = type;
2019	attr.expected_attach_type = 0;
2020
2021	return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2022}
2023
2024int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2025			struct bpf_object **pobj, int *prog_fd)
2026{
2027	struct bpf_program *prog, *first_prog = NULL;
2028	enum bpf_attach_type expected_attach_type;
2029	enum bpf_prog_type prog_type;
2030	struct bpf_object *obj;
2031	int section_idx;
2032	int err;
2033
2034	if (!attr)
2035		return -EINVAL;
2036
2037	obj = bpf_object__open(attr->file);
2038	if (IS_ERR_OR_NULL(obj))
2039		return -ENOENT;
2040
2041	bpf_object__for_each_program(prog, obj) {
2042		/*
2043		 * If type is not specified, try to guess it based on
2044		 * section name.
2045		 */
2046		prog_type = attr->prog_type;
2047		expected_attach_type = attr->expected_attach_type;
2048		if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2049			section_idx = bpf_program__identify_section(prog);
2050			if (section_idx < 0) {
2051				bpf_object__close(obj);
2052				return -EINVAL;
2053			}
2054			prog_type = section_names[section_idx].prog_type;
2055			expected_attach_type =
2056				section_names[section_idx].expected_attach_type;
2057		}
2058
2059		bpf_program__set_type(prog, prog_type);
2060		bpf_program__set_expected_attach_type(prog,
2061						      expected_attach_type);
2062
2063		if (prog->idx != obj->efile.text_shndx && !first_prog)
2064			first_prog = prog;
2065	}
2066
2067	if (!first_prog) {
2068		pr_warning("object file doesn't contain bpf program\n");
2069		bpf_object__close(obj);
2070		return -ENOENT;
2071	}
2072
2073	err = bpf_object__load(obj);
2074	if (err) {
2075		bpf_object__close(obj);
2076		return -EINVAL;
2077	}
2078
2079	*pobj = obj;
2080	*prog_fd = bpf_program__fd(first_prog);
2081	return 0;
2082}