Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: LGPL-2.1
2
3/*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public
13 * License as published by the Free Software Foundation;
14 * version 2.1 of the License (not later!)
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Lesser General Public License for more details.
20 *
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with this program; if not, see <http://www.gnu.org/licenses>
23 */
24
25#include <stdlib.h>
26#include <stdio.h>
27#include <stdarg.h>
28#include <libgen.h>
29#include <inttypes.h>
30#include <string.h>
31#include <unistd.h>
32#include <fcntl.h>
33#include <errno.h>
34#include <asm/unistd.h>
35#include <linux/err.h>
36#include <linux/kernel.h>
37#include <linux/bpf.h>
38#include <linux/list.h>
39#include <linux/limits.h>
40#include <sys/stat.h>
41#include <sys/types.h>
42#include <sys/vfs.h>
43#include <libelf.h>
44#include <gelf.h>
45
46#include "libbpf.h"
47#include "bpf.h"
48
49#ifndef EM_BPF
50#define EM_BPF 247
51#endif
52
53#ifndef BPF_FS_MAGIC
54#define BPF_FS_MAGIC 0xcafe4a11
55#endif
56
57#define __printf(a, b) __attribute__((format(printf, a, b)))
58
59__printf(1, 2)
60static int __base_pr(const char *format, ...)
61{
62 va_list args;
63 int err;
64
65 va_start(args, format);
66 err = vfprintf(stderr, format, args);
67 va_end(args);
68 return err;
69}
70
71static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
72static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
73static __printf(1, 2) libbpf_print_fn_t __pr_debug;
74
75#define __pr(func, fmt, ...) \
76do { \
77 if ((func)) \
78 (func)("libbpf: " fmt, ##__VA_ARGS__); \
79} while (0)
80
81#define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
82#define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
83#define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
84
85void libbpf_set_print(libbpf_print_fn_t warn,
86 libbpf_print_fn_t info,
87 libbpf_print_fn_t debug)
88{
89 __pr_warning = warn;
90 __pr_info = info;
91 __pr_debug = debug;
92}
93
94#define STRERR_BUFSIZE 128
95
96#define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START)
97#define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c)
98#define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
99
100static const char *libbpf_strerror_table[NR_ERRNO] = {
101 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf",
102 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid",
103 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost",
104 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch",
105 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf",
106 [ERRCODE_OFFSET(RELOC)] = "Relocation failed",
107 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading",
108 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big",
109 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version",
110 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type",
111 [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message",
112 [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence",
113};
114
115int libbpf_strerror(int err, char *buf, size_t size)
116{
117 if (!buf || !size)
118 return -1;
119
120 err = err > 0 ? err : -err;
121
122 if (err < __LIBBPF_ERRNO__START) {
123 int ret;
124
125 ret = strerror_r(err, buf, size);
126 buf[size - 1] = '\0';
127 return ret;
128 }
129
130 if (err < __LIBBPF_ERRNO__END) {
131 const char *msg;
132
133 msg = libbpf_strerror_table[ERRNO_OFFSET(err)];
134 snprintf(buf, size, "%s", msg);
135 buf[size - 1] = '\0';
136 return 0;
137 }
138
139 snprintf(buf, size, "Unknown libbpf error %d", err);
140 buf[size - 1] = '\0';
141 return -1;
142}
143
144#define CHECK_ERR(action, err, out) do { \
145 err = action; \
146 if (err) \
147 goto out; \
148} while(0)
149
150
151/* Copied from tools/perf/util/util.h */
152#ifndef zfree
153# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
154#endif
155
156#ifndef zclose
157# define zclose(fd) ({ \
158 int ___err = 0; \
159 if ((fd) >= 0) \
160 ___err = close((fd)); \
161 fd = -1; \
162 ___err; })
163#endif
164
165#ifdef HAVE_LIBELF_MMAP_SUPPORT
166# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
167#else
168# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
169#endif
170
171/*
172 * bpf_prog should be a better name but it has been used in
173 * linux/filter.h.
174 */
175struct bpf_program {
176 /* Index in elf obj file, for relocation use. */
177 int idx;
178 char *name;
179 char *section_name;
180 struct bpf_insn *insns;
181 size_t insns_cnt, main_prog_cnt;
182 enum bpf_prog_type type;
183
184 struct reloc_desc {
185 enum {
186 RELO_LD64,
187 RELO_CALL,
188 } type;
189 int insn_idx;
190 union {
191 int map_idx;
192 int text_off;
193 };
194 } *reloc_desc;
195 int nr_reloc;
196
197 struct {
198 int nr;
199 int *fds;
200 } instances;
201 bpf_program_prep_t preprocessor;
202
203 struct bpf_object *obj;
204 void *priv;
205 bpf_program_clear_priv_t clear_priv;
206
207 enum bpf_attach_type expected_attach_type;
208};
209
210struct bpf_map {
211 int fd;
212 char *name;
213 size_t offset;
214 struct bpf_map_def def;
215 void *priv;
216 bpf_map_clear_priv_t clear_priv;
217};
218
219static LIST_HEAD(bpf_objects_list);
220
221struct bpf_object {
222 char license[64];
223 u32 kern_version;
224
225 struct bpf_program *programs;
226 size_t nr_programs;
227 struct bpf_map *maps;
228 size_t nr_maps;
229
230 bool loaded;
231
232 /*
233 * Information when doing elf related work. Only valid if fd
234 * is valid.
235 */
236 struct {
237 int fd;
238 void *obj_buf;
239 size_t obj_buf_sz;
240 Elf *elf;
241 GElf_Ehdr ehdr;
242 Elf_Data *symbols;
243 size_t strtabidx;
244 struct {
245 GElf_Shdr shdr;
246 Elf_Data *data;
247 } *reloc;
248 int nr_reloc;
249 int maps_shndx;
250 int text_shndx;
251 } efile;
252 /*
253 * All loaded bpf_object is linked in a list, which is
254 * hidden to caller. bpf_objects__<func> handlers deal with
255 * all objects.
256 */
257 struct list_head list;
258
259 void *priv;
260 bpf_object_clear_priv_t clear_priv;
261
262 char path[];
263};
264#define obj_elf_valid(o) ((o)->efile.elf)
265
266static void bpf_program__unload(struct bpf_program *prog)
267{
268 int i;
269
270 if (!prog)
271 return;
272
273 /*
274 * If the object is opened but the program was never loaded,
275 * it is possible that prog->instances.nr == -1.
276 */
277 if (prog->instances.nr > 0) {
278 for (i = 0; i < prog->instances.nr; i++)
279 zclose(prog->instances.fds[i]);
280 } else if (prog->instances.nr != -1) {
281 pr_warning("Internal error: instances.nr is %d\n",
282 prog->instances.nr);
283 }
284
285 prog->instances.nr = -1;
286 zfree(&prog->instances.fds);
287}
288
289static void bpf_program__exit(struct bpf_program *prog)
290{
291 if (!prog)
292 return;
293
294 if (prog->clear_priv)
295 prog->clear_priv(prog, prog->priv);
296
297 prog->priv = NULL;
298 prog->clear_priv = NULL;
299
300 bpf_program__unload(prog);
301 zfree(&prog->name);
302 zfree(&prog->section_name);
303 zfree(&prog->insns);
304 zfree(&prog->reloc_desc);
305
306 prog->nr_reloc = 0;
307 prog->insns_cnt = 0;
308 prog->idx = -1;
309}
310
311static int
312bpf_program__init(void *data, size_t size, char *section_name, int idx,
313 struct bpf_program *prog)
314{
315 if (size < sizeof(struct bpf_insn)) {
316 pr_warning("corrupted section '%s'\n", section_name);
317 return -EINVAL;
318 }
319
320 bzero(prog, sizeof(*prog));
321
322 prog->section_name = strdup(section_name);
323 if (!prog->section_name) {
324 pr_warning("failed to alloc name for prog under section(%d) %s\n",
325 idx, section_name);
326 goto errout;
327 }
328
329 prog->insns = malloc(size);
330 if (!prog->insns) {
331 pr_warning("failed to alloc insns for prog under section %s\n",
332 section_name);
333 goto errout;
334 }
335 prog->insns_cnt = size / sizeof(struct bpf_insn);
336 memcpy(prog->insns, data,
337 prog->insns_cnt * sizeof(struct bpf_insn));
338 prog->idx = idx;
339 prog->instances.fds = NULL;
340 prog->instances.nr = -1;
341 prog->type = BPF_PROG_TYPE_KPROBE;
342
343 return 0;
344errout:
345 bpf_program__exit(prog);
346 return -ENOMEM;
347}
348
349static int
350bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
351 char *section_name, int idx)
352{
353 struct bpf_program prog, *progs;
354 int nr_progs, err;
355
356 err = bpf_program__init(data, size, section_name, idx, &prog);
357 if (err)
358 return err;
359
360 progs = obj->programs;
361 nr_progs = obj->nr_programs;
362
363 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
364 if (!progs) {
365 /*
366 * In this case the original obj->programs
367 * is still valid, so don't need special treat for
368 * bpf_close_object().
369 */
370 pr_warning("failed to alloc a new program under section '%s'\n",
371 section_name);
372 bpf_program__exit(&prog);
373 return -ENOMEM;
374 }
375
376 pr_debug("found program %s\n", prog.section_name);
377 obj->programs = progs;
378 obj->nr_programs = nr_progs + 1;
379 prog.obj = obj;
380 progs[nr_progs] = prog;
381 return 0;
382}
383
384static int
385bpf_object__init_prog_names(struct bpf_object *obj)
386{
387 Elf_Data *symbols = obj->efile.symbols;
388 struct bpf_program *prog;
389 size_t pi, si;
390
391 for (pi = 0; pi < obj->nr_programs; pi++) {
392 const char *name = NULL;
393
394 prog = &obj->programs[pi];
395 if (prog->idx == obj->efile.text_shndx) {
396 name = ".text";
397 goto skip_search;
398 }
399
400 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
401 si++) {
402 GElf_Sym sym;
403
404 if (!gelf_getsym(symbols, si, &sym))
405 continue;
406 if (sym.st_shndx != prog->idx)
407 continue;
408 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
409 continue;
410
411 name = elf_strptr(obj->efile.elf,
412 obj->efile.strtabidx,
413 sym.st_name);
414 if (!name) {
415 pr_warning("failed to get sym name string for prog %s\n",
416 prog->section_name);
417 return -LIBBPF_ERRNO__LIBELF;
418 }
419 }
420
421 if (!name) {
422 pr_warning("failed to find sym for prog %s\n",
423 prog->section_name);
424 return -EINVAL;
425 }
426skip_search:
427 prog->name = strdup(name);
428 if (!prog->name) {
429 pr_warning("failed to allocate memory for prog sym %s\n",
430 name);
431 return -ENOMEM;
432 }
433 }
434
435 return 0;
436}
437
438static struct bpf_object *bpf_object__new(const char *path,
439 void *obj_buf,
440 size_t obj_buf_sz)
441{
442 struct bpf_object *obj;
443
444 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
445 if (!obj) {
446 pr_warning("alloc memory failed for %s\n", path);
447 return ERR_PTR(-ENOMEM);
448 }
449
450 strcpy(obj->path, path);
451 obj->efile.fd = -1;
452
453 /*
454 * Caller of this function should also calls
455 * bpf_object__elf_finish() after data collection to return
456 * obj_buf to user. If not, we should duplicate the buffer to
457 * avoid user freeing them before elf finish.
458 */
459 obj->efile.obj_buf = obj_buf;
460 obj->efile.obj_buf_sz = obj_buf_sz;
461 obj->efile.maps_shndx = -1;
462
463 obj->loaded = false;
464
465 INIT_LIST_HEAD(&obj->list);
466 list_add(&obj->list, &bpf_objects_list);
467 return obj;
468}
469
470static void bpf_object__elf_finish(struct bpf_object *obj)
471{
472 if (!obj_elf_valid(obj))
473 return;
474
475 if (obj->efile.elf) {
476 elf_end(obj->efile.elf);
477 obj->efile.elf = NULL;
478 }
479 obj->efile.symbols = NULL;
480
481 zfree(&obj->efile.reloc);
482 obj->efile.nr_reloc = 0;
483 zclose(obj->efile.fd);
484 obj->efile.obj_buf = NULL;
485 obj->efile.obj_buf_sz = 0;
486}
487
488static int bpf_object__elf_init(struct bpf_object *obj)
489{
490 int err = 0;
491 GElf_Ehdr *ep;
492
493 if (obj_elf_valid(obj)) {
494 pr_warning("elf init: internal error\n");
495 return -LIBBPF_ERRNO__LIBELF;
496 }
497
498 if (obj->efile.obj_buf_sz > 0) {
499 /*
500 * obj_buf should have been validated by
501 * bpf_object__open_buffer().
502 */
503 obj->efile.elf = elf_memory(obj->efile.obj_buf,
504 obj->efile.obj_buf_sz);
505 } else {
506 obj->efile.fd = open(obj->path, O_RDONLY);
507 if (obj->efile.fd < 0) {
508 pr_warning("failed to open %s: %s\n", obj->path,
509 strerror(errno));
510 return -errno;
511 }
512
513 obj->efile.elf = elf_begin(obj->efile.fd,
514 LIBBPF_ELF_C_READ_MMAP,
515 NULL);
516 }
517
518 if (!obj->efile.elf) {
519 pr_warning("failed to open %s as ELF file\n",
520 obj->path);
521 err = -LIBBPF_ERRNO__LIBELF;
522 goto errout;
523 }
524
525 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
526 pr_warning("failed to get EHDR from %s\n",
527 obj->path);
528 err = -LIBBPF_ERRNO__FORMAT;
529 goto errout;
530 }
531 ep = &obj->efile.ehdr;
532
533 /* Old LLVM set e_machine to EM_NONE */
534 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
535 pr_warning("%s is not an eBPF object file\n",
536 obj->path);
537 err = -LIBBPF_ERRNO__FORMAT;
538 goto errout;
539 }
540
541 return 0;
542errout:
543 bpf_object__elf_finish(obj);
544 return err;
545}
546
547static int
548bpf_object__check_endianness(struct bpf_object *obj)
549{
550 static unsigned int const endian = 1;
551
552 switch (obj->efile.ehdr.e_ident[EI_DATA]) {
553 case ELFDATA2LSB:
554 /* We are big endian, BPF obj is little endian. */
555 if (*(unsigned char const *)&endian != 1)
556 goto mismatch;
557 break;
558
559 case ELFDATA2MSB:
560 /* We are little endian, BPF obj is big endian. */
561 if (*(unsigned char const *)&endian != 0)
562 goto mismatch;
563 break;
564 default:
565 return -LIBBPF_ERRNO__ENDIAN;
566 }
567
568 return 0;
569
570mismatch:
571 pr_warning("Error: endianness mismatch.\n");
572 return -LIBBPF_ERRNO__ENDIAN;
573}
574
575static int
576bpf_object__init_license(struct bpf_object *obj,
577 void *data, size_t size)
578{
579 memcpy(obj->license, data,
580 min(size, sizeof(obj->license) - 1));
581 pr_debug("license of %s is %s\n", obj->path, obj->license);
582 return 0;
583}
584
585static int
586bpf_object__init_kversion(struct bpf_object *obj,
587 void *data, size_t size)
588{
589 u32 kver;
590
591 if (size != sizeof(kver)) {
592 pr_warning("invalid kver section in %s\n", obj->path);
593 return -LIBBPF_ERRNO__FORMAT;
594 }
595 memcpy(&kver, data, sizeof(kver));
596 obj->kern_version = kver;
597 pr_debug("kernel version of %s is %x\n", obj->path,
598 obj->kern_version);
599 return 0;
600}
601
602static int compare_bpf_map(const void *_a, const void *_b)
603{
604 const struct bpf_map *a = _a;
605 const struct bpf_map *b = _b;
606
607 return a->offset - b->offset;
608}
609
610static int
611bpf_object__init_maps(struct bpf_object *obj)
612{
613 int i, map_idx, map_def_sz, nr_maps = 0;
614 Elf_Scn *scn;
615 Elf_Data *data;
616 Elf_Data *symbols = obj->efile.symbols;
617
618 if (obj->efile.maps_shndx < 0)
619 return -EINVAL;
620 if (!symbols)
621 return -EINVAL;
622
623 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
624 if (scn)
625 data = elf_getdata(scn, NULL);
626 if (!scn || !data) {
627 pr_warning("failed to get Elf_Data from map section %d\n",
628 obj->efile.maps_shndx);
629 return -EINVAL;
630 }
631
632 /*
633 * Count number of maps. Each map has a name.
634 * Array of maps is not supported: only the first element is
635 * considered.
636 *
637 * TODO: Detect array of map and report error.
638 */
639 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
640 GElf_Sym sym;
641
642 if (!gelf_getsym(symbols, i, &sym))
643 continue;
644 if (sym.st_shndx != obj->efile.maps_shndx)
645 continue;
646 nr_maps++;
647 }
648
649 /* Alloc obj->maps and fill nr_maps. */
650 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
651 nr_maps, data->d_size);
652
653 if (!nr_maps)
654 return 0;
655
656 /* Assume equally sized map definitions */
657 map_def_sz = data->d_size / nr_maps;
658 if (!data->d_size || (data->d_size % nr_maps) != 0) {
659 pr_warning("unable to determine map definition size "
660 "section %s, %d maps in %zd bytes\n",
661 obj->path, nr_maps, data->d_size);
662 return -EINVAL;
663 }
664
665 obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
666 if (!obj->maps) {
667 pr_warning("alloc maps for object failed\n");
668 return -ENOMEM;
669 }
670 obj->nr_maps = nr_maps;
671
672 /*
673 * fill all fd with -1 so won't close incorrect
674 * fd (fd=0 is stdin) when failure (zclose won't close
675 * negative fd)).
676 */
677 for (i = 0; i < nr_maps; i++)
678 obj->maps[i].fd = -1;
679
680 /*
681 * Fill obj->maps using data in "maps" section.
682 */
683 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
684 GElf_Sym sym;
685 const char *map_name;
686 struct bpf_map_def *def;
687
688 if (!gelf_getsym(symbols, i, &sym))
689 continue;
690 if (sym.st_shndx != obj->efile.maps_shndx)
691 continue;
692
693 map_name = elf_strptr(obj->efile.elf,
694 obj->efile.strtabidx,
695 sym.st_name);
696 obj->maps[map_idx].offset = sym.st_value;
697 if (sym.st_value + map_def_sz > data->d_size) {
698 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
699 obj->path, map_name);
700 return -EINVAL;
701 }
702
703 obj->maps[map_idx].name = strdup(map_name);
704 if (!obj->maps[map_idx].name) {
705 pr_warning("failed to alloc map name\n");
706 return -ENOMEM;
707 }
708 pr_debug("map %d is \"%s\"\n", map_idx,
709 obj->maps[map_idx].name);
710 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
711 /*
712 * If the definition of the map in the object file fits in
713 * bpf_map_def, copy it. Any extra fields in our version
714 * of bpf_map_def will default to zero as a result of the
715 * calloc above.
716 */
717 if (map_def_sz <= sizeof(struct bpf_map_def)) {
718 memcpy(&obj->maps[map_idx].def, def, map_def_sz);
719 } else {
720 /*
721 * Here the map structure being read is bigger than what
722 * we expect, truncate if the excess bits are all zero.
723 * If they are not zero, reject this map as
724 * incompatible.
725 */
726 char *b;
727 for (b = ((char *)def) + sizeof(struct bpf_map_def);
728 b < ((char *)def) + map_def_sz; b++) {
729 if (*b != 0) {
730 pr_warning("maps section in %s: \"%s\" "
731 "has unrecognized, non-zero "
732 "options\n",
733 obj->path, map_name);
734 return -EINVAL;
735 }
736 }
737 memcpy(&obj->maps[map_idx].def, def,
738 sizeof(struct bpf_map_def));
739 }
740 map_idx++;
741 }
742
743 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
744 return 0;
745}
746
747static bool section_have_execinstr(struct bpf_object *obj, int idx)
748{
749 Elf_Scn *scn;
750 GElf_Shdr sh;
751
752 scn = elf_getscn(obj->efile.elf, idx);
753 if (!scn)
754 return false;
755
756 if (gelf_getshdr(scn, &sh) != &sh)
757 return false;
758
759 if (sh.sh_flags & SHF_EXECINSTR)
760 return true;
761
762 return false;
763}
764
765static int bpf_object__elf_collect(struct bpf_object *obj)
766{
767 Elf *elf = obj->efile.elf;
768 GElf_Ehdr *ep = &obj->efile.ehdr;
769 Elf_Scn *scn = NULL;
770 int idx = 0, err = 0;
771
772 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
773 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
774 pr_warning("failed to get e_shstrndx from %s\n",
775 obj->path);
776 return -LIBBPF_ERRNO__FORMAT;
777 }
778
779 while ((scn = elf_nextscn(elf, scn)) != NULL) {
780 char *name;
781 GElf_Shdr sh;
782 Elf_Data *data;
783
784 idx++;
785 if (gelf_getshdr(scn, &sh) != &sh) {
786 pr_warning("failed to get section(%d) header from %s\n",
787 idx, obj->path);
788 err = -LIBBPF_ERRNO__FORMAT;
789 goto out;
790 }
791
792 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
793 if (!name) {
794 pr_warning("failed to get section(%d) name from %s\n",
795 idx, obj->path);
796 err = -LIBBPF_ERRNO__FORMAT;
797 goto out;
798 }
799
800 data = elf_getdata(scn, 0);
801 if (!data) {
802 pr_warning("failed to get section(%d) data from %s(%s)\n",
803 idx, name, obj->path);
804 err = -LIBBPF_ERRNO__FORMAT;
805 goto out;
806 }
807 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
808 idx, name, (unsigned long)data->d_size,
809 (int)sh.sh_link, (unsigned long)sh.sh_flags,
810 (int)sh.sh_type);
811
812 if (strcmp(name, "license") == 0)
813 err = bpf_object__init_license(obj,
814 data->d_buf,
815 data->d_size);
816 else if (strcmp(name, "version") == 0)
817 err = bpf_object__init_kversion(obj,
818 data->d_buf,
819 data->d_size);
820 else if (strcmp(name, "maps") == 0)
821 obj->efile.maps_shndx = idx;
822 else if (sh.sh_type == SHT_SYMTAB) {
823 if (obj->efile.symbols) {
824 pr_warning("bpf: multiple SYMTAB in %s\n",
825 obj->path);
826 err = -LIBBPF_ERRNO__FORMAT;
827 } else {
828 obj->efile.symbols = data;
829 obj->efile.strtabidx = sh.sh_link;
830 }
831 } else if ((sh.sh_type == SHT_PROGBITS) &&
832 (sh.sh_flags & SHF_EXECINSTR) &&
833 (data->d_size > 0)) {
834 if (strcmp(name, ".text") == 0)
835 obj->efile.text_shndx = idx;
836 err = bpf_object__add_program(obj, data->d_buf,
837 data->d_size, name, idx);
838 if (err) {
839 char errmsg[STRERR_BUFSIZE];
840
841 strerror_r(-err, errmsg, sizeof(errmsg));
842 pr_warning("failed to alloc program %s (%s): %s",
843 name, obj->path, errmsg);
844 }
845 } else if (sh.sh_type == SHT_REL) {
846 void *reloc = obj->efile.reloc;
847 int nr_reloc = obj->efile.nr_reloc + 1;
848 int sec = sh.sh_info; /* points to other section */
849
850 /* Only do relo for section with exec instructions */
851 if (!section_have_execinstr(obj, sec)) {
852 pr_debug("skip relo %s(%d) for section(%d)\n",
853 name, idx, sec);
854 continue;
855 }
856
857 reloc = realloc(reloc,
858 sizeof(*obj->efile.reloc) * nr_reloc);
859 if (!reloc) {
860 pr_warning("realloc failed\n");
861 err = -ENOMEM;
862 } else {
863 int n = nr_reloc - 1;
864
865 obj->efile.reloc = reloc;
866 obj->efile.nr_reloc = nr_reloc;
867
868 obj->efile.reloc[n].shdr = sh;
869 obj->efile.reloc[n].data = data;
870 }
871 } else {
872 pr_debug("skip section(%d) %s\n", idx, name);
873 }
874 if (err)
875 goto out;
876 }
877
878 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
879 pr_warning("Corrupted ELF file: index of strtab invalid\n");
880 return LIBBPF_ERRNO__FORMAT;
881 }
882 if (obj->efile.maps_shndx >= 0) {
883 err = bpf_object__init_maps(obj);
884 if (err)
885 goto out;
886 }
887 err = bpf_object__init_prog_names(obj);
888out:
889 return err;
890}
891
892static struct bpf_program *
893bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
894{
895 struct bpf_program *prog;
896 size_t i;
897
898 for (i = 0; i < obj->nr_programs; i++) {
899 prog = &obj->programs[i];
900 if (prog->idx == idx)
901 return prog;
902 }
903 return NULL;
904}
905
906static int
907bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
908 Elf_Data *data, struct bpf_object *obj)
909{
910 Elf_Data *symbols = obj->efile.symbols;
911 int text_shndx = obj->efile.text_shndx;
912 int maps_shndx = obj->efile.maps_shndx;
913 struct bpf_map *maps = obj->maps;
914 size_t nr_maps = obj->nr_maps;
915 int i, nrels;
916
917 pr_debug("collecting relocating info for: '%s'\n",
918 prog->section_name);
919 nrels = shdr->sh_size / shdr->sh_entsize;
920
921 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
922 if (!prog->reloc_desc) {
923 pr_warning("failed to alloc memory in relocation\n");
924 return -ENOMEM;
925 }
926 prog->nr_reloc = nrels;
927
928 for (i = 0; i < nrels; i++) {
929 GElf_Sym sym;
930 GElf_Rel rel;
931 unsigned int insn_idx;
932 struct bpf_insn *insns = prog->insns;
933 size_t map_idx;
934
935 if (!gelf_getrel(data, i, &rel)) {
936 pr_warning("relocation: failed to get %d reloc\n", i);
937 return -LIBBPF_ERRNO__FORMAT;
938 }
939
940 if (!gelf_getsym(symbols,
941 GELF_R_SYM(rel.r_info),
942 &sym)) {
943 pr_warning("relocation: symbol %"PRIx64" not found\n",
944 GELF_R_SYM(rel.r_info));
945 return -LIBBPF_ERRNO__FORMAT;
946 }
947 pr_debug("relo for %lld value %lld name %d\n",
948 (long long) (rel.r_info >> 32),
949 (long long) sym.st_value, sym.st_name);
950
951 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
952 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
953 prog->section_name, sym.st_shndx);
954 return -LIBBPF_ERRNO__RELOC;
955 }
956
957 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
958 pr_debug("relocation: insn_idx=%u\n", insn_idx);
959
960 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
961 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
962 pr_warning("incorrect bpf_call opcode\n");
963 return -LIBBPF_ERRNO__RELOC;
964 }
965 prog->reloc_desc[i].type = RELO_CALL;
966 prog->reloc_desc[i].insn_idx = insn_idx;
967 prog->reloc_desc[i].text_off = sym.st_value;
968 continue;
969 }
970
971 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
972 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
973 insn_idx, insns[insn_idx].code);
974 return -LIBBPF_ERRNO__RELOC;
975 }
976
977 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
978 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
979 if (maps[map_idx].offset == sym.st_value) {
980 pr_debug("relocation: find map %zd (%s) for insn %u\n",
981 map_idx, maps[map_idx].name, insn_idx);
982 break;
983 }
984 }
985
986 if (map_idx >= nr_maps) {
987 pr_warning("bpf relocation: map_idx %d large than %d\n",
988 (int)map_idx, (int)nr_maps - 1);
989 return -LIBBPF_ERRNO__RELOC;
990 }
991
992 prog->reloc_desc[i].type = RELO_LD64;
993 prog->reloc_desc[i].insn_idx = insn_idx;
994 prog->reloc_desc[i].map_idx = map_idx;
995 }
996 return 0;
997}
998
999static int
1000bpf_object__create_maps(struct bpf_object *obj)
1001{
1002 unsigned int i;
1003
1004 for (i = 0; i < obj->nr_maps; i++) {
1005 struct bpf_map_def *def = &obj->maps[i].def;
1006 int *pfd = &obj->maps[i].fd;
1007
1008 *pfd = bpf_create_map_name(def->type,
1009 obj->maps[i].name,
1010 def->key_size,
1011 def->value_size,
1012 def->max_entries,
1013 def->map_flags);
1014 if (*pfd < 0) {
1015 size_t j;
1016 int err = *pfd;
1017
1018 pr_warning("failed to create map (name: '%s'): %s\n",
1019 obj->maps[i].name,
1020 strerror(errno));
1021 for (j = 0; j < i; j++)
1022 zclose(obj->maps[j].fd);
1023 return err;
1024 }
1025 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd);
1026 }
1027
1028 return 0;
1029}
1030
1031static int
1032bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1033 struct reloc_desc *relo)
1034{
1035 struct bpf_insn *insn, *new_insn;
1036 struct bpf_program *text;
1037 size_t new_cnt;
1038
1039 if (relo->type != RELO_CALL)
1040 return -LIBBPF_ERRNO__RELOC;
1041
1042 if (prog->idx == obj->efile.text_shndx) {
1043 pr_warning("relo in .text insn %d into off %d\n",
1044 relo->insn_idx, relo->text_off);
1045 return -LIBBPF_ERRNO__RELOC;
1046 }
1047
1048 if (prog->main_prog_cnt == 0) {
1049 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1050 if (!text) {
1051 pr_warning("no .text section found yet relo into text exist\n");
1052 return -LIBBPF_ERRNO__RELOC;
1053 }
1054 new_cnt = prog->insns_cnt + text->insns_cnt;
1055 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn));
1056 if (!new_insn) {
1057 pr_warning("oom in prog realloc\n");
1058 return -ENOMEM;
1059 }
1060 memcpy(new_insn + prog->insns_cnt, text->insns,
1061 text->insns_cnt * sizeof(*insn));
1062 prog->insns = new_insn;
1063 prog->main_prog_cnt = prog->insns_cnt;
1064 prog->insns_cnt = new_cnt;
1065 pr_debug("added %zd insn from %s to prog %s\n",
1066 text->insns_cnt, text->section_name,
1067 prog->section_name);
1068 }
1069 insn = &prog->insns[relo->insn_idx];
1070 insn->imm += prog->main_prog_cnt - relo->insn_idx;
1071 return 0;
1072}
1073
1074static int
1075bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1076{
1077 int i, err;
1078
1079 if (!prog || !prog->reloc_desc)
1080 return 0;
1081
1082 for (i = 0; i < prog->nr_reloc; i++) {
1083 if (prog->reloc_desc[i].type == RELO_LD64) {
1084 struct bpf_insn *insns = prog->insns;
1085 int insn_idx, map_idx;
1086
1087 insn_idx = prog->reloc_desc[i].insn_idx;
1088 map_idx = prog->reloc_desc[i].map_idx;
1089
1090 if (insn_idx >= (int)prog->insns_cnt) {
1091 pr_warning("relocation out of range: '%s'\n",
1092 prog->section_name);
1093 return -LIBBPF_ERRNO__RELOC;
1094 }
1095 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1096 insns[insn_idx].imm = obj->maps[map_idx].fd;
1097 } else {
1098 err = bpf_program__reloc_text(prog, obj,
1099 &prog->reloc_desc[i]);
1100 if (err)
1101 return err;
1102 }
1103 }
1104
1105 zfree(&prog->reloc_desc);
1106 prog->nr_reloc = 0;
1107 return 0;
1108}
1109
1110
1111static int
1112bpf_object__relocate(struct bpf_object *obj)
1113{
1114 struct bpf_program *prog;
1115 size_t i;
1116 int err;
1117
1118 for (i = 0; i < obj->nr_programs; i++) {
1119 prog = &obj->programs[i];
1120
1121 err = bpf_program__relocate(prog, obj);
1122 if (err) {
1123 pr_warning("failed to relocate '%s'\n",
1124 prog->section_name);
1125 return err;
1126 }
1127 }
1128 return 0;
1129}
1130
1131static int bpf_object__collect_reloc(struct bpf_object *obj)
1132{
1133 int i, err;
1134
1135 if (!obj_elf_valid(obj)) {
1136 pr_warning("Internal error: elf object is closed\n");
1137 return -LIBBPF_ERRNO__INTERNAL;
1138 }
1139
1140 for (i = 0; i < obj->efile.nr_reloc; i++) {
1141 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1142 Elf_Data *data = obj->efile.reloc[i].data;
1143 int idx = shdr->sh_info;
1144 struct bpf_program *prog;
1145
1146 if (shdr->sh_type != SHT_REL) {
1147 pr_warning("internal error at %d\n", __LINE__);
1148 return -LIBBPF_ERRNO__INTERNAL;
1149 }
1150
1151 prog = bpf_object__find_prog_by_idx(obj, idx);
1152 if (!prog) {
1153 pr_warning("relocation failed: no section(%d)\n", idx);
1154 return -LIBBPF_ERRNO__RELOC;
1155 }
1156
1157 err = bpf_program__collect_reloc(prog,
1158 shdr, data,
1159 obj);
1160 if (err)
1161 return err;
1162 }
1163 return 0;
1164}
1165
1166static int
1167load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1168 const char *name, struct bpf_insn *insns, int insns_cnt,
1169 char *license, u32 kern_version, int *pfd)
1170{
1171 struct bpf_load_program_attr load_attr;
1172 char *log_buf;
1173 int ret;
1174
1175 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1176 load_attr.prog_type = type;
1177 load_attr.expected_attach_type = expected_attach_type;
1178 load_attr.name = name;
1179 load_attr.insns = insns;
1180 load_attr.insns_cnt = insns_cnt;
1181 load_attr.license = license;
1182 load_attr.kern_version = kern_version;
1183
1184 if (!load_attr.insns || !load_attr.insns_cnt)
1185 return -EINVAL;
1186
1187 log_buf = malloc(BPF_LOG_BUF_SIZE);
1188 if (!log_buf)
1189 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1190
1191 ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1192
1193 if (ret >= 0) {
1194 *pfd = ret;
1195 ret = 0;
1196 goto out;
1197 }
1198
1199 ret = -LIBBPF_ERRNO__LOAD;
1200 pr_warning("load bpf program failed: %s\n", strerror(errno));
1201
1202 if (log_buf && log_buf[0] != '\0') {
1203 ret = -LIBBPF_ERRNO__VERIFY;
1204 pr_warning("-- BEGIN DUMP LOG ---\n");
1205 pr_warning("\n%s\n", log_buf);
1206 pr_warning("-- END LOG --\n");
1207 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1208 pr_warning("Program too large (%zu insns), at most %d insns\n",
1209 load_attr.insns_cnt, BPF_MAXINSNS);
1210 ret = -LIBBPF_ERRNO__PROG2BIG;
1211 } else {
1212 /* Wrong program type? */
1213 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1214 int fd;
1215
1216 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1217 load_attr.expected_attach_type = 0;
1218 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1219 if (fd >= 0) {
1220 close(fd);
1221 ret = -LIBBPF_ERRNO__PROGTYPE;
1222 goto out;
1223 }
1224 }
1225
1226 if (log_buf)
1227 ret = -LIBBPF_ERRNO__KVER;
1228 }
1229
1230out:
1231 free(log_buf);
1232 return ret;
1233}
1234
1235static int
1236bpf_program__load(struct bpf_program *prog,
1237 char *license, u32 kern_version)
1238{
1239 int err = 0, fd, i;
1240
1241 if (prog->instances.nr < 0 || !prog->instances.fds) {
1242 if (prog->preprocessor) {
1243 pr_warning("Internal error: can't load program '%s'\n",
1244 prog->section_name);
1245 return -LIBBPF_ERRNO__INTERNAL;
1246 }
1247
1248 prog->instances.fds = malloc(sizeof(int));
1249 if (!prog->instances.fds) {
1250 pr_warning("Not enough memory for BPF fds\n");
1251 return -ENOMEM;
1252 }
1253 prog->instances.nr = 1;
1254 prog->instances.fds[0] = -1;
1255 }
1256
1257 if (!prog->preprocessor) {
1258 if (prog->instances.nr != 1) {
1259 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1260 prog->section_name, prog->instances.nr);
1261 }
1262 err = load_program(prog->type, prog->expected_attach_type,
1263 prog->name, prog->insns, prog->insns_cnt,
1264 license, kern_version, &fd);
1265 if (!err)
1266 prog->instances.fds[0] = fd;
1267 goto out;
1268 }
1269
1270 for (i = 0; i < prog->instances.nr; i++) {
1271 struct bpf_prog_prep_result result;
1272 bpf_program_prep_t preprocessor = prog->preprocessor;
1273
1274 bzero(&result, sizeof(result));
1275 err = preprocessor(prog, i, prog->insns,
1276 prog->insns_cnt, &result);
1277 if (err) {
1278 pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1279 i, prog->section_name);
1280 goto out;
1281 }
1282
1283 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1284 pr_debug("Skip loading the %dth instance of program '%s'\n",
1285 i, prog->section_name);
1286 prog->instances.fds[i] = -1;
1287 if (result.pfd)
1288 *result.pfd = -1;
1289 continue;
1290 }
1291
1292 err = load_program(prog->type, prog->expected_attach_type,
1293 prog->name, result.new_insn_ptr,
1294 result.new_insn_cnt,
1295 license, kern_version, &fd);
1296
1297 if (err) {
1298 pr_warning("Loading the %dth instance of program '%s' failed\n",
1299 i, prog->section_name);
1300 goto out;
1301 }
1302
1303 if (result.pfd)
1304 *result.pfd = fd;
1305 prog->instances.fds[i] = fd;
1306 }
1307out:
1308 if (err)
1309 pr_warning("failed to load program '%s'\n",
1310 prog->section_name);
1311 zfree(&prog->insns);
1312 prog->insns_cnt = 0;
1313 return err;
1314}
1315
1316static int
1317bpf_object__load_progs(struct bpf_object *obj)
1318{
1319 size_t i;
1320 int err;
1321
1322 for (i = 0; i < obj->nr_programs; i++) {
1323 if (obj->programs[i].idx == obj->efile.text_shndx)
1324 continue;
1325 err = bpf_program__load(&obj->programs[i],
1326 obj->license,
1327 obj->kern_version);
1328 if (err)
1329 return err;
1330 }
1331 return 0;
1332}
1333
1334static int bpf_object__validate(struct bpf_object *obj)
1335{
1336 if (obj->kern_version == 0) {
1337 pr_warning("%s doesn't provide kernel version\n",
1338 obj->path);
1339 return -LIBBPF_ERRNO__KVERSION;
1340 }
1341 return 0;
1342}
1343
1344static struct bpf_object *
1345__bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz)
1346{
1347 struct bpf_object *obj;
1348 int err;
1349
1350 if (elf_version(EV_CURRENT) == EV_NONE) {
1351 pr_warning("failed to init libelf for %s\n", path);
1352 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1353 }
1354
1355 obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1356 if (IS_ERR(obj))
1357 return obj;
1358
1359 CHECK_ERR(bpf_object__elf_init(obj), err, out);
1360 CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1361 CHECK_ERR(bpf_object__elf_collect(obj), err, out);
1362 CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1363 CHECK_ERR(bpf_object__validate(obj), err, out);
1364
1365 bpf_object__elf_finish(obj);
1366 return obj;
1367out:
1368 bpf_object__close(obj);
1369 return ERR_PTR(err);
1370}
1371
1372struct bpf_object *bpf_object__open(const char *path)
1373{
1374 /* param validation */
1375 if (!path)
1376 return NULL;
1377
1378 pr_debug("loading %s\n", path);
1379
1380 return __bpf_object__open(path, NULL, 0);
1381}
1382
1383struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1384 size_t obj_buf_sz,
1385 const char *name)
1386{
1387 char tmp_name[64];
1388
1389 /* param validation */
1390 if (!obj_buf || obj_buf_sz <= 0)
1391 return NULL;
1392
1393 if (!name) {
1394 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1395 (unsigned long)obj_buf,
1396 (unsigned long)obj_buf_sz);
1397 tmp_name[sizeof(tmp_name) - 1] = '\0';
1398 name = tmp_name;
1399 }
1400 pr_debug("loading object '%s' from buffer\n",
1401 name);
1402
1403 return __bpf_object__open(name, obj_buf, obj_buf_sz);
1404}
1405
1406int bpf_object__unload(struct bpf_object *obj)
1407{
1408 size_t i;
1409
1410 if (!obj)
1411 return -EINVAL;
1412
1413 for (i = 0; i < obj->nr_maps; i++)
1414 zclose(obj->maps[i].fd);
1415
1416 for (i = 0; i < obj->nr_programs; i++)
1417 bpf_program__unload(&obj->programs[i]);
1418
1419 return 0;
1420}
1421
1422int bpf_object__load(struct bpf_object *obj)
1423{
1424 int err;
1425
1426 if (!obj)
1427 return -EINVAL;
1428
1429 if (obj->loaded) {
1430 pr_warning("object should not be loaded twice\n");
1431 return -EINVAL;
1432 }
1433
1434 obj->loaded = true;
1435
1436 CHECK_ERR(bpf_object__create_maps(obj), err, out);
1437 CHECK_ERR(bpf_object__relocate(obj), err, out);
1438 CHECK_ERR(bpf_object__load_progs(obj), err, out);
1439
1440 return 0;
1441out:
1442 bpf_object__unload(obj);
1443 pr_warning("failed to load object '%s'\n", obj->path);
1444 return err;
1445}
1446
1447static int check_path(const char *path)
1448{
1449 struct statfs st_fs;
1450 char *dname, *dir;
1451 int err = 0;
1452
1453 if (path == NULL)
1454 return -EINVAL;
1455
1456 dname = strdup(path);
1457 if (dname == NULL)
1458 return -ENOMEM;
1459
1460 dir = dirname(dname);
1461 if (statfs(dir, &st_fs)) {
1462 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno));
1463 err = -errno;
1464 }
1465 free(dname);
1466
1467 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1468 pr_warning("specified path %s is not on BPF FS\n", path);
1469 err = -EINVAL;
1470 }
1471
1472 return err;
1473}
1474
1475int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1476 int instance)
1477{
1478 int err;
1479
1480 err = check_path(path);
1481 if (err)
1482 return err;
1483
1484 if (prog == NULL) {
1485 pr_warning("invalid program pointer\n");
1486 return -EINVAL;
1487 }
1488
1489 if (instance < 0 || instance >= prog->instances.nr) {
1490 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1491 instance, prog->section_name, prog->instances.nr);
1492 return -EINVAL;
1493 }
1494
1495 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1496 pr_warning("failed to pin program: %s\n", strerror(errno));
1497 return -errno;
1498 }
1499 pr_debug("pinned program '%s'\n", path);
1500
1501 return 0;
1502}
1503
1504static int make_dir(const char *path)
1505{
1506 int err = 0;
1507
1508 if (mkdir(path, 0700) && errno != EEXIST)
1509 err = -errno;
1510
1511 if (err)
1512 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err));
1513 return err;
1514}
1515
1516int bpf_program__pin(struct bpf_program *prog, const char *path)
1517{
1518 int i, err;
1519
1520 err = check_path(path);
1521 if (err)
1522 return err;
1523
1524 if (prog == NULL) {
1525 pr_warning("invalid program pointer\n");
1526 return -EINVAL;
1527 }
1528
1529 if (prog->instances.nr <= 0) {
1530 pr_warning("no instances of prog %s to pin\n",
1531 prog->section_name);
1532 return -EINVAL;
1533 }
1534
1535 err = make_dir(path);
1536 if (err)
1537 return err;
1538
1539 for (i = 0; i < prog->instances.nr; i++) {
1540 char buf[PATH_MAX];
1541 int len;
1542
1543 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1544 if (len < 0)
1545 return -EINVAL;
1546 else if (len >= PATH_MAX)
1547 return -ENAMETOOLONG;
1548
1549 err = bpf_program__pin_instance(prog, buf, i);
1550 if (err)
1551 return err;
1552 }
1553
1554 return 0;
1555}
1556
1557int bpf_map__pin(struct bpf_map *map, const char *path)
1558{
1559 int err;
1560
1561 err = check_path(path);
1562 if (err)
1563 return err;
1564
1565 if (map == NULL) {
1566 pr_warning("invalid map pointer\n");
1567 return -EINVAL;
1568 }
1569
1570 if (bpf_obj_pin(map->fd, path)) {
1571 pr_warning("failed to pin map: %s\n", strerror(errno));
1572 return -errno;
1573 }
1574
1575 pr_debug("pinned map '%s'\n", path);
1576 return 0;
1577}
1578
1579int bpf_object__pin(struct bpf_object *obj, const char *path)
1580{
1581 struct bpf_program *prog;
1582 struct bpf_map *map;
1583 int err;
1584
1585 if (!obj)
1586 return -ENOENT;
1587
1588 if (!obj->loaded) {
1589 pr_warning("object not yet loaded; load it first\n");
1590 return -ENOENT;
1591 }
1592
1593 err = make_dir(path);
1594 if (err)
1595 return err;
1596
1597 bpf_map__for_each(map, obj) {
1598 char buf[PATH_MAX];
1599 int len;
1600
1601 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1602 bpf_map__name(map));
1603 if (len < 0)
1604 return -EINVAL;
1605 else if (len >= PATH_MAX)
1606 return -ENAMETOOLONG;
1607
1608 err = bpf_map__pin(map, buf);
1609 if (err)
1610 return err;
1611 }
1612
1613 bpf_object__for_each_program(prog, obj) {
1614 char buf[PATH_MAX];
1615 int len;
1616
1617 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1618 prog->section_name);
1619 if (len < 0)
1620 return -EINVAL;
1621 else if (len >= PATH_MAX)
1622 return -ENAMETOOLONG;
1623
1624 err = bpf_program__pin(prog, buf);
1625 if (err)
1626 return err;
1627 }
1628
1629 return 0;
1630}
1631
1632void bpf_object__close(struct bpf_object *obj)
1633{
1634 size_t i;
1635
1636 if (!obj)
1637 return;
1638
1639 if (obj->clear_priv)
1640 obj->clear_priv(obj, obj->priv);
1641
1642 bpf_object__elf_finish(obj);
1643 bpf_object__unload(obj);
1644
1645 for (i = 0; i < obj->nr_maps; i++) {
1646 zfree(&obj->maps[i].name);
1647 if (obj->maps[i].clear_priv)
1648 obj->maps[i].clear_priv(&obj->maps[i],
1649 obj->maps[i].priv);
1650 obj->maps[i].priv = NULL;
1651 obj->maps[i].clear_priv = NULL;
1652 }
1653 zfree(&obj->maps);
1654 obj->nr_maps = 0;
1655
1656 if (obj->programs && obj->nr_programs) {
1657 for (i = 0; i < obj->nr_programs; i++)
1658 bpf_program__exit(&obj->programs[i]);
1659 }
1660 zfree(&obj->programs);
1661
1662 list_del(&obj->list);
1663 free(obj);
1664}
1665
1666struct bpf_object *
1667bpf_object__next(struct bpf_object *prev)
1668{
1669 struct bpf_object *next;
1670
1671 if (!prev)
1672 next = list_first_entry(&bpf_objects_list,
1673 struct bpf_object,
1674 list);
1675 else
1676 next = list_next_entry(prev, list);
1677
1678 /* Empty list is noticed here so don't need checking on entry. */
1679 if (&next->list == &bpf_objects_list)
1680 return NULL;
1681
1682 return next;
1683}
1684
1685const char *bpf_object__name(struct bpf_object *obj)
1686{
1687 return obj ? obj->path : ERR_PTR(-EINVAL);
1688}
1689
1690unsigned int bpf_object__kversion(struct bpf_object *obj)
1691{
1692 return obj ? obj->kern_version : 0;
1693}
1694
1695int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1696 bpf_object_clear_priv_t clear_priv)
1697{
1698 if (obj->priv && obj->clear_priv)
1699 obj->clear_priv(obj, obj->priv);
1700
1701 obj->priv = priv;
1702 obj->clear_priv = clear_priv;
1703 return 0;
1704}
1705
1706void *bpf_object__priv(struct bpf_object *obj)
1707{
1708 return obj ? obj->priv : ERR_PTR(-EINVAL);
1709}
1710
1711struct bpf_program *
1712bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1713{
1714 size_t idx;
1715
1716 if (!obj->programs)
1717 return NULL;
1718 /* First handler */
1719 if (prev == NULL)
1720 return &obj->programs[0];
1721
1722 if (prev->obj != obj) {
1723 pr_warning("error: program handler doesn't match object\n");
1724 return NULL;
1725 }
1726
1727 idx = (prev - obj->programs) + 1;
1728 if (idx >= obj->nr_programs)
1729 return NULL;
1730 return &obj->programs[idx];
1731}
1732
1733int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1734 bpf_program_clear_priv_t clear_priv)
1735{
1736 if (prog->priv && prog->clear_priv)
1737 prog->clear_priv(prog, prog->priv);
1738
1739 prog->priv = priv;
1740 prog->clear_priv = clear_priv;
1741 return 0;
1742}
1743
1744void *bpf_program__priv(struct bpf_program *prog)
1745{
1746 return prog ? prog->priv : ERR_PTR(-EINVAL);
1747}
1748
1749const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1750{
1751 const char *title;
1752
1753 title = prog->section_name;
1754 if (needs_copy) {
1755 title = strdup(title);
1756 if (!title) {
1757 pr_warning("failed to strdup program title\n");
1758 return ERR_PTR(-ENOMEM);
1759 }
1760 }
1761
1762 return title;
1763}
1764
1765int bpf_program__fd(struct bpf_program *prog)
1766{
1767 return bpf_program__nth_fd(prog, 0);
1768}
1769
1770int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1771 bpf_program_prep_t prep)
1772{
1773 int *instances_fds;
1774
1775 if (nr_instances <= 0 || !prep)
1776 return -EINVAL;
1777
1778 if (prog->instances.nr > 0 || prog->instances.fds) {
1779 pr_warning("Can't set pre-processor after loading\n");
1780 return -EINVAL;
1781 }
1782
1783 instances_fds = malloc(sizeof(int) * nr_instances);
1784 if (!instances_fds) {
1785 pr_warning("alloc memory failed for fds\n");
1786 return -ENOMEM;
1787 }
1788
1789 /* fill all fd with -1 */
1790 memset(instances_fds, -1, sizeof(int) * nr_instances);
1791
1792 prog->instances.nr = nr_instances;
1793 prog->instances.fds = instances_fds;
1794 prog->preprocessor = prep;
1795 return 0;
1796}
1797
1798int bpf_program__nth_fd(struct bpf_program *prog, int n)
1799{
1800 int fd;
1801
1802 if (n >= prog->instances.nr || n < 0) {
1803 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
1804 n, prog->section_name, prog->instances.nr);
1805 return -EINVAL;
1806 }
1807
1808 fd = prog->instances.fds[n];
1809 if (fd < 0) {
1810 pr_warning("%dth instance of program '%s' is invalid\n",
1811 n, prog->section_name);
1812 return -ENOENT;
1813 }
1814
1815 return fd;
1816}
1817
1818void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
1819{
1820 prog->type = type;
1821}
1822
1823static bool bpf_program__is_type(struct bpf_program *prog,
1824 enum bpf_prog_type type)
1825{
1826 return prog ? (prog->type == type) : false;
1827}
1828
1829#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
1830int bpf_program__set_##NAME(struct bpf_program *prog) \
1831{ \
1832 if (!prog) \
1833 return -EINVAL; \
1834 bpf_program__set_type(prog, TYPE); \
1835 return 0; \
1836} \
1837 \
1838bool bpf_program__is_##NAME(struct bpf_program *prog) \
1839{ \
1840 return bpf_program__is_type(prog, TYPE); \
1841} \
1842
1843BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
1844BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
1845BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
1846BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
1847BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
1848BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
1849BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
1850
1851static void bpf_program__set_expected_attach_type(struct bpf_program *prog,
1852 enum bpf_attach_type type)
1853{
1854 prog->expected_attach_type = type;
1855}
1856
1857#define BPF_PROG_SEC_FULL(string, ptype, atype) \
1858 { string, sizeof(string) - 1, ptype, atype }
1859
1860#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0)
1861
1862#define BPF_SA_PROG_SEC(string, ptype) \
1863 BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype)
1864
1865static const struct {
1866 const char *sec;
1867 size_t len;
1868 enum bpf_prog_type prog_type;
1869 enum bpf_attach_type expected_attach_type;
1870} section_names[] = {
1871 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
1872 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),
1873 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),
1874 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
1875 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
1876 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),
1877 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
1878 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
1879 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
1880 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK),
1881 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE),
1882 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
1883 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
1884 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
1885 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS),
1886 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB),
1887 BPF_PROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG),
1888 BPF_SA_PROG_SEC("cgroup/bind4", BPF_CGROUP_INET4_BIND),
1889 BPF_SA_PROG_SEC("cgroup/bind6", BPF_CGROUP_INET6_BIND),
1890 BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT),
1891 BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT),
1892};
1893
1894#undef BPF_PROG_SEC
1895#undef BPF_PROG_SEC_FULL
1896#undef BPF_SA_PROG_SEC
1897
1898static int bpf_program__identify_section(struct bpf_program *prog)
1899{
1900 int i;
1901
1902 if (!prog->section_name)
1903 goto err;
1904
1905 for (i = 0; i < ARRAY_SIZE(section_names); i++)
1906 if (strncmp(prog->section_name, section_names[i].sec,
1907 section_names[i].len) == 0)
1908 return i;
1909
1910err:
1911 pr_warning("failed to guess program type based on section name %s\n",
1912 prog->section_name);
1913
1914 return -1;
1915}
1916
1917int bpf_map__fd(struct bpf_map *map)
1918{
1919 return map ? map->fd : -EINVAL;
1920}
1921
1922const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
1923{
1924 return map ? &map->def : ERR_PTR(-EINVAL);
1925}
1926
1927const char *bpf_map__name(struct bpf_map *map)
1928{
1929 return map ? map->name : NULL;
1930}
1931
1932int bpf_map__set_priv(struct bpf_map *map, void *priv,
1933 bpf_map_clear_priv_t clear_priv)
1934{
1935 if (!map)
1936 return -EINVAL;
1937
1938 if (map->priv) {
1939 if (map->clear_priv)
1940 map->clear_priv(map, map->priv);
1941 }
1942
1943 map->priv = priv;
1944 map->clear_priv = clear_priv;
1945 return 0;
1946}
1947
1948void *bpf_map__priv(struct bpf_map *map)
1949{
1950 return map ? map->priv : ERR_PTR(-EINVAL);
1951}
1952
1953struct bpf_map *
1954bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
1955{
1956 size_t idx;
1957 struct bpf_map *s, *e;
1958
1959 if (!obj || !obj->maps)
1960 return NULL;
1961
1962 s = obj->maps;
1963 e = obj->maps + obj->nr_maps;
1964
1965 if (prev == NULL)
1966 return s;
1967
1968 if ((prev < s) || (prev >= e)) {
1969 pr_warning("error in %s: map handler doesn't belong to object\n",
1970 __func__);
1971 return NULL;
1972 }
1973
1974 idx = (prev - obj->maps) + 1;
1975 if (idx >= obj->nr_maps)
1976 return NULL;
1977 return &obj->maps[idx];
1978}
1979
1980struct bpf_map *
1981bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
1982{
1983 struct bpf_map *pos;
1984
1985 bpf_map__for_each(pos, obj) {
1986 if (pos->name && !strcmp(pos->name, name))
1987 return pos;
1988 }
1989 return NULL;
1990}
1991
1992struct bpf_map *
1993bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
1994{
1995 int i;
1996
1997 for (i = 0; i < obj->nr_maps; i++) {
1998 if (obj->maps[i].offset == offset)
1999 return &obj->maps[i];
2000 }
2001 return ERR_PTR(-ENOENT);
2002}
2003
2004long libbpf_get_error(const void *ptr)
2005{
2006 if (IS_ERR(ptr))
2007 return PTR_ERR(ptr);
2008 return 0;
2009}
2010
2011int bpf_prog_load(const char *file, enum bpf_prog_type type,
2012 struct bpf_object **pobj, int *prog_fd)
2013{
2014 struct bpf_prog_load_attr attr;
2015
2016 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2017 attr.file = file;
2018 attr.prog_type = type;
2019 attr.expected_attach_type = 0;
2020
2021 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2022}
2023
2024int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2025 struct bpf_object **pobj, int *prog_fd)
2026{
2027 struct bpf_program *prog, *first_prog = NULL;
2028 enum bpf_attach_type expected_attach_type;
2029 enum bpf_prog_type prog_type;
2030 struct bpf_object *obj;
2031 int section_idx;
2032 int err;
2033
2034 if (!attr)
2035 return -EINVAL;
2036
2037 obj = bpf_object__open(attr->file);
2038 if (IS_ERR_OR_NULL(obj))
2039 return -ENOENT;
2040
2041 bpf_object__for_each_program(prog, obj) {
2042 /*
2043 * If type is not specified, try to guess it based on
2044 * section name.
2045 */
2046 prog_type = attr->prog_type;
2047 expected_attach_type = attr->expected_attach_type;
2048 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2049 section_idx = bpf_program__identify_section(prog);
2050 if (section_idx < 0) {
2051 bpf_object__close(obj);
2052 return -EINVAL;
2053 }
2054 prog_type = section_names[section_idx].prog_type;
2055 expected_attach_type =
2056 section_names[section_idx].expected_attach_type;
2057 }
2058
2059 bpf_program__set_type(prog, prog_type);
2060 bpf_program__set_expected_attach_type(prog,
2061 expected_attach_type);
2062
2063 if (prog->idx != obj->efile.text_shndx && !first_prog)
2064 first_prog = prog;
2065 }
2066
2067 if (!first_prog) {
2068 pr_warning("object file doesn't contain bpf program\n");
2069 bpf_object__close(obj);
2070 return -ENOENT;
2071 }
2072
2073 err = bpf_object__load(obj);
2074 if (err) {
2075 bpf_object__close(obj);
2076 return -EINVAL;
2077 }
2078
2079 *pobj = obj;
2080 *prog_fd = bpf_program__fd(first_prog);
2081 return 0;
2082}