Loading...
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <errno.h>
14#include <linux/err.h>
15#include <linux/btf.h>
16#include <elf.h>
17#include <libelf.h>
18#include <fcntl.h>
19#include "libbpf.h"
20#include "btf.h"
21#include "libbpf_internal.h"
22#include "strset.h"
23#include "str_error.h"
24
25#define BTF_EXTERN_SEC ".extern"
26
27struct src_sec {
28 const char *sec_name;
29 /* positional (not necessarily ELF) index in an array of sections */
30 int id;
31 /* positional (not necessarily ELF) index of a matching section in a final object file */
32 int dst_id;
33 /* section data offset in a matching output section */
34 int dst_off;
35 /* whether section is omitted from the final ELF file */
36 bool skipped;
37 /* whether section is an ephemeral section, not mapped to an ELF section */
38 bool ephemeral;
39
40 /* ELF info */
41 size_t sec_idx;
42 Elf_Scn *scn;
43 Elf64_Shdr *shdr;
44 Elf_Data *data;
45
46 /* corresponding BTF DATASEC type ID */
47 int sec_type_id;
48};
49
50struct src_obj {
51 const char *filename;
52 int fd;
53 Elf *elf;
54 /* Section header strings section index */
55 size_t shstrs_sec_idx;
56 /* SYMTAB section index */
57 size_t symtab_sec_idx;
58
59 struct btf *btf;
60 struct btf_ext *btf_ext;
61
62 /* List of sections (including ephemeral). Slot zero is unused. */
63 struct src_sec *secs;
64 int sec_cnt;
65
66 /* mapping of symbol indices from src to dst ELF */
67 int *sym_map;
68 /* mapping from the src BTF type IDs to dst ones */
69 int *btf_type_map;
70};
71
72/* single .BTF.ext data section */
73struct btf_ext_sec_data {
74 size_t rec_cnt;
75 __u32 rec_sz;
76 void *recs;
77};
78
79struct glob_sym {
80 /* ELF symbol index */
81 int sym_idx;
82 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
83 int sec_id;
84 /* extern name offset in STRTAB */
85 int name_off;
86 /* optional associated BTF type ID */
87 int btf_id;
88 /* BTF type ID to which VAR/FUNC type is pointing to; used for
89 * rewriting types when extern VAR/FUNC is resolved to a concrete
90 * definition
91 */
92 int underlying_btf_id;
93 /* sec_var index in the corresponding dst_sec, if exists */
94 int var_idx;
95
96 /* extern or resolved/global symbol */
97 bool is_extern;
98 /* weak or strong symbol, never goes back from strong to weak */
99 bool is_weak;
100};
101
102struct dst_sec {
103 char *sec_name;
104 /* positional (not necessarily ELF) index in an array of sections */
105 int id;
106
107 bool ephemeral;
108
109 /* ELF info */
110 size_t sec_idx;
111 Elf_Scn *scn;
112 Elf64_Shdr *shdr;
113 Elf_Data *data;
114
115 /* final output section size */
116 int sec_sz;
117 /* final output contents of the section */
118 void *raw_data;
119
120 /* corresponding STT_SECTION symbol index in SYMTAB */
121 int sec_sym_idx;
122
123 /* section's DATASEC variable info, emitted on BTF finalization */
124 bool has_btf;
125 int sec_var_cnt;
126 struct btf_var_secinfo *sec_vars;
127
128 /* section's .BTF.ext data */
129 struct btf_ext_sec_data func_info;
130 struct btf_ext_sec_data line_info;
131 struct btf_ext_sec_data core_relo_info;
132};
133
134struct bpf_linker {
135 char *filename;
136 int fd;
137 Elf *elf;
138 Elf64_Ehdr *elf_hdr;
139 bool swapped_endian;
140
141 /* Output sections metadata */
142 struct dst_sec *secs;
143 int sec_cnt;
144
145 struct strset *strtab_strs; /* STRTAB unique strings */
146 size_t strtab_sec_idx; /* STRTAB section index */
147 size_t symtab_sec_idx; /* SYMTAB section index */
148
149 struct btf *btf;
150 struct btf_ext *btf_ext;
151
152 /* global (including extern) ELF symbols */
153 int glob_sym_cnt;
154 struct glob_sym *glob_syms;
155};
156
157#define pr_warn_elf(fmt, ...) \
158 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
159
160static int init_output_elf(struct bpf_linker *linker, const char *file);
161
162static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
163 const struct bpf_linker_file_opts *opts,
164 struct src_obj *obj);
165static int linker_sanity_check_elf(struct src_obj *obj);
166static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
167static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
168static int linker_sanity_check_btf(struct src_obj *obj);
169static int linker_sanity_check_btf_ext(struct src_obj *obj);
170static int linker_fixup_btf(struct src_obj *obj);
171static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
172static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
173static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
174 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
175static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
176static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
177static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
178
179static int finalize_btf(struct bpf_linker *linker);
180static int finalize_btf_ext(struct bpf_linker *linker);
181
182void bpf_linker__free(struct bpf_linker *linker)
183{
184 int i;
185
186 if (!linker)
187 return;
188
189 free(linker->filename);
190
191 if (linker->elf)
192 elf_end(linker->elf);
193
194 if (linker->fd >= 0)
195 close(linker->fd);
196
197 strset__free(linker->strtab_strs);
198
199 btf__free(linker->btf);
200 btf_ext__free(linker->btf_ext);
201
202 for (i = 1; i < linker->sec_cnt; i++) {
203 struct dst_sec *sec = &linker->secs[i];
204
205 free(sec->sec_name);
206 free(sec->raw_data);
207 free(sec->sec_vars);
208
209 free(sec->func_info.recs);
210 free(sec->line_info.recs);
211 free(sec->core_relo_info.recs);
212 }
213 free(linker->secs);
214
215 free(linker->glob_syms);
216 free(linker);
217}
218
219struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
220{
221 struct bpf_linker *linker;
222 int err;
223
224 if (!OPTS_VALID(opts, bpf_linker_opts))
225 return errno = EINVAL, NULL;
226
227 if (elf_version(EV_CURRENT) == EV_NONE) {
228 pr_warn_elf("libelf initialization failed");
229 return errno = EINVAL, NULL;
230 }
231
232 linker = calloc(1, sizeof(*linker));
233 if (!linker)
234 return errno = ENOMEM, NULL;
235
236 linker->fd = -1;
237
238 err = init_output_elf(linker, filename);
239 if (err)
240 goto err_out;
241
242 return linker;
243
244err_out:
245 bpf_linker__free(linker);
246 return errno = -err, NULL;
247}
248
249static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
250{
251 struct dst_sec *secs = linker->secs, *sec;
252 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
253
254 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
255 if (!secs)
256 return NULL;
257
258 /* zero out newly allocated memory */
259 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
260
261 linker->secs = secs;
262 linker->sec_cnt = new_cnt;
263
264 sec = &linker->secs[new_cnt - 1];
265 sec->id = new_cnt - 1;
266 sec->sec_name = strdup(sec_name);
267 if (!sec->sec_name)
268 return NULL;
269
270 return sec;
271}
272
273static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
274{
275 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
276 Elf64_Sym *syms, *sym;
277 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
278
279 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
280 if (!syms)
281 return NULL;
282
283 sym = &syms[sym_cnt];
284 memset(sym, 0, sizeof(*sym));
285
286 symtab->raw_data = syms;
287 symtab->sec_sz += sizeof(*sym);
288 symtab->shdr->sh_size += sizeof(*sym);
289 symtab->data->d_size += sizeof(*sym);
290
291 if (sym_idx)
292 *sym_idx = sym_cnt;
293
294 return sym;
295}
296
297static int init_output_elf(struct bpf_linker *linker, const char *file)
298{
299 int err, str_off;
300 Elf64_Sym *init_sym;
301 struct dst_sec *sec;
302
303 linker->filename = strdup(file);
304 if (!linker->filename)
305 return -ENOMEM;
306
307 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
308 if (linker->fd < 0) {
309 err = -errno;
310 pr_warn("failed to create '%s': %s\n", file, errstr(err));
311 return err;
312 }
313
314 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
315 if (!linker->elf) {
316 pr_warn_elf("failed to create ELF object");
317 return -EINVAL;
318 }
319
320 /* ELF header */
321 linker->elf_hdr = elf64_newehdr(linker->elf);
322 if (!linker->elf_hdr) {
323 pr_warn_elf("failed to create ELF header");
324 return -EINVAL;
325 }
326
327 linker->elf_hdr->e_machine = EM_BPF;
328 linker->elf_hdr->e_type = ET_REL;
329 /* Set unknown ELF endianness, assign later from input files */
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
331
332 /* STRTAB */
333 /* initialize strset with an empty string to conform to ELF */
334 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
335 if (libbpf_get_error(linker->strtab_strs))
336 return libbpf_get_error(linker->strtab_strs);
337
338 sec = add_dst_sec(linker, ".strtab");
339 if (!sec)
340 return -ENOMEM;
341
342 sec->scn = elf_newscn(linker->elf);
343 if (!sec->scn) {
344 pr_warn_elf("failed to create STRTAB section");
345 return -EINVAL;
346 }
347
348 sec->shdr = elf64_getshdr(sec->scn);
349 if (!sec->shdr)
350 return -EINVAL;
351
352 sec->data = elf_newdata(sec->scn);
353 if (!sec->data) {
354 pr_warn_elf("failed to create STRTAB data");
355 return -EINVAL;
356 }
357
358 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
359 if (str_off < 0)
360 return str_off;
361
362 sec->sec_idx = elf_ndxscn(sec->scn);
363 linker->elf_hdr->e_shstrndx = sec->sec_idx;
364 linker->strtab_sec_idx = sec->sec_idx;
365
366 sec->shdr->sh_name = str_off;
367 sec->shdr->sh_type = SHT_STRTAB;
368 sec->shdr->sh_flags = SHF_STRINGS;
369 sec->shdr->sh_offset = 0;
370 sec->shdr->sh_link = 0;
371 sec->shdr->sh_info = 0;
372 sec->shdr->sh_addralign = 1;
373 sec->shdr->sh_size = sec->sec_sz = 0;
374 sec->shdr->sh_entsize = 0;
375
376 /* SYMTAB */
377 sec = add_dst_sec(linker, ".symtab");
378 if (!sec)
379 return -ENOMEM;
380
381 sec->scn = elf_newscn(linker->elf);
382 if (!sec->scn) {
383 pr_warn_elf("failed to create SYMTAB section");
384 return -EINVAL;
385 }
386
387 sec->shdr = elf64_getshdr(sec->scn);
388 if (!sec->shdr)
389 return -EINVAL;
390
391 sec->data = elf_newdata(sec->scn);
392 if (!sec->data) {
393 pr_warn_elf("failed to create SYMTAB data");
394 return -EINVAL;
395 }
396 /* Ensure libelf translates byte-order of symbol records */
397 sec->data->d_type = ELF_T_SYM;
398
399 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
400 if (str_off < 0)
401 return str_off;
402
403 sec->sec_idx = elf_ndxscn(sec->scn);
404 linker->symtab_sec_idx = sec->sec_idx;
405
406 sec->shdr->sh_name = str_off;
407 sec->shdr->sh_type = SHT_SYMTAB;
408 sec->shdr->sh_flags = 0;
409 sec->shdr->sh_offset = 0;
410 sec->shdr->sh_link = linker->strtab_sec_idx;
411 /* sh_info should be one greater than the index of the last local
412 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
413 */
414 sec->shdr->sh_info = 0;
415 sec->shdr->sh_addralign = 8;
416 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
417
418 /* .BTF */
419 linker->btf = btf__new_empty();
420 err = libbpf_get_error(linker->btf);
421 if (err)
422 return err;
423
424 /* add the special all-zero symbol */
425 init_sym = add_new_sym(linker, NULL);
426 if (!init_sym)
427 return -EINVAL;
428
429 init_sym->st_name = 0;
430 init_sym->st_info = 0;
431 init_sym->st_other = 0;
432 init_sym->st_shndx = SHN_UNDEF;
433 init_sym->st_value = 0;
434 init_sym->st_size = 0;
435
436 return 0;
437}
438
439int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
440 const struct bpf_linker_file_opts *opts)
441{
442 struct src_obj obj = {};
443 int err = 0;
444
445 if (!OPTS_VALID(opts, bpf_linker_file_opts))
446 return libbpf_err(-EINVAL);
447
448 if (!linker->elf)
449 return libbpf_err(-EINVAL);
450
451 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
452 err = err ?: linker_append_sec_data(linker, &obj);
453 err = err ?: linker_append_elf_syms(linker, &obj);
454 err = err ?: linker_append_elf_relos(linker, &obj);
455 err = err ?: linker_append_btf(linker, &obj);
456 err = err ?: linker_append_btf_ext(linker, &obj);
457
458 /* free up src_obj resources */
459 free(obj.btf_type_map);
460 btf__free(obj.btf);
461 btf_ext__free(obj.btf_ext);
462 free(obj.secs);
463 free(obj.sym_map);
464 if (obj.elf)
465 elf_end(obj.elf);
466 if (obj.fd >= 0)
467 close(obj.fd);
468
469 return libbpf_err(err);
470}
471
472static bool is_dwarf_sec_name(const char *name)
473{
474 /* approximation, but the actual list is too long */
475 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
476}
477
478static bool is_ignored_sec(struct src_sec *sec)
479{
480 Elf64_Shdr *shdr = sec->shdr;
481 const char *name = sec->sec_name;
482
483 /* no special handling of .strtab */
484 if (shdr->sh_type == SHT_STRTAB)
485 return true;
486
487 /* ignore .llvm_addrsig section as well */
488 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
489 return true;
490
491 /* no subprograms will lead to an empty .text section, ignore it */
492 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
493 strcmp(sec->sec_name, ".text") == 0)
494 return true;
495
496 /* DWARF sections */
497 if (is_dwarf_sec_name(sec->sec_name))
498 return true;
499
500 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
501 name += sizeof(".rel") - 1;
502 /* DWARF section relocations */
503 if (is_dwarf_sec_name(name))
504 return true;
505
506 /* .BTF and .BTF.ext don't need relocations */
507 if (strcmp(name, BTF_ELF_SEC) == 0 ||
508 strcmp(name, BTF_EXT_ELF_SEC) == 0)
509 return true;
510 }
511
512 return false;
513}
514
515static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
516{
517 struct src_sec *secs = obj->secs, *sec;
518 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
519
520 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
521 if (!secs)
522 return NULL;
523
524 /* zero out newly allocated memory */
525 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
526
527 obj->secs = secs;
528 obj->sec_cnt = new_cnt;
529
530 sec = &obj->secs[new_cnt - 1];
531 sec->id = new_cnt - 1;
532 sec->sec_name = sec_name;
533
534 return sec;
535}
536
537static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
538 const struct bpf_linker_file_opts *opts,
539 struct src_obj *obj)
540{
541 int err = 0;
542 Elf_Scn *scn;
543 Elf_Data *data;
544 Elf64_Ehdr *ehdr;
545 Elf64_Shdr *shdr;
546 struct src_sec *sec;
547 unsigned char obj_byteorder;
548 unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
549#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
550 const unsigned char host_byteorder = ELFDATA2LSB;
551#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
552 const unsigned char host_byteorder = ELFDATA2MSB;
553#else
554#error "Unknown __BYTE_ORDER__"
555#endif
556
557 pr_debug("linker: adding object file '%s'...\n", filename);
558
559 obj->filename = filename;
560
561 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
562 if (obj->fd < 0) {
563 err = -errno;
564 pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
565 return err;
566 }
567 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
568 if (!obj->elf) {
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return -EINVAL;
571 }
572
573 /* Sanity check ELF file high-level properties */
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 pr_warn_elf("failed to get ELF header for %s", filename);
577 return -EINVAL;
578 }
579
580 /* Linker output endianness set by first input object */
581 obj_byteorder = ehdr->e_ident[EI_DATA];
582 if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
583 err = -EOPNOTSUPP;
584 pr_warn("unknown byte order of ELF file %s\n", filename);
585 return err;
586 }
587 if (link_byteorder == ELFDATANONE) {
588 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
589 linker->swapped_endian = obj_byteorder != host_byteorder;
590 pr_debug("linker: set %s-endian output byte order\n",
591 obj_byteorder == ELFDATA2MSB ? "big" : "little");
592 } else if (link_byteorder != obj_byteorder) {
593 err = -EOPNOTSUPP;
594 pr_warn("byte order mismatch with ELF file %s\n", filename);
595 return err;
596 }
597
598 if (ehdr->e_type != ET_REL
599 || ehdr->e_machine != EM_BPF
600 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
601 err = -EOPNOTSUPP;
602 pr_warn_elf("unsupported kind of ELF file %s", filename);
603 return err;
604 }
605
606 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
607 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
608 return -EINVAL;
609 }
610
611 scn = NULL;
612 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
613 size_t sec_idx = elf_ndxscn(scn);
614 const char *sec_name;
615
616 shdr = elf64_getshdr(scn);
617 if (!shdr) {
618 pr_warn_elf("failed to get section #%zu header for %s",
619 sec_idx, filename);
620 return -EINVAL;
621 }
622
623 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
624 if (!sec_name) {
625 pr_warn_elf("failed to get section #%zu name for %s",
626 sec_idx, filename);
627 return -EINVAL;
628 }
629
630 data = elf_getdata(scn, 0);
631 if (!data) {
632 pr_warn_elf("failed to get section #%zu (%s) data from %s",
633 sec_idx, sec_name, filename);
634 return -EINVAL;
635 }
636
637 sec = add_src_sec(obj, sec_name);
638 if (!sec)
639 return -ENOMEM;
640
641 sec->scn = scn;
642 sec->shdr = shdr;
643 sec->data = data;
644 sec->sec_idx = elf_ndxscn(scn);
645
646 if (is_ignored_sec(sec)) {
647 sec->skipped = true;
648 continue;
649 }
650
651 switch (shdr->sh_type) {
652 case SHT_SYMTAB:
653 if (obj->symtab_sec_idx) {
654 err = -EOPNOTSUPP;
655 pr_warn("multiple SYMTAB sections found, not supported\n");
656 return err;
657 }
658 obj->symtab_sec_idx = sec_idx;
659 break;
660 case SHT_STRTAB:
661 /* we'll construct our own string table */
662 break;
663 case SHT_PROGBITS:
664 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
665 obj->btf = btf__new(data->d_buf, shdr->sh_size);
666 err = libbpf_get_error(obj->btf);
667 if (err) {
668 pr_warn("failed to parse .BTF from %s: %s\n",
669 filename, errstr(err));
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
676 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
677 err = libbpf_get_error(obj->btf_ext);
678 if (err) {
679 pr_warn("failed to parse .BTF.ext from '%s': %s\n",
680 filename, errstr(err));
681 return err;
682 }
683 sec->skipped = true;
684 continue;
685 }
686
687 /* data & code */
688 break;
689 case SHT_NOBITS:
690 /* BSS */
691 break;
692 case SHT_REL:
693 /* relocations */
694 break;
695 default:
696 pr_warn("unrecognized section #%zu (%s) in %s\n",
697 sec_idx, sec_name, filename);
698 err = -EINVAL;
699 return err;
700 }
701 }
702
703 err = err ?: linker_sanity_check_elf(obj);
704 err = err ?: linker_sanity_check_btf(obj);
705 err = err ?: linker_sanity_check_btf_ext(obj);
706 err = err ?: linker_fixup_btf(obj);
707
708 return err;
709}
710
711static int linker_sanity_check_elf(struct src_obj *obj)
712{
713 struct src_sec *sec;
714 int i, err;
715
716 if (!obj->symtab_sec_idx) {
717 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
718 return -EINVAL;
719 }
720 if (!obj->shstrs_sec_idx) {
721 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
722 return -EINVAL;
723 }
724
725 for (i = 1; i < obj->sec_cnt; i++) {
726 sec = &obj->secs[i];
727
728 if (sec->sec_name[0] == '\0') {
729 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
730 return -EINVAL;
731 }
732
733 if (is_dwarf_sec_name(sec->sec_name))
734 continue;
735
736 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
737 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
738 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
739 obj->filename);
740 return -EINVAL;
741 }
742 if (sec->shdr->sh_addralign != sec->data->d_align) {
743 pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
744 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
745 (long long unsigned)sec->data->d_align, obj->filename);
746 return -EINVAL;
747 }
748
749 if (sec->shdr->sh_size != sec->data->d_size) {
750 pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
751 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
752 (long long unsigned)sec->data->d_size, obj->filename);
753 return -EINVAL;
754 }
755
756 switch (sec->shdr->sh_type) {
757 case SHT_SYMTAB:
758 err = linker_sanity_check_elf_symtab(obj, sec);
759 if (err)
760 return err;
761 break;
762 case SHT_STRTAB:
763 break;
764 case SHT_PROGBITS:
765 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
766 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
767 pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
768 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
769 obj->filename);
770 return -EINVAL;
771 }
772 }
773 break;
774 case SHT_NOBITS:
775 break;
776 case SHT_REL:
777 err = linker_sanity_check_elf_relos(obj, sec);
778 if (err)
779 return err;
780 break;
781 case SHT_LLVM_ADDRSIG:
782 break;
783 default:
784 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
785 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
786 return -EINVAL;
787 }
788 }
789
790 return 0;
791}
792
793static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
794{
795 struct src_sec *link_sec;
796 Elf64_Sym *sym;
797 int i, n;
798
799 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
800 return -EINVAL;
801 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
802 return -EINVAL;
803
804 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
805 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
806 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
807 return -EINVAL;
808 }
809 link_sec = &obj->secs[sec->shdr->sh_link];
810 if (link_sec->shdr->sh_type != SHT_STRTAB) {
811 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
812 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
813 return -EINVAL;
814 }
815
816 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
817 sym = sec->data->d_buf;
818 for (i = 0; i < n; i++, sym++) {
819 int sym_type = ELF64_ST_TYPE(sym->st_info);
820 int sym_bind = ELF64_ST_BIND(sym->st_info);
821 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
822
823 if (i == 0) {
824 if (sym->st_name != 0 || sym->st_info != 0
825 || sym->st_other != 0 || sym->st_shndx != 0
826 || sym->st_value != 0 || sym->st_size != 0) {
827 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
828 return -EINVAL;
829 }
830 continue;
831 }
832 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
833 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
834 i, sec->sec_idx, sym_bind);
835 return -EINVAL;
836 }
837 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
838 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
839 i, sec->sec_idx, sym_vis);
840 return -EINVAL;
841 }
842 if (sym->st_shndx == 0) {
843 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
844 || sym->st_value != 0 || sym->st_size != 0) {
845 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
846 i, obj->filename);
847
848 return -EINVAL;
849 }
850 continue;
851 }
852 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
853 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
854 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
855 return -EINVAL;
856 }
857 if (sym_type == STT_SECTION) {
858 if (sym->st_value != 0)
859 return -EINVAL;
860 continue;
861 }
862 }
863
864 return 0;
865}
866
867static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
868{
869 struct src_sec *link_sec, *sym_sec;
870 Elf64_Rel *relo;
871 int i, n;
872
873 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
874 return -EINVAL;
875 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
876 return -EINVAL;
877
878 /* SHT_REL's sh_link should point to SYMTAB */
879 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
880 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
881 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
882 return -EINVAL;
883 }
884
885 /* SHT_REL's sh_info points to relocated section */
886 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
887 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
888 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
889 return -EINVAL;
890 }
891 link_sec = &obj->secs[sec->shdr->sh_info];
892
893 /* .rel<secname> -> <secname> pattern is followed */
894 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
895 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
896 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
897 sec->sec_idx, obj->filename);
898 return -EINVAL;
899 }
900
901 /* don't further validate relocations for ignored sections */
902 if (link_sec->skipped)
903 return 0;
904
905 /* relocatable section is data or instructions */
906 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
907 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
908 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
909 return -EINVAL;
910 }
911
912 /* check sanity of each relocation */
913 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
914 relo = sec->data->d_buf;
915 sym_sec = &obj->secs[obj->symtab_sec_idx];
916 for (i = 0; i < n; i++, relo++) {
917 size_t sym_idx = ELF64_R_SYM(relo->r_info);
918 size_t sym_type = ELF64_R_TYPE(relo->r_info);
919
920 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
921 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
922 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
923 i, sec->sec_idx, sym_type, obj->filename);
924 return -EINVAL;
925 }
926
927 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
928 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
929 i, sec->sec_idx, sym_idx, obj->filename);
930 return -EINVAL;
931 }
932
933 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
934 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
935 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
936 i, sec->sec_idx, sym_idx, obj->filename);
937 return -EINVAL;
938 }
939 }
940 }
941
942 return 0;
943}
944
945static int check_btf_type_id(__u32 *type_id, void *ctx)
946{
947 struct btf *btf = ctx;
948
949 if (*type_id >= btf__type_cnt(btf))
950 return -EINVAL;
951
952 return 0;
953}
954
955static int check_btf_str_off(__u32 *str_off, void *ctx)
956{
957 struct btf *btf = ctx;
958 const char *s;
959
960 s = btf__str_by_offset(btf, *str_off);
961
962 if (!s)
963 return -EINVAL;
964
965 return 0;
966}
967
968static int linker_sanity_check_btf(struct src_obj *obj)
969{
970 struct btf_type *t;
971 int i, n, err;
972
973 if (!obj->btf)
974 return 0;
975
976 n = btf__type_cnt(obj->btf);
977 for (i = 1; i < n; i++) {
978 struct btf_field_iter it;
979 __u32 *type_id, *str_off;
980
981 t = btf_type_by_id(obj->btf, i);
982
983 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
984 if (err)
985 return err;
986 while ((type_id = btf_field_iter_next(&it))) {
987 if (*type_id >= n)
988 return -EINVAL;
989 }
990
991 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
992 if (err)
993 return err;
994 while ((str_off = btf_field_iter_next(&it))) {
995 if (!btf__str_by_offset(obj->btf, *str_off))
996 return -EINVAL;
997 }
998 }
999
1000 return 0;
1001}
1002
1003static int linker_sanity_check_btf_ext(struct src_obj *obj)
1004{
1005 int err = 0;
1006
1007 if (!obj->btf_ext)
1008 return 0;
1009
1010 /* can't use .BTF.ext without .BTF */
1011 if (!obj->btf)
1012 return -EINVAL;
1013
1014 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1015 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1016 if (err)
1017 return err;
1018
1019 return 0;
1020}
1021
1022static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1023{
1024 Elf_Scn *scn;
1025 Elf_Data *data;
1026 Elf64_Shdr *shdr;
1027 int name_off;
1028
1029 dst_sec->sec_sz = 0;
1030 dst_sec->sec_idx = 0;
1031 dst_sec->ephemeral = src_sec->ephemeral;
1032
1033 /* ephemeral sections are just thin section shells lacking most parts */
1034 if (src_sec->ephemeral)
1035 return 0;
1036
1037 scn = elf_newscn(linker->elf);
1038 if (!scn)
1039 return -ENOMEM;
1040 data = elf_newdata(scn);
1041 if (!data)
1042 return -ENOMEM;
1043 shdr = elf64_getshdr(scn);
1044 if (!shdr)
1045 return -ENOMEM;
1046
1047 dst_sec->scn = scn;
1048 dst_sec->shdr = shdr;
1049 dst_sec->data = data;
1050 dst_sec->sec_idx = elf_ndxscn(scn);
1051
1052 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1053 if (name_off < 0)
1054 return name_off;
1055
1056 shdr->sh_name = name_off;
1057 shdr->sh_type = src_sec->shdr->sh_type;
1058 shdr->sh_flags = src_sec->shdr->sh_flags;
1059 shdr->sh_size = 0;
1060 /* sh_link and sh_info have different meaning for different types of
1061 * sections, so we leave it up to the caller code to fill them in, if
1062 * necessary
1063 */
1064 shdr->sh_link = 0;
1065 shdr->sh_info = 0;
1066 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1067 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1068
1069 data->d_type = src_sec->data->d_type;
1070 data->d_size = 0;
1071 data->d_buf = NULL;
1072 data->d_align = src_sec->data->d_align;
1073 data->d_off = 0;
1074
1075 return 0;
1076}
1077
1078static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1079{
1080 struct dst_sec *sec;
1081 int i;
1082
1083 for (i = 1; i < linker->sec_cnt; i++) {
1084 sec = &linker->secs[i];
1085
1086 if (strcmp(sec->sec_name, sec_name) == 0)
1087 return sec;
1088 }
1089
1090 return NULL;
1091}
1092
1093static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1094{
1095 if (dst->ephemeral || src->ephemeral)
1096 return true;
1097
1098 if (dst->shdr->sh_type != src->shdr->sh_type) {
1099 pr_warn("sec %s types mismatch\n", dst->sec_name);
1100 return false;
1101 }
1102 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1103 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1104 return false;
1105 }
1106 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1107 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1108 return false;
1109 }
1110
1111 return true;
1112}
1113
1114static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1115{
1116 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1117 return false;
1118 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1119 return false;
1120 return true;
1121}
1122
1123static bool is_exec_sec(struct dst_sec *sec)
1124{
1125 if (!sec || sec->ephemeral)
1126 return false;
1127 return (sec->shdr->sh_type == SHT_PROGBITS) &&
1128 (sec->shdr->sh_flags & SHF_EXECINSTR);
1129}
1130
1131static void exec_sec_bswap(void *raw_data, int size)
1132{
1133 const int insn_cnt = size / sizeof(struct bpf_insn);
1134 struct bpf_insn *insn = raw_data;
1135 int i;
1136
1137 for (i = 0; i < insn_cnt; i++, insn++)
1138 bpf_insn_bswap(insn);
1139}
1140
1141static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1142{
1143 void *tmp;
1144 size_t dst_align, src_align;
1145 size_t dst_align_sz, dst_final_sz;
1146 int err;
1147
1148 /* Ephemeral source section doesn't contribute anything to ELF
1149 * section data.
1150 */
1151 if (src->ephemeral)
1152 return 0;
1153
1154 /* Some sections (like .maps) can contain both externs (and thus be
1155 * ephemeral) and non-externs (map definitions). So it's possible that
1156 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1157 * first non-ephemeral entity appears. In such case, we add ELF
1158 * section, data, etc.
1159 */
1160 if (dst->ephemeral) {
1161 err = init_sec(linker, dst, src);
1162 if (err)
1163 return err;
1164 }
1165
1166 dst_align = dst->shdr->sh_addralign;
1167 src_align = src->shdr->sh_addralign;
1168 if (dst_align == 0)
1169 dst_align = 1;
1170 if (dst_align < src_align)
1171 dst_align = src_align;
1172
1173 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1174
1175 /* no need to re-align final size */
1176 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1177
1178 if (src->shdr->sh_type != SHT_NOBITS) {
1179 tmp = realloc(dst->raw_data, dst_final_sz);
1180 /* If dst_align_sz == 0, realloc() behaves in a special way:
1181 * 1. When dst->raw_data is NULL it returns:
1182 * "either NULL or a pointer suitable to be passed to free()" [1].
1183 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1184 * thus invalidating any "pointer suitable to be passed to free()" obtained
1185 * at step (1).
1186 *
1187 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1188 * dst->raw_data would be freed again in bpf_linker__free().
1189 *
1190 * [1] man 3 realloc
1191 */
1192 if (!tmp && dst_align_sz > 0)
1193 return -ENOMEM;
1194 dst->raw_data = tmp;
1195
1196 /* pad dst section, if it's alignment forced size increase */
1197 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1198 /* now copy src data at a properly aligned offset */
1199 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1200
1201 /* convert added bpf insns to native byte-order */
1202 if (linker->swapped_endian && is_exec_sec(dst))
1203 exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1204 }
1205
1206 dst->sec_sz = dst_final_sz;
1207 dst->shdr->sh_size = dst_final_sz;
1208 dst->data->d_size = dst_final_sz;
1209
1210 dst->shdr->sh_addralign = dst_align;
1211 dst->data->d_align = dst_align;
1212
1213 src->dst_off = dst_align_sz;
1214
1215 return 0;
1216}
1217
1218static bool is_data_sec(struct src_sec *sec)
1219{
1220 if (!sec || sec->skipped)
1221 return false;
1222 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1223 if (sec->ephemeral)
1224 return true;
1225 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1226}
1227
1228static bool is_relo_sec(struct src_sec *sec)
1229{
1230 if (!sec || sec->skipped || sec->ephemeral)
1231 return false;
1232 return sec->shdr->sh_type == SHT_REL;
1233}
1234
1235static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1236{
1237 int i, err;
1238
1239 for (i = 1; i < obj->sec_cnt; i++) {
1240 struct src_sec *src_sec;
1241 struct dst_sec *dst_sec;
1242
1243 src_sec = &obj->secs[i];
1244 if (!is_data_sec(src_sec))
1245 continue;
1246
1247 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1248 if (!dst_sec) {
1249 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1250 if (!dst_sec)
1251 return -ENOMEM;
1252 err = init_sec(linker, dst_sec, src_sec);
1253 if (err) {
1254 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1255 return err;
1256 }
1257 } else {
1258 if (!secs_match(dst_sec, src_sec)) {
1259 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1260 return -1;
1261 }
1262
1263 /* "license" and "version" sections are deduped */
1264 if (strcmp(src_sec->sec_name, "license") == 0
1265 || strcmp(src_sec->sec_name, "version") == 0) {
1266 if (!sec_content_is_same(dst_sec, src_sec)) {
1267 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1268 return -EINVAL;
1269 }
1270 src_sec->skipped = true;
1271 src_sec->dst_id = dst_sec->id;
1272 continue;
1273 }
1274 }
1275
1276 /* record mapped section index */
1277 src_sec->dst_id = dst_sec->id;
1278
1279 err = extend_sec(linker, dst_sec, src_sec);
1280 if (err)
1281 return err;
1282 }
1283
1284 return 0;
1285}
1286
1287static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1288{
1289 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1290 Elf64_Sym *sym = symtab->data->d_buf;
1291 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1292 int str_sec_idx = symtab->shdr->sh_link;
1293 const char *sym_name;
1294
1295 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1296 if (!obj->sym_map)
1297 return -ENOMEM;
1298
1299 for (i = 0; i < n; i++, sym++) {
1300 /* We already validated all-zero symbol #0 and we already
1301 * appended it preventively to the final SYMTAB, so skip it.
1302 */
1303 if (i == 0)
1304 continue;
1305
1306 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1307 if (!sym_name) {
1308 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1309 return -EINVAL;
1310 }
1311
1312 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1313 if (err)
1314 return err;
1315 }
1316
1317 return 0;
1318}
1319
1320static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1321{
1322 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1323 Elf64_Sym *syms = symtab->raw_data;
1324
1325 return &syms[sym_idx];
1326}
1327
1328static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1329{
1330 struct glob_sym *glob_sym;
1331 const char *name;
1332 int i;
1333
1334 for (i = 0; i < linker->glob_sym_cnt; i++) {
1335 glob_sym = &linker->glob_syms[i];
1336 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1337
1338 if (strcmp(name, sym_name) == 0)
1339 return glob_sym;
1340 }
1341
1342 return NULL;
1343}
1344
1345static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1346{
1347 struct glob_sym *syms, *sym;
1348
1349 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1350 sizeof(*linker->glob_syms));
1351 if (!syms)
1352 return NULL;
1353
1354 sym = &syms[linker->glob_sym_cnt];
1355 memset(sym, 0, sizeof(*sym));
1356 sym->var_idx = -1;
1357
1358 linker->glob_syms = syms;
1359 linker->glob_sym_cnt++;
1360
1361 return sym;
1362}
1363
1364static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1365 const struct btf *btf1, __u32 id1,
1366 const struct btf *btf2, __u32 id2)
1367{
1368 const struct btf_type *t1, *t2;
1369 bool is_static1, is_static2;
1370 const char *n1, *n2;
1371 int i, n;
1372
1373recur:
1374 n1 = n2 = NULL;
1375 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1376 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1377
1378 /* check if only one side is FWD, otherwise handle with common logic */
1379 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1380 n1 = btf__str_by_offset(btf1, t1->name_off);
1381 n2 = btf__str_by_offset(btf2, t2->name_off);
1382 if (strcmp(n1, n2) != 0) {
1383 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1384 sym_name, n1, n2);
1385 return false;
1386 }
1387 /* validate if FWD kind matches concrete kind */
1388 if (btf_is_fwd(t1)) {
1389 if (btf_kflag(t1) && btf_is_union(t2))
1390 return true;
1391 if (!btf_kflag(t1) && btf_is_struct(t2))
1392 return true;
1393 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1394 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1395 } else {
1396 if (btf_kflag(t2) && btf_is_union(t1))
1397 return true;
1398 if (!btf_kflag(t2) && btf_is_struct(t1))
1399 return true;
1400 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1401 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1402 }
1403 return false;
1404 }
1405
1406 if (btf_kind(t1) != btf_kind(t2)) {
1407 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1408 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1409 return false;
1410 }
1411
1412 switch (btf_kind(t1)) {
1413 case BTF_KIND_STRUCT:
1414 case BTF_KIND_UNION:
1415 case BTF_KIND_ENUM:
1416 case BTF_KIND_ENUM64:
1417 case BTF_KIND_FWD:
1418 case BTF_KIND_FUNC:
1419 case BTF_KIND_VAR:
1420 n1 = btf__str_by_offset(btf1, t1->name_off);
1421 n2 = btf__str_by_offset(btf2, t2->name_off);
1422 if (strcmp(n1, n2) != 0) {
1423 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1424 sym_name, btf_kind_str(t1), n1, n2);
1425 return false;
1426 }
1427 break;
1428 default:
1429 break;
1430 }
1431
1432 switch (btf_kind(t1)) {
1433 case BTF_KIND_UNKN: /* void */
1434 case BTF_KIND_FWD:
1435 return true;
1436 case BTF_KIND_INT:
1437 case BTF_KIND_FLOAT:
1438 case BTF_KIND_ENUM:
1439 case BTF_KIND_ENUM64:
1440 /* ignore encoding for int and enum values for enum */
1441 if (t1->size != t2->size) {
1442 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1443 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1444 return false;
1445 }
1446 return true;
1447 case BTF_KIND_PTR:
1448 /* just validate overall shape of the referenced type, so no
1449 * contents comparison for struct/union, and allowed fwd vs
1450 * struct/union
1451 */
1452 exact = false;
1453 id1 = t1->type;
1454 id2 = t2->type;
1455 goto recur;
1456 case BTF_KIND_ARRAY:
1457 /* ignore index type and array size */
1458 id1 = btf_array(t1)->type;
1459 id2 = btf_array(t2)->type;
1460 goto recur;
1461 case BTF_KIND_FUNC:
1462 /* extern and global linkages are compatible */
1463 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1464 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1465 if (is_static1 != is_static2) {
1466 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1467 return false;
1468 }
1469
1470 id1 = t1->type;
1471 id2 = t2->type;
1472 goto recur;
1473 case BTF_KIND_VAR:
1474 /* extern and global linkages are compatible */
1475 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1476 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1477 if (is_static1 != is_static2) {
1478 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1479 return false;
1480 }
1481
1482 id1 = t1->type;
1483 id2 = t2->type;
1484 goto recur;
1485 case BTF_KIND_STRUCT:
1486 case BTF_KIND_UNION: {
1487 const struct btf_member *m1, *m2;
1488
1489 if (!exact)
1490 return true;
1491
1492 if (btf_vlen(t1) != btf_vlen(t2)) {
1493 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1494 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1495 return false;
1496 }
1497
1498 n = btf_vlen(t1);
1499 m1 = btf_members(t1);
1500 m2 = btf_members(t2);
1501 for (i = 0; i < n; i++, m1++, m2++) {
1502 n1 = btf__str_by_offset(btf1, m1->name_off);
1503 n2 = btf__str_by_offset(btf2, m2->name_off);
1504 if (strcmp(n1, n2) != 0) {
1505 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1506 sym_name, i, n1, n2);
1507 return false;
1508 }
1509 if (m1->offset != m2->offset) {
1510 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1511 sym_name, i, n1);
1512 return false;
1513 }
1514 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1515 return false;
1516 }
1517
1518 return true;
1519 }
1520 case BTF_KIND_FUNC_PROTO: {
1521 const struct btf_param *m1, *m2;
1522
1523 if (btf_vlen(t1) != btf_vlen(t2)) {
1524 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1525 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1526 return false;
1527 }
1528
1529 n = btf_vlen(t1);
1530 m1 = btf_params(t1);
1531 m2 = btf_params(t2);
1532 for (i = 0; i < n; i++, m1++, m2++) {
1533 /* ignore func arg names */
1534 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1535 return false;
1536 }
1537
1538 /* now check return type as well */
1539 id1 = t1->type;
1540 id2 = t2->type;
1541 goto recur;
1542 }
1543
1544 /* skip_mods_and_typedefs() make this impossible */
1545 case BTF_KIND_TYPEDEF:
1546 case BTF_KIND_VOLATILE:
1547 case BTF_KIND_CONST:
1548 case BTF_KIND_RESTRICT:
1549 /* DATASECs are never compared with each other */
1550 case BTF_KIND_DATASEC:
1551 default:
1552 pr_warn("global '%s': unsupported BTF kind %s\n",
1553 sym_name, btf_kind_str(t1));
1554 return false;
1555 }
1556}
1557
1558static bool map_defs_match(const char *sym_name,
1559 const struct btf *main_btf,
1560 const struct btf_map_def *main_def,
1561 const struct btf_map_def *main_inner_def,
1562 const struct btf *extra_btf,
1563 const struct btf_map_def *extra_def,
1564 const struct btf_map_def *extra_inner_def)
1565{
1566 const char *reason;
1567
1568 if (main_def->map_type != extra_def->map_type) {
1569 reason = "type";
1570 goto mismatch;
1571 }
1572
1573 /* check key type/size match */
1574 if (main_def->key_size != extra_def->key_size) {
1575 reason = "key_size";
1576 goto mismatch;
1577 }
1578 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1579 reason = "key type";
1580 goto mismatch;
1581 }
1582 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1583 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1584 main_btf, main_def->key_type_id,
1585 extra_btf, extra_def->key_type_id)) {
1586 reason = "key type";
1587 goto mismatch;
1588 }
1589
1590 /* validate value type/size match */
1591 if (main_def->value_size != extra_def->value_size) {
1592 reason = "value_size";
1593 goto mismatch;
1594 }
1595 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1596 reason = "value type";
1597 goto mismatch;
1598 }
1599 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1600 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1601 main_btf, main_def->value_type_id,
1602 extra_btf, extra_def->value_type_id)) {
1603 reason = "key type";
1604 goto mismatch;
1605 }
1606
1607 if (main_def->max_entries != extra_def->max_entries) {
1608 reason = "max_entries";
1609 goto mismatch;
1610 }
1611 if (main_def->map_flags != extra_def->map_flags) {
1612 reason = "map_flags";
1613 goto mismatch;
1614 }
1615 if (main_def->numa_node != extra_def->numa_node) {
1616 reason = "numa_node";
1617 goto mismatch;
1618 }
1619 if (main_def->pinning != extra_def->pinning) {
1620 reason = "pinning";
1621 goto mismatch;
1622 }
1623
1624 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1625 reason = "inner map";
1626 goto mismatch;
1627 }
1628
1629 if (main_def->parts & MAP_DEF_INNER_MAP) {
1630 char inner_map_name[128];
1631
1632 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1633
1634 return map_defs_match(inner_map_name,
1635 main_btf, main_inner_def, NULL,
1636 extra_btf, extra_inner_def, NULL);
1637 }
1638
1639 return true;
1640
1641mismatch:
1642 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1643 return false;
1644}
1645
1646static bool glob_map_defs_match(const char *sym_name,
1647 struct bpf_linker *linker, struct glob_sym *glob_sym,
1648 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1649{
1650 struct btf_map_def dst_def = {}, dst_inner_def = {};
1651 struct btf_map_def src_def = {}, src_inner_def = {};
1652 const struct btf_type *t;
1653 int err;
1654
1655 t = btf__type_by_id(obj->btf, btf_id);
1656 if (!btf_is_var(t)) {
1657 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1658 return false;
1659 }
1660 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1661
1662 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1663 if (err) {
1664 pr_warn("global '%s': invalid map definition\n", sym_name);
1665 return false;
1666 }
1667
1668 /* re-parse existing map definition */
1669 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1670 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1671 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1672 if (err) {
1673 /* this should not happen, because we already validated it */
1674 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1675 return false;
1676 }
1677
1678 /* Currently extern map definition has to be complete and match
1679 * concrete map definition exactly. This restriction might be lifted
1680 * in the future.
1681 */
1682 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1683 obj->btf, &src_def, &src_inner_def);
1684}
1685
1686static bool glob_syms_match(const char *sym_name,
1687 struct bpf_linker *linker, struct glob_sym *glob_sym,
1688 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1689{
1690 const struct btf_type *src_t;
1691
1692 /* if we are dealing with externs, BTF types describing both global
1693 * and extern VARs/FUNCs should be completely present in all files
1694 */
1695 if (!glob_sym->btf_id || !btf_id) {
1696 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1697 return false;
1698 }
1699
1700 src_t = btf__type_by_id(obj->btf, btf_id);
1701 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1702 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1703 btf_kind_str(src_t), sym_name);
1704 return false;
1705 }
1706
1707 /* deal with .maps definitions specially */
1708 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1709 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1710
1711 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1712 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1713 return false;
1714
1715 return true;
1716}
1717
1718static bool btf_is_non_static(const struct btf_type *t)
1719{
1720 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1721 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1722}
1723
1724static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1725 int *out_btf_sec_id, int *out_btf_id)
1726{
1727 int i, j, n, m, btf_id = 0;
1728 const struct btf_type *t;
1729 const struct btf_var_secinfo *vi;
1730 const char *name;
1731
1732 if (!obj->btf) {
1733 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1734 return -EINVAL;
1735 }
1736
1737 n = btf__type_cnt(obj->btf);
1738 for (i = 1; i < n; i++) {
1739 t = btf__type_by_id(obj->btf, i);
1740
1741 /* some global and extern FUNCs and VARs might not be associated with any
1742 * DATASEC, so try to detect them in the same pass
1743 */
1744 if (btf_is_non_static(t)) {
1745 name = btf__str_by_offset(obj->btf, t->name_off);
1746 if (strcmp(name, sym_name) != 0)
1747 continue;
1748
1749 /* remember and still try to find DATASEC */
1750 btf_id = i;
1751 continue;
1752 }
1753
1754 if (!btf_is_datasec(t))
1755 continue;
1756
1757 vi = btf_var_secinfos(t);
1758 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1759 t = btf__type_by_id(obj->btf, vi->type);
1760 name = btf__str_by_offset(obj->btf, t->name_off);
1761
1762 if (strcmp(name, sym_name) != 0)
1763 continue;
1764 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1765 continue;
1766 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1767 continue;
1768
1769 if (btf_id && btf_id != vi->type) {
1770 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1771 sym_name, btf_id, vi->type);
1772 return -EINVAL;
1773 }
1774
1775 *out_btf_sec_id = i;
1776 *out_btf_id = vi->type;
1777
1778 return 0;
1779 }
1780 }
1781
1782 /* free-floating extern or global FUNC */
1783 if (btf_id) {
1784 *out_btf_sec_id = 0;
1785 *out_btf_id = btf_id;
1786 return 0;
1787 }
1788
1789 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1790 return -ENOENT;
1791}
1792
1793static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1794{
1795 struct src_sec *sec;
1796 int i;
1797
1798 for (i = 1; i < obj->sec_cnt; i++) {
1799 sec = &obj->secs[i];
1800
1801 if (strcmp(sec->sec_name, sec_name) == 0)
1802 return sec;
1803 }
1804
1805 return NULL;
1806}
1807
1808static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1809 struct btf *src_btf, int src_id)
1810{
1811 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1812 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1813 struct btf_param *src_p, *dst_p;
1814 const char *s;
1815 int i, n, off;
1816
1817 /* We already made sure that source and destination types (FUNC or
1818 * VAR) match in terms of types and argument names.
1819 */
1820 if (btf_is_var(dst_t)) {
1821 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1822 return 0;
1823 }
1824
1825 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1826
1827 /* now onto FUNC_PROTO types */
1828 src_t = btf_type_by_id(src_btf, src_t->type);
1829 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1830
1831 /* Fill in all the argument names, which for extern FUNCs are missing.
1832 * We'll end up with two copies of FUNCs/VARs for externs, but that
1833 * will be taken care of by BTF dedup at the very end.
1834 * It might be that BTF types for extern in one file has less/more BTF
1835 * information (e.g., FWD instead of full STRUCT/UNION information),
1836 * but that should be (in most cases, subject to BTF dedup rules)
1837 * handled and resolved by BTF dedup algorithm as well, so we won't
1838 * worry about it. Our only job is to make sure that argument names
1839 * are populated on both sides, otherwise BTF dedup will pedantically
1840 * consider them different.
1841 */
1842 src_p = btf_params(src_t);
1843 dst_p = btf_params(dst_t);
1844 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1845 if (!src_p->name_off)
1846 continue;
1847
1848 /* src_btf has more complete info, so add name to dst_btf */
1849 s = btf__str_by_offset(src_btf, src_p->name_off);
1850 off = btf__add_str(dst_btf, s);
1851 if (off < 0)
1852 return off;
1853 dst_p->name_off = off;
1854 }
1855 return 0;
1856}
1857
1858static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1859{
1860 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1861}
1862
1863static void sym_update_type(Elf64_Sym *sym, int sym_type)
1864{
1865 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1866}
1867
1868static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1869{
1870 /* libelf doesn't provide setters for ST_VISIBILITY,
1871 * but it is stored in the lower 2 bits of st_other
1872 */
1873 sym->st_other &= ~0x03;
1874 sym->st_other |= sym_vis;
1875}
1876
1877static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1878 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1879{
1880 struct src_sec *src_sec = NULL;
1881 struct dst_sec *dst_sec = NULL;
1882 struct glob_sym *glob_sym = NULL;
1883 int name_off, sym_type, sym_bind, sym_vis, err;
1884 int btf_sec_id = 0, btf_id = 0;
1885 size_t dst_sym_idx;
1886 Elf64_Sym *dst_sym;
1887 bool sym_is_extern;
1888
1889 sym_type = ELF64_ST_TYPE(sym->st_info);
1890 sym_bind = ELF64_ST_BIND(sym->st_info);
1891 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1892 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1893
1894 if (sym_is_extern) {
1895 if (!obj->btf) {
1896 pr_warn("externs without BTF info are not supported\n");
1897 return -ENOTSUP;
1898 }
1899 } else if (sym->st_shndx < SHN_LORESERVE) {
1900 src_sec = &obj->secs[sym->st_shndx];
1901 if (src_sec->skipped)
1902 return 0;
1903 dst_sec = &linker->secs[src_sec->dst_id];
1904
1905 /* allow only one STT_SECTION symbol per section */
1906 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1907 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1908 return 0;
1909 }
1910 }
1911
1912 if (sym_bind == STB_LOCAL)
1913 goto add_sym;
1914
1915 /* find matching BTF info */
1916 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1917 if (err)
1918 return err;
1919
1920 if (sym_is_extern && btf_sec_id) {
1921 const char *sec_name = NULL;
1922 const struct btf_type *t;
1923
1924 t = btf__type_by_id(obj->btf, btf_sec_id);
1925 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1926
1927 /* Clang puts unannotated extern vars into
1928 * '.extern' BTF DATASEC. Treat them the same
1929 * as unannotated extern funcs (which are
1930 * currently not put into any DATASECs).
1931 * Those don't have associated src_sec/dst_sec.
1932 */
1933 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1934 src_sec = find_src_sec_by_name(obj, sec_name);
1935 if (!src_sec) {
1936 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1937 return -ENOENT;
1938 }
1939 dst_sec = &linker->secs[src_sec->dst_id];
1940 }
1941 }
1942
1943 glob_sym = find_glob_sym(linker, sym_name);
1944 if (glob_sym) {
1945 /* Preventively resolve to existing symbol. This is
1946 * needed for further relocation symbol remapping in
1947 * the next step of linking.
1948 */
1949 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1950
1951 /* If both symbols are non-externs, at least one of
1952 * them has to be STB_WEAK, otherwise they are in
1953 * a conflict with each other.
1954 */
1955 if (!sym_is_extern && !glob_sym->is_extern
1956 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1957 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1958 src_sym_idx, sym_name, obj->filename);
1959 return -EINVAL;
1960 }
1961
1962 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1963 return -EINVAL;
1964
1965 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1966
1967 /* If new symbol is strong, then force dst_sym to be strong as
1968 * well; this way a mix of weak and non-weak extern
1969 * definitions will end up being strong.
1970 */
1971 if (sym_bind == STB_GLOBAL) {
1972 /* We still need to preserve type (NOTYPE or
1973 * OBJECT/FUNC, depending on whether the symbol is
1974 * extern or not)
1975 */
1976 sym_update_bind(dst_sym, STB_GLOBAL);
1977 glob_sym->is_weak = false;
1978 }
1979
1980 /* Non-default visibility is "contaminating", with stricter
1981 * visibility overwriting more permissive ones, even if more
1982 * permissive visibility comes from just an extern definition.
1983 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1984 * ensured by ELF symbol sanity checks above.
1985 */
1986 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1987 sym_update_visibility(dst_sym, sym_vis);
1988
1989 /* If the new symbol is extern, then regardless if
1990 * existing symbol is extern or resolved global, just
1991 * keep the existing one untouched.
1992 */
1993 if (sym_is_extern)
1994 return 0;
1995
1996 /* If existing symbol is a strong resolved symbol, bail out,
1997 * because we lost resolution battle have nothing to
1998 * contribute. We already checked above that there is no
1999 * strong-strong conflict. We also already tightened binding
2000 * and visibility, so nothing else to contribute at that point.
2001 */
2002 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2003 return 0;
2004
2005 /* At this point, new symbol is strong non-extern,
2006 * so overwrite glob_sym with new symbol information.
2007 * Preserve binding and visibility.
2008 */
2009 sym_update_type(dst_sym, sym_type);
2010 dst_sym->st_shndx = dst_sec->sec_idx;
2011 dst_sym->st_value = src_sec->dst_off + sym->st_value;
2012 dst_sym->st_size = sym->st_size;
2013
2014 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
2015 glob_sym->sec_id = dst_sec->id;
2016 glob_sym->is_extern = false;
2017
2018 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2019 obj->btf, btf_id))
2020 return -EINVAL;
2021
2022 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2023 glob_sym->underlying_btf_id = 0;
2024
2025 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2026 return 0;
2027 }
2028
2029add_sym:
2030 name_off = strset__add_str(linker->strtab_strs, sym_name);
2031 if (name_off < 0)
2032 return name_off;
2033
2034 dst_sym = add_new_sym(linker, &dst_sym_idx);
2035 if (!dst_sym)
2036 return -ENOMEM;
2037
2038 dst_sym->st_name = name_off;
2039 dst_sym->st_info = sym->st_info;
2040 dst_sym->st_other = sym->st_other;
2041 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2042 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2043 dst_sym->st_size = sym->st_size;
2044
2045 obj->sym_map[src_sym_idx] = dst_sym_idx;
2046
2047 if (sym_type == STT_SECTION && dst_sym) {
2048 dst_sec->sec_sym_idx = dst_sym_idx;
2049 dst_sym->st_value = 0;
2050 }
2051
2052 if (sym_bind != STB_LOCAL) {
2053 glob_sym = add_glob_sym(linker);
2054 if (!glob_sym)
2055 return -ENOMEM;
2056
2057 glob_sym->sym_idx = dst_sym_idx;
2058 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2059 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2060 * sec_idx (as they don't have corresponding ELF section), but
2061 * still have id. .extern doesn't have even ephemeral section
2062 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2063 */
2064 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2065 glob_sym->name_off = name_off;
2066 /* we will fill btf_id in during BTF merging step */
2067 glob_sym->btf_id = 0;
2068 glob_sym->is_extern = sym_is_extern;
2069 glob_sym->is_weak = sym_bind == STB_WEAK;
2070 }
2071
2072 return 0;
2073}
2074
2075static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2076{
2077 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2078 int i, err;
2079
2080 for (i = 1; i < obj->sec_cnt; i++) {
2081 struct src_sec *src_sec, *src_linked_sec;
2082 struct dst_sec *dst_sec, *dst_linked_sec;
2083 Elf64_Rel *src_rel, *dst_rel;
2084 int j, n;
2085
2086 src_sec = &obj->secs[i];
2087 if (!is_relo_sec(src_sec))
2088 continue;
2089
2090 /* shdr->sh_info points to relocatable section */
2091 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2092 if (src_linked_sec->skipped)
2093 continue;
2094
2095 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2096 if (!dst_sec) {
2097 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2098 if (!dst_sec)
2099 return -ENOMEM;
2100 err = init_sec(linker, dst_sec, src_sec);
2101 if (err) {
2102 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2103 return err;
2104 }
2105 } else if (!secs_match(dst_sec, src_sec)) {
2106 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2107 return -1;
2108 }
2109
2110 /* shdr->sh_link points to SYMTAB */
2111 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2112
2113 /* shdr->sh_info points to relocated section */
2114 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2115 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2116
2117 src_sec->dst_id = dst_sec->id;
2118 err = extend_sec(linker, dst_sec, src_sec);
2119 if (err)
2120 return err;
2121
2122 src_rel = src_sec->data->d_buf;
2123 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2124 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2125 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2126 size_t src_sym_idx, dst_sym_idx, sym_type;
2127 Elf64_Sym *src_sym;
2128
2129 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2130 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2131
2132 dst_sym_idx = obj->sym_map[src_sym_idx];
2133 dst_rel->r_offset += src_linked_sec->dst_off;
2134 sym_type = ELF64_R_TYPE(src_rel->r_info);
2135 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2136
2137 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2138 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2139 struct bpf_insn *insn;
2140
2141 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2142 /* calls to the very first static function inside
2143 * .text section at offset 0 will
2144 * reference section symbol, not the
2145 * function symbol. Fix that up,
2146 * otherwise it won't be possible to
2147 * relocate calls to two different
2148 * static functions with the same name
2149 * (rom two different object files)
2150 */
2151 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2152 if (insn->code == (BPF_JMP | BPF_CALL))
2153 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2154 else
2155 insn->imm += sec->dst_off;
2156 } else {
2157 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2158 return -EINVAL;
2159 }
2160 }
2161
2162 }
2163 }
2164
2165 return 0;
2166}
2167
2168static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2169 int sym_type, const char *sym_name)
2170{
2171 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2172 Elf64_Sym *sym = symtab->data->d_buf;
2173 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2174 int str_sec_idx = symtab->shdr->sh_link;
2175 const char *name;
2176
2177 for (i = 0; i < n; i++, sym++) {
2178 if (sym->st_shndx != sec_idx)
2179 continue;
2180 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2181 continue;
2182
2183 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2184 if (!name)
2185 return NULL;
2186
2187 if (strcmp(sym_name, name) != 0)
2188 continue;
2189
2190 return sym;
2191 }
2192
2193 return NULL;
2194}
2195
2196static int linker_fixup_btf(struct src_obj *obj)
2197{
2198 const char *sec_name;
2199 struct src_sec *sec;
2200 int i, j, n, m;
2201
2202 if (!obj->btf)
2203 return 0;
2204
2205 n = btf__type_cnt(obj->btf);
2206 for (i = 1; i < n; i++) {
2207 struct btf_var_secinfo *vi;
2208 struct btf_type *t;
2209
2210 t = btf_type_by_id(obj->btf, i);
2211 if (btf_kind(t) != BTF_KIND_DATASEC)
2212 continue;
2213
2214 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2215 sec = find_src_sec_by_name(obj, sec_name);
2216 if (sec) {
2217 /* record actual section size, unless ephemeral */
2218 if (sec->shdr)
2219 t->size = sec->shdr->sh_size;
2220 } else {
2221 /* BTF can have some sections that are not represented
2222 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2223 * for special extern variables.
2224 *
2225 * For all but one such special (ephemeral)
2226 * sections, we pre-create "section shells" to be able
2227 * to keep track of extra per-section metadata later
2228 * (e.g., those BTF extern variables).
2229 *
2230 * .extern is even more special, though, because it
2231 * contains extern variables that need to be resolved
2232 * by static linker, not libbpf and kernel. When such
2233 * externs are resolved, we are going to remove them
2234 * from .extern BTF section and might end up not
2235 * needing it at all. Each resolved extern should have
2236 * matching non-extern VAR/FUNC in other sections.
2237 *
2238 * We do support leaving some of the externs
2239 * unresolved, though, to support cases of building
2240 * libraries, which will later be linked against final
2241 * BPF applications. So if at finalization we still
2242 * see unresolved externs, we'll create .extern
2243 * section on our own.
2244 */
2245 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2246 continue;
2247
2248 sec = add_src_sec(obj, sec_name);
2249 if (!sec)
2250 return -ENOMEM;
2251
2252 sec->ephemeral = true;
2253 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2254 }
2255
2256 /* remember ELF section and its BTF type ID match */
2257 sec->sec_type_id = i;
2258
2259 /* fix up variable offsets */
2260 vi = btf_var_secinfos(t);
2261 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2262 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2263 const char *var_name;
2264 int var_linkage;
2265 Elf64_Sym *sym;
2266
2267 /* could be a variable or function */
2268 if (!btf_is_var(vt))
2269 continue;
2270
2271 var_name = btf__str_by_offset(obj->btf, vt->name_off);
2272 var_linkage = btf_var(vt)->linkage;
2273
2274 /* no need to patch up static or extern vars */
2275 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2276 continue;
2277
2278 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2279 if (!sym) {
2280 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2281 return -ENOENT;
2282 }
2283
2284 vi->offset = sym->st_value;
2285 }
2286 }
2287
2288 return 0;
2289}
2290
2291static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2292{
2293 const struct btf_type *t;
2294 int i, j, n, start_id, id, err;
2295 const char *name;
2296
2297 if (!obj->btf)
2298 return 0;
2299
2300 start_id = btf__type_cnt(linker->btf);
2301 n = btf__type_cnt(obj->btf);
2302
2303 obj->btf_type_map = calloc(n + 1, sizeof(int));
2304 if (!obj->btf_type_map)
2305 return -ENOMEM;
2306
2307 for (i = 1; i < n; i++) {
2308 struct glob_sym *glob_sym = NULL;
2309
2310 t = btf__type_by_id(obj->btf, i);
2311
2312 /* DATASECs are handled specially below */
2313 if (btf_kind(t) == BTF_KIND_DATASEC)
2314 continue;
2315
2316 if (btf_is_non_static(t)) {
2317 /* there should be glob_sym already */
2318 name = btf__str_by_offset(obj->btf, t->name_off);
2319 glob_sym = find_glob_sym(linker, name);
2320
2321 /* VARs without corresponding glob_sym are those that
2322 * belong to skipped/deduplicated sections (i.e.,
2323 * license and version), so just skip them
2324 */
2325 if (!glob_sym)
2326 continue;
2327
2328 /* linker_append_elf_sym() might have requested
2329 * updating underlying type ID, if extern was resolved
2330 * to strong symbol or weak got upgraded to non-weak
2331 */
2332 if (glob_sym->underlying_btf_id == 0)
2333 glob_sym->underlying_btf_id = -t->type;
2334
2335 /* globals from previous object files that match our
2336 * VAR/FUNC already have a corresponding associated
2337 * BTF type, so just make sure to use it
2338 */
2339 if (glob_sym->btf_id) {
2340 /* reuse existing BTF type for global var/func */
2341 obj->btf_type_map[i] = glob_sym->btf_id;
2342 continue;
2343 }
2344 }
2345
2346 id = btf__add_type(linker->btf, obj->btf, t);
2347 if (id < 0) {
2348 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2349 return id;
2350 }
2351
2352 obj->btf_type_map[i] = id;
2353
2354 /* record just appended BTF type for var/func */
2355 if (glob_sym) {
2356 glob_sym->btf_id = id;
2357 glob_sym->underlying_btf_id = -t->type;
2358 }
2359 }
2360
2361 /* remap all the types except DATASECs */
2362 n = btf__type_cnt(linker->btf);
2363 for (i = start_id; i < n; i++) {
2364 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2365 struct btf_field_iter it;
2366 __u32 *type_id;
2367
2368 err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2369 if (err)
2370 return err;
2371
2372 while ((type_id = btf_field_iter_next(&it))) {
2373 int new_id = obj->btf_type_map[*type_id];
2374
2375 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2376 if (new_id == 0 && *type_id != 0) {
2377 pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2378 *type_id);
2379 return -EINVAL;
2380 }
2381
2382 *type_id = obj->btf_type_map[*type_id];
2383 }
2384 }
2385
2386 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2387 * actual type), if necessary
2388 */
2389 for (i = 0; i < linker->glob_sym_cnt; i++) {
2390 struct glob_sym *glob_sym = &linker->glob_syms[i];
2391 struct btf_type *glob_t;
2392
2393 if (glob_sym->underlying_btf_id >= 0)
2394 continue;
2395
2396 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2397
2398 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2399 glob_t->type = glob_sym->underlying_btf_id;
2400 }
2401
2402 /* append DATASEC info */
2403 for (i = 1; i < obj->sec_cnt; i++) {
2404 struct src_sec *src_sec;
2405 struct dst_sec *dst_sec;
2406 const struct btf_var_secinfo *src_var;
2407 struct btf_var_secinfo *dst_var;
2408
2409 src_sec = &obj->secs[i];
2410 if (!src_sec->sec_type_id || src_sec->skipped)
2411 continue;
2412 dst_sec = &linker->secs[src_sec->dst_id];
2413
2414 /* Mark section as having BTF regardless of the presence of
2415 * variables. In some cases compiler might generate empty BTF
2416 * with no variables information. E.g., when promoting local
2417 * array/structure variable initial values and BPF object
2418 * file otherwise has no read-only static variables in
2419 * .rodata. We need to preserve such empty BTF and just set
2420 * correct section size.
2421 */
2422 dst_sec->has_btf = true;
2423
2424 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2425 src_var = btf_var_secinfos(t);
2426 n = btf_vlen(t);
2427 for (j = 0; j < n; j++, src_var++) {
2428 void *sec_vars = dst_sec->sec_vars;
2429 int new_id = obj->btf_type_map[src_var->type];
2430 struct glob_sym *glob_sym = NULL;
2431
2432 t = btf_type_by_id(linker->btf, new_id);
2433 if (btf_is_non_static(t)) {
2434 name = btf__str_by_offset(linker->btf, t->name_off);
2435 glob_sym = find_glob_sym(linker, name);
2436 if (glob_sym->sec_id != dst_sec->id) {
2437 pr_warn("global '%s': section mismatch %d vs %d\n",
2438 name, glob_sym->sec_id, dst_sec->id);
2439 return -EINVAL;
2440 }
2441 }
2442
2443 /* If there is already a member (VAR or FUNC) mapped
2444 * to the same type, don't add a duplicate entry.
2445 * This will happen when multiple object files define
2446 * the same extern VARs/FUNCs.
2447 */
2448 if (glob_sym && glob_sym->var_idx >= 0) {
2449 __s64 sz;
2450
2451 /* FUNCs don't have size, nothing to update */
2452 if (btf_is_func(t))
2453 continue;
2454
2455 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2456 /* Because underlying BTF type might have
2457 * changed, so might its size have changed, so
2458 * re-calculate and update it in sec_var.
2459 */
2460 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2461 if (sz < 0) {
2462 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2463 name, (int)sz);
2464 return -EINVAL;
2465 }
2466 dst_var->size = sz;
2467 continue;
2468 }
2469
2470 sec_vars = libbpf_reallocarray(sec_vars,
2471 dst_sec->sec_var_cnt + 1,
2472 sizeof(*dst_sec->sec_vars));
2473 if (!sec_vars)
2474 return -ENOMEM;
2475
2476 dst_sec->sec_vars = sec_vars;
2477 dst_sec->sec_var_cnt++;
2478
2479 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2480 dst_var->type = obj->btf_type_map[src_var->type];
2481 dst_var->size = src_var->size;
2482 dst_var->offset = src_sec->dst_off + src_var->offset;
2483
2484 if (glob_sym)
2485 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2486 }
2487 }
2488
2489 return 0;
2490}
2491
2492static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2493{
2494 void *tmp;
2495
2496 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2497 if (!tmp)
2498 return NULL;
2499 ext_data->recs = tmp;
2500
2501 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2502 memcpy(tmp, src_rec, ext_data->rec_sz);
2503
2504 ext_data->rec_cnt++;
2505
2506 return tmp;
2507}
2508
2509static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2510{
2511 const struct btf_ext_info_sec *ext_sec;
2512 const char *sec_name, *s;
2513 struct src_sec *src_sec;
2514 struct dst_sec *dst_sec;
2515 int rec_sz, str_off, i;
2516
2517 if (!obj->btf_ext)
2518 return 0;
2519
2520 rec_sz = obj->btf_ext->func_info.rec_size;
2521 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2522 struct bpf_func_info_min *src_rec, *dst_rec;
2523
2524 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2525 src_sec = find_src_sec_by_name(obj, sec_name);
2526 if (!src_sec) {
2527 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2528 return -EINVAL;
2529 }
2530 dst_sec = &linker->secs[src_sec->dst_id];
2531
2532 if (dst_sec->func_info.rec_sz == 0)
2533 dst_sec->func_info.rec_sz = rec_sz;
2534 if (dst_sec->func_info.rec_sz != rec_sz) {
2535 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2536 return -EINVAL;
2537 }
2538
2539 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2540 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2541 if (!dst_rec)
2542 return -ENOMEM;
2543
2544 dst_rec->insn_off += src_sec->dst_off;
2545 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2546 }
2547 }
2548
2549 rec_sz = obj->btf_ext->line_info.rec_size;
2550 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2551 struct bpf_line_info_min *src_rec, *dst_rec;
2552
2553 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2554 src_sec = find_src_sec_by_name(obj, sec_name);
2555 if (!src_sec) {
2556 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2557 return -EINVAL;
2558 }
2559 dst_sec = &linker->secs[src_sec->dst_id];
2560
2561 if (dst_sec->line_info.rec_sz == 0)
2562 dst_sec->line_info.rec_sz = rec_sz;
2563 if (dst_sec->line_info.rec_sz != rec_sz) {
2564 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2565 return -EINVAL;
2566 }
2567
2568 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2569 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2570 if (!dst_rec)
2571 return -ENOMEM;
2572
2573 dst_rec->insn_off += src_sec->dst_off;
2574
2575 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2576 str_off = btf__add_str(linker->btf, s);
2577 if (str_off < 0)
2578 return -ENOMEM;
2579 dst_rec->file_name_off = str_off;
2580
2581 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2582 str_off = btf__add_str(linker->btf, s);
2583 if (str_off < 0)
2584 return -ENOMEM;
2585 dst_rec->line_off = str_off;
2586
2587 /* dst_rec->line_col is fine */
2588 }
2589 }
2590
2591 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2592 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2593 struct bpf_core_relo *src_rec, *dst_rec;
2594
2595 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2596 src_sec = find_src_sec_by_name(obj, sec_name);
2597 if (!src_sec) {
2598 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2599 return -EINVAL;
2600 }
2601 dst_sec = &linker->secs[src_sec->dst_id];
2602
2603 if (dst_sec->core_relo_info.rec_sz == 0)
2604 dst_sec->core_relo_info.rec_sz = rec_sz;
2605 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2606 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2607 return -EINVAL;
2608 }
2609
2610 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2611 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2612 if (!dst_rec)
2613 return -ENOMEM;
2614
2615 dst_rec->insn_off += src_sec->dst_off;
2616 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2617
2618 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2619 str_off = btf__add_str(linker->btf, s);
2620 if (str_off < 0)
2621 return -ENOMEM;
2622 dst_rec->access_str_off = str_off;
2623
2624 /* dst_rec->kind is fine */
2625 }
2626 }
2627
2628 return 0;
2629}
2630
2631int bpf_linker__finalize(struct bpf_linker *linker)
2632{
2633 struct dst_sec *sec;
2634 size_t strs_sz;
2635 const void *strs;
2636 int err, i;
2637
2638 if (!linker->elf)
2639 return libbpf_err(-EINVAL);
2640
2641 err = finalize_btf(linker);
2642 if (err)
2643 return libbpf_err(err);
2644
2645 /* Finalize strings */
2646 strs_sz = strset__data_size(linker->strtab_strs);
2647 strs = strset__data(linker->strtab_strs);
2648
2649 sec = &linker->secs[linker->strtab_sec_idx];
2650 sec->data->d_align = 1;
2651 sec->data->d_off = 0LL;
2652 sec->data->d_buf = (void *)strs;
2653 sec->data->d_type = ELF_T_BYTE;
2654 sec->data->d_size = strs_sz;
2655 sec->shdr->sh_size = strs_sz;
2656
2657 for (i = 1; i < linker->sec_cnt; i++) {
2658 sec = &linker->secs[i];
2659
2660 /* STRTAB is handled specially above */
2661 if (sec->sec_idx == linker->strtab_sec_idx)
2662 continue;
2663
2664 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2665 if (!sec->scn)
2666 continue;
2667
2668 /* restore sections with bpf insns to target byte-order */
2669 if (linker->swapped_endian && is_exec_sec(sec))
2670 exec_sec_bswap(sec->raw_data, sec->sec_sz);
2671
2672 sec->data->d_buf = sec->raw_data;
2673 }
2674
2675 /* Finalize ELF layout */
2676 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2677 err = -EINVAL;
2678 pr_warn_elf("failed to finalize ELF layout");
2679 return libbpf_err(err);
2680 }
2681
2682 /* Write out final ELF contents */
2683 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2684 err = -EINVAL;
2685 pr_warn_elf("failed to write ELF contents");
2686 return libbpf_err(err);
2687 }
2688
2689 elf_end(linker->elf);
2690 close(linker->fd);
2691
2692 linker->elf = NULL;
2693 linker->fd = -1;
2694
2695 return 0;
2696}
2697
2698static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2699 size_t align, const void *raw_data, size_t raw_sz)
2700{
2701 Elf_Scn *scn;
2702 Elf_Data *data;
2703 Elf64_Shdr *shdr;
2704 int name_off;
2705
2706 name_off = strset__add_str(linker->strtab_strs, sec_name);
2707 if (name_off < 0)
2708 return name_off;
2709
2710 scn = elf_newscn(linker->elf);
2711 if (!scn)
2712 return -ENOMEM;
2713 data = elf_newdata(scn);
2714 if (!data)
2715 return -ENOMEM;
2716 shdr = elf64_getshdr(scn);
2717 if (!shdr)
2718 return -EINVAL;
2719
2720 shdr->sh_name = name_off;
2721 shdr->sh_type = SHT_PROGBITS;
2722 shdr->sh_flags = 0;
2723 shdr->sh_size = raw_sz;
2724 shdr->sh_link = 0;
2725 shdr->sh_info = 0;
2726 shdr->sh_addralign = align;
2727 shdr->sh_entsize = 0;
2728
2729 data->d_type = ELF_T_BYTE;
2730 data->d_size = raw_sz;
2731 data->d_buf = (void *)raw_data;
2732 data->d_align = align;
2733 data->d_off = 0;
2734
2735 return 0;
2736}
2737
2738static int finalize_btf(struct bpf_linker *linker)
2739{
2740 enum btf_endianness link_endianness;
2741 LIBBPF_OPTS(btf_dedup_opts, opts);
2742 struct btf *btf = linker->btf;
2743 const void *raw_data;
2744 int i, j, id, err;
2745 __u32 raw_sz;
2746
2747 /* bail out if no BTF data was produced */
2748 if (btf__type_cnt(linker->btf) == 1)
2749 return 0;
2750
2751 for (i = 1; i < linker->sec_cnt; i++) {
2752 struct dst_sec *sec = &linker->secs[i];
2753
2754 if (!sec->has_btf)
2755 continue;
2756
2757 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2758 if (id < 0) {
2759 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2760 sec->sec_name, id);
2761 return id;
2762 }
2763
2764 for (j = 0; j < sec->sec_var_cnt; j++) {
2765 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2766
2767 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2768 return -EINVAL;
2769 }
2770 }
2771
2772 err = finalize_btf_ext(linker);
2773 if (err) {
2774 pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2775 return err;
2776 }
2777
2778 opts.btf_ext = linker->btf_ext;
2779 err = btf__dedup(linker->btf, &opts);
2780 if (err) {
2781 pr_warn("BTF dedup failed: %s\n", errstr(err));
2782 return err;
2783 }
2784
2785 /* Set .BTF and .BTF.ext output byte order */
2786 link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2787 BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2788 btf__set_endianness(linker->btf, link_endianness);
2789 if (linker->btf_ext)
2790 btf_ext__set_endianness(linker->btf_ext, link_endianness);
2791
2792 /* Emit .BTF section */
2793 raw_data = btf__raw_data(linker->btf, &raw_sz);
2794 if (!raw_data)
2795 return -ENOMEM;
2796
2797 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2798 if (err) {
2799 pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2800 return err;
2801 }
2802
2803 /* Emit .BTF.ext section */
2804 if (linker->btf_ext) {
2805 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2806 if (!raw_data)
2807 return -ENOMEM;
2808
2809 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2810 if (err) {
2811 pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2812 return err;
2813 }
2814 }
2815
2816 return 0;
2817}
2818
2819static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2820 const char *sec_name, struct btf_ext_sec_data *sec_data)
2821{
2822 struct btf_ext_info_sec *sec_info;
2823 void *cur = output;
2824 int str_off;
2825 size_t sz;
2826
2827 if (!sec_data->rec_cnt)
2828 return 0;
2829
2830 str_off = btf__add_str(linker->btf, sec_name);
2831 if (str_off < 0)
2832 return -ENOMEM;
2833
2834 sec_info = cur;
2835 sec_info->sec_name_off = str_off;
2836 sec_info->num_info = sec_data->rec_cnt;
2837 cur += sizeof(struct btf_ext_info_sec);
2838
2839 sz = sec_data->rec_cnt * sec_data->rec_sz;
2840 memcpy(cur, sec_data->recs, sz);
2841 cur += sz;
2842
2843 return cur - output;
2844}
2845
2846static int finalize_btf_ext(struct bpf_linker *linker)
2847{
2848 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2849 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2850 struct btf_ext_header *hdr;
2851 void *data, *cur;
2852 int i, err, sz;
2853
2854 /* validate that all sections have the same .BTF.ext record sizes
2855 * and calculate total data size for each type of data (func info,
2856 * line info, core relos)
2857 */
2858 for (i = 1; i < linker->sec_cnt; i++) {
2859 struct dst_sec *sec = &linker->secs[i];
2860
2861 if (sec->func_info.rec_cnt) {
2862 if (func_rec_sz == 0)
2863 func_rec_sz = sec->func_info.rec_sz;
2864 if (func_rec_sz != sec->func_info.rec_sz) {
2865 pr_warn("mismatch in func_info record size %zu != %u\n",
2866 func_rec_sz, sec->func_info.rec_sz);
2867 return -EINVAL;
2868 }
2869
2870 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2871 }
2872 if (sec->line_info.rec_cnt) {
2873 if (line_rec_sz == 0)
2874 line_rec_sz = sec->line_info.rec_sz;
2875 if (line_rec_sz != sec->line_info.rec_sz) {
2876 pr_warn("mismatch in line_info record size %zu != %u\n",
2877 line_rec_sz, sec->line_info.rec_sz);
2878 return -EINVAL;
2879 }
2880
2881 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2882 }
2883 if (sec->core_relo_info.rec_cnt) {
2884 if (core_relo_rec_sz == 0)
2885 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2886 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2887 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2888 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2889 return -EINVAL;
2890 }
2891
2892 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2893 }
2894 }
2895
2896 if (!funcs_sz && !lines_sz && !core_relos_sz)
2897 return 0;
2898
2899 total_sz += sizeof(struct btf_ext_header);
2900 if (funcs_sz) {
2901 funcs_sz += sizeof(__u32); /* record size prefix */
2902 total_sz += funcs_sz;
2903 }
2904 if (lines_sz) {
2905 lines_sz += sizeof(__u32); /* record size prefix */
2906 total_sz += lines_sz;
2907 }
2908 if (core_relos_sz) {
2909 core_relos_sz += sizeof(__u32); /* record size prefix */
2910 total_sz += core_relos_sz;
2911 }
2912
2913 cur = data = calloc(1, total_sz);
2914 if (!data)
2915 return -ENOMEM;
2916
2917 hdr = cur;
2918 hdr->magic = BTF_MAGIC;
2919 hdr->version = BTF_VERSION;
2920 hdr->flags = 0;
2921 hdr->hdr_len = sizeof(struct btf_ext_header);
2922 cur += sizeof(struct btf_ext_header);
2923
2924 /* All offsets are in bytes relative to the end of this header */
2925 hdr->func_info_off = 0;
2926 hdr->func_info_len = funcs_sz;
2927 hdr->line_info_off = funcs_sz;
2928 hdr->line_info_len = lines_sz;
2929 hdr->core_relo_off = funcs_sz + lines_sz;
2930 hdr->core_relo_len = core_relos_sz;
2931
2932 if (funcs_sz) {
2933 *(__u32 *)cur = func_rec_sz;
2934 cur += sizeof(__u32);
2935
2936 for (i = 1; i < linker->sec_cnt; i++) {
2937 struct dst_sec *sec = &linker->secs[i];
2938
2939 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2940 if (sz < 0) {
2941 err = sz;
2942 goto out;
2943 }
2944
2945 cur += sz;
2946 }
2947 }
2948
2949 if (lines_sz) {
2950 *(__u32 *)cur = line_rec_sz;
2951 cur += sizeof(__u32);
2952
2953 for (i = 1; i < linker->sec_cnt; i++) {
2954 struct dst_sec *sec = &linker->secs[i];
2955
2956 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2957 if (sz < 0) {
2958 err = sz;
2959 goto out;
2960 }
2961
2962 cur += sz;
2963 }
2964 }
2965
2966 if (core_relos_sz) {
2967 *(__u32 *)cur = core_relo_rec_sz;
2968 cur += sizeof(__u32);
2969
2970 for (i = 1; i < linker->sec_cnt; i++) {
2971 struct dst_sec *sec = &linker->secs[i];
2972
2973 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2974 if (sz < 0) {
2975 err = sz;
2976 goto out;
2977 }
2978
2979 cur += sz;
2980 }
2981 }
2982
2983 linker->btf_ext = btf_ext__new(data, total_sz);
2984 err = libbpf_get_error(linker->btf_ext);
2985 if (err) {
2986 linker->btf_ext = NULL;
2987 pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
2988 goto out;
2989 }
2990
2991out:
2992 free(data);
2993 return err;
2994}
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <errno.h>
14#include <linux/err.h>
15#include <linux/btf.h>
16#include <elf.h>
17#include <libelf.h>
18#include <gelf.h>
19#include <fcntl.h>
20#include "libbpf.h"
21#include "btf.h"
22#include "libbpf_internal.h"
23#include "strset.h"
24
25#define BTF_EXTERN_SEC ".extern"
26
27struct src_sec {
28 const char *sec_name;
29 /* positional (not necessarily ELF) index in an array of sections */
30 int id;
31 /* positional (not necessarily ELF) index of a matching section in a final object file */
32 int dst_id;
33 /* section data offset in a matching output section */
34 int dst_off;
35 /* whether section is omitted from the final ELF file */
36 bool skipped;
37 /* whether section is an ephemeral section, not mapped to an ELF section */
38 bool ephemeral;
39
40 /* ELF info */
41 size_t sec_idx;
42 Elf_Scn *scn;
43 Elf64_Shdr *shdr;
44 Elf_Data *data;
45
46 /* corresponding BTF DATASEC type ID */
47 int sec_type_id;
48};
49
50struct src_obj {
51 const char *filename;
52 int fd;
53 Elf *elf;
54 /* Section header strings section index */
55 size_t shstrs_sec_idx;
56 /* SYMTAB section index */
57 size_t symtab_sec_idx;
58
59 struct btf *btf;
60 struct btf_ext *btf_ext;
61
62 /* List of sections (including ephemeral). Slot zero is unused. */
63 struct src_sec *secs;
64 int sec_cnt;
65
66 /* mapping of symbol indices from src to dst ELF */
67 int *sym_map;
68 /* mapping from the src BTF type IDs to dst ones */
69 int *btf_type_map;
70};
71
72/* single .BTF.ext data section */
73struct btf_ext_sec_data {
74 size_t rec_cnt;
75 __u32 rec_sz;
76 void *recs;
77};
78
79struct glob_sym {
80 /* ELF symbol index */
81 int sym_idx;
82 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
83 int sec_id;
84 /* extern name offset in STRTAB */
85 int name_off;
86 /* optional associated BTF type ID */
87 int btf_id;
88 /* BTF type ID to which VAR/FUNC type is pointing to; used for
89 * rewriting types when extern VAR/FUNC is resolved to a concrete
90 * definition
91 */
92 int underlying_btf_id;
93 /* sec_var index in the corresponding dst_sec, if exists */
94 int var_idx;
95
96 /* extern or resolved/global symbol */
97 bool is_extern;
98 /* weak or strong symbol, never goes back from strong to weak */
99 bool is_weak;
100};
101
102struct dst_sec {
103 char *sec_name;
104 /* positional (not necessarily ELF) index in an array of sections */
105 int id;
106
107 bool ephemeral;
108
109 /* ELF info */
110 size_t sec_idx;
111 Elf_Scn *scn;
112 Elf64_Shdr *shdr;
113 Elf_Data *data;
114
115 /* final output section size */
116 int sec_sz;
117 /* final output contents of the section */
118 void *raw_data;
119
120 /* corresponding STT_SECTION symbol index in SYMTAB */
121 int sec_sym_idx;
122
123 /* section's DATASEC variable info, emitted on BTF finalization */
124 bool has_btf;
125 int sec_var_cnt;
126 struct btf_var_secinfo *sec_vars;
127
128 /* section's .BTF.ext data */
129 struct btf_ext_sec_data func_info;
130 struct btf_ext_sec_data line_info;
131 struct btf_ext_sec_data core_relo_info;
132};
133
134struct bpf_linker {
135 char *filename;
136 int fd;
137 Elf *elf;
138 Elf64_Ehdr *elf_hdr;
139
140 /* Output sections metadata */
141 struct dst_sec *secs;
142 int sec_cnt;
143
144 struct strset *strtab_strs; /* STRTAB unique strings */
145 size_t strtab_sec_idx; /* STRTAB section index */
146 size_t symtab_sec_idx; /* SYMTAB section index */
147
148 struct btf *btf;
149 struct btf_ext *btf_ext;
150
151 /* global (including extern) ELF symbols */
152 int glob_sym_cnt;
153 struct glob_sym *glob_syms;
154};
155
156#define pr_warn_elf(fmt, ...) \
157 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
158
159static int init_output_elf(struct bpf_linker *linker, const char *file);
160
161static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
162 const struct bpf_linker_file_opts *opts,
163 struct src_obj *obj);
164static int linker_sanity_check_elf(struct src_obj *obj);
165static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
166static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
167static int linker_sanity_check_btf(struct src_obj *obj);
168static int linker_sanity_check_btf_ext(struct src_obj *obj);
169static int linker_fixup_btf(struct src_obj *obj);
170static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
171static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
172static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
173 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
174static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
175static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
176static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
177
178static int finalize_btf(struct bpf_linker *linker);
179static int finalize_btf_ext(struct bpf_linker *linker);
180
181void bpf_linker__free(struct bpf_linker *linker)
182{
183 int i;
184
185 if (!linker)
186 return;
187
188 free(linker->filename);
189
190 if (linker->elf)
191 elf_end(linker->elf);
192
193 if (linker->fd >= 0)
194 close(linker->fd);
195
196 strset__free(linker->strtab_strs);
197
198 btf__free(linker->btf);
199 btf_ext__free(linker->btf_ext);
200
201 for (i = 1; i < linker->sec_cnt; i++) {
202 struct dst_sec *sec = &linker->secs[i];
203
204 free(sec->sec_name);
205 free(sec->raw_data);
206 free(sec->sec_vars);
207
208 free(sec->func_info.recs);
209 free(sec->line_info.recs);
210 free(sec->core_relo_info.recs);
211 }
212 free(linker->secs);
213
214 free(linker);
215}
216
217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218{
219 struct bpf_linker *linker;
220 int err;
221
222 if (!OPTS_VALID(opts, bpf_linker_opts))
223 return errno = EINVAL, NULL;
224
225 if (elf_version(EV_CURRENT) == EV_NONE) {
226 pr_warn_elf("libelf initialization failed");
227 return errno = EINVAL, NULL;
228 }
229
230 linker = calloc(1, sizeof(*linker));
231 if (!linker)
232 return errno = ENOMEM, NULL;
233
234 linker->fd = -1;
235
236 err = init_output_elf(linker, filename);
237 if (err)
238 goto err_out;
239
240 return linker;
241
242err_out:
243 bpf_linker__free(linker);
244 return errno = -err, NULL;
245}
246
247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248{
249 struct dst_sec *secs = linker->secs, *sec;
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251
252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 if (!secs)
254 return NULL;
255
256 /* zero out newly allocated memory */
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258
259 linker->secs = secs;
260 linker->sec_cnt = new_cnt;
261
262 sec = &linker->secs[new_cnt - 1];
263 sec->id = new_cnt - 1;
264 sec->sec_name = strdup(sec_name);
265 if (!sec->sec_name)
266 return NULL;
267
268 return sec;
269}
270
271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272{
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 Elf64_Sym *syms, *sym;
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 if (!syms)
279 return NULL;
280
281 sym = &syms[sym_cnt];
282 memset(sym, 0, sizeof(*sym));
283
284 symtab->raw_data = syms;
285 symtab->sec_sz += sizeof(*sym);
286 symtab->shdr->sh_size += sizeof(*sym);
287 symtab->data->d_size += sizeof(*sym);
288
289 if (sym_idx)
290 *sym_idx = sym_cnt;
291
292 return sym;
293}
294
295static int init_output_elf(struct bpf_linker *linker, const char *file)
296{
297 int err, str_off;
298 Elf64_Sym *init_sym;
299 struct dst_sec *sec;
300
301 linker->filename = strdup(file);
302 if (!linker->filename)
303 return -ENOMEM;
304
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
306 if (linker->fd < 0) {
307 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);
309 return err;
310 }
311
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 if (!linker->elf) {
314 pr_warn_elf("failed to create ELF object");
315 return -EINVAL;
316 }
317
318 /* ELF header */
319 linker->elf_hdr = elf64_newehdr(linker->elf);
320 if (!linker->elf_hdr) {
321 pr_warn_elf("failed to create ELF header");
322 return -EINVAL;
323 }
324
325 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;
327#if __BYTE_ORDER == __LITTLE_ENDIAN
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329#elif __BYTE_ORDER == __BIG_ENDIAN
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331#else
332#error "Unknown __BYTE_ORDER"
333#endif
334
335 /* STRTAB */
336 /* initialize strset with an empty string to conform to ELF */
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 if (libbpf_get_error(linker->strtab_strs))
339 return libbpf_get_error(linker->strtab_strs);
340
341 sec = add_dst_sec(linker, ".strtab");
342 if (!sec)
343 return -ENOMEM;
344
345 sec->scn = elf_newscn(linker->elf);
346 if (!sec->scn) {
347 pr_warn_elf("failed to create STRTAB section");
348 return -EINVAL;
349 }
350
351 sec->shdr = elf64_getshdr(sec->scn);
352 if (!sec->shdr)
353 return -EINVAL;
354
355 sec->data = elf_newdata(sec->scn);
356 if (!sec->data) {
357 pr_warn_elf("failed to create STRTAB data");
358 return -EINVAL;
359 }
360
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 if (str_off < 0)
363 return str_off;
364
365 sec->sec_idx = elf_ndxscn(sec->scn);
366 linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 linker->strtab_sec_idx = sec->sec_idx;
368
369 sec->shdr->sh_name = str_off;
370 sec->shdr->sh_type = SHT_STRTAB;
371 sec->shdr->sh_flags = SHF_STRINGS;
372 sec->shdr->sh_offset = 0;
373 sec->shdr->sh_link = 0;
374 sec->shdr->sh_info = 0;
375 sec->shdr->sh_addralign = 1;
376 sec->shdr->sh_size = sec->sec_sz = 0;
377 sec->shdr->sh_entsize = 0;
378
379 /* SYMTAB */
380 sec = add_dst_sec(linker, ".symtab");
381 if (!sec)
382 return -ENOMEM;
383
384 sec->scn = elf_newscn(linker->elf);
385 if (!sec->scn) {
386 pr_warn_elf("failed to create SYMTAB section");
387 return -EINVAL;
388 }
389
390 sec->shdr = elf64_getshdr(sec->scn);
391 if (!sec->shdr)
392 return -EINVAL;
393
394 sec->data = elf_newdata(sec->scn);
395 if (!sec->data) {
396 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;
398 }
399
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)
402 return str_off;
403
404 sec->sec_idx = elf_ndxscn(sec->scn);
405 linker->symtab_sec_idx = sec->sec_idx;
406
407 sec->shdr->sh_name = str_off;
408 sec->shdr->sh_type = SHT_SYMTAB;
409 sec->shdr->sh_flags = 0;
410 sec->shdr->sh_offset = 0;
411 sec->shdr->sh_link = linker->strtab_sec_idx;
412 /* sh_info should be one greater than the index of the last local
413 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
414 */
415 sec->shdr->sh_info = 0;
416 sec->shdr->sh_addralign = 8;
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418
419 /* .BTF */
420 linker->btf = btf__new_empty();
421 err = libbpf_get_error(linker->btf);
422 if (err)
423 return err;
424
425 /* add the special all-zero symbol */
426 init_sym = add_new_sym(linker, NULL);
427 if (!init_sym)
428 return -EINVAL;
429
430 init_sym->st_name = 0;
431 init_sym->st_info = 0;
432 init_sym->st_other = 0;
433 init_sym->st_shndx = SHN_UNDEF;
434 init_sym->st_value = 0;
435 init_sym->st_size = 0;
436
437 return 0;
438}
439
440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 const struct bpf_linker_file_opts *opts)
442{
443 struct src_obj obj = {};
444 int err = 0;
445
446 if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 return libbpf_err(-EINVAL);
448
449 if (!linker->elf)
450 return libbpf_err(-EINVAL);
451
452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 err = err ?: linker_append_sec_data(linker, &obj);
454 err = err ?: linker_append_elf_syms(linker, &obj);
455 err = err ?: linker_append_elf_relos(linker, &obj);
456 err = err ?: linker_append_btf(linker, &obj);
457 err = err ?: linker_append_btf_ext(linker, &obj);
458
459 /* free up src_obj resources */
460 free(obj.btf_type_map);
461 btf__free(obj.btf);
462 btf_ext__free(obj.btf_ext);
463 free(obj.secs);
464 free(obj.sym_map);
465 if (obj.elf)
466 elf_end(obj.elf);
467 if (obj.fd >= 0)
468 close(obj.fd);
469
470 return libbpf_err(err);
471}
472
473static bool is_dwarf_sec_name(const char *name)
474{
475 /* approximation, but the actual list is too long */
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477}
478
479static bool is_ignored_sec(struct src_sec *sec)
480{
481 Elf64_Shdr *shdr = sec->shdr;
482 const char *name = sec->sec_name;
483
484 /* no special handling of .strtab */
485 if (shdr->sh_type == SHT_STRTAB)
486 return true;
487
488 /* ignore .llvm_addrsig section as well */
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 return true;
491
492 /* no subprograms will lead to an empty .text section, ignore it */
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 strcmp(sec->sec_name, ".text") == 0)
495 return true;
496
497 /* DWARF sections */
498 if (is_dwarf_sec_name(sec->sec_name))
499 return true;
500
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 name += sizeof(".rel") - 1;
503 /* DWARF section relocations */
504 if (is_dwarf_sec_name(name))
505 return true;
506
507 /* .BTF and .BTF.ext don't need relocations */
508 if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 return true;
511 }
512
513 return false;
514}
515
516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517{
518 struct src_sec *secs = obj->secs, *sec;
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520
521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 if (!secs)
523 return NULL;
524
525 /* zero out newly allocated memory */
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527
528 obj->secs = secs;
529 obj->sec_cnt = new_cnt;
530
531 sec = &obj->secs[new_cnt - 1];
532 sec->id = new_cnt - 1;
533 sec->sec_name = sec_name;
534
535 return sec;
536}
537
538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)
541{
542#if __BYTE_ORDER == __LITTLE_ENDIAN
543 const int host_endianness = ELFDATA2LSB;
544#elif __BYTE_ORDER == __BIG_ENDIAN
545 const int host_endianness = ELFDATA2MSB;
546#else
547#error "Unknown __BYTE_ORDER"
548#endif
549 int err = 0;
550 Elf_Scn *scn;
551 Elf_Data *data;
552 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;
554 struct src_sec *sec;
555
556 pr_debug("linker: adding object file '%s'...\n", filename);
557
558 obj->filename = filename;
559
560 obj->fd = open(filename, O_RDONLY);
561 if (obj->fd < 0) {
562 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);
564 return err;
565 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 if (!obj->elf) {
568 err = -errno;
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return err;
571 }
572
573 /* Sanity check ELF file high-level properties */
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 err = -errno;
577 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;
579 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 return err;
584 }
585 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 err = -EOPNOTSUPP;
589 pr_warn_elf("unsupported kind of ELF file %s", filename);
590 return err;
591 }
592
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 err = -errno;
595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 return err;
597 }
598
599 scn = NULL;
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 size_t sec_idx = elf_ndxscn(scn);
602 const char *sec_name;
603
604 shdr = elf64_getshdr(scn);
605 if (!shdr) {
606 err = -errno;
607 pr_warn_elf("failed to get section #%zu header for %s",
608 sec_idx, filename);
609 return err;
610 }
611
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 if (!sec_name) {
614 err = -errno;
615 pr_warn_elf("failed to get section #%zu name for %s",
616 sec_idx, filename);
617 return err;
618 }
619
620 data = elf_getdata(scn, 0);
621 if (!data) {
622 err = -errno;
623 pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 sec_idx, sec_name, filename);
625 return err;
626 }
627
628 sec = add_src_sec(obj, sec_name);
629 if (!sec)
630 return -ENOMEM;
631
632 sec->scn = scn;
633 sec->shdr = shdr;
634 sec->data = data;
635 sec->sec_idx = elf_ndxscn(scn);
636
637 if (is_ignored_sec(sec)) {
638 sec->skipped = true;
639 continue;
640 }
641
642 switch (shdr->sh_type) {
643 case SHT_SYMTAB:
644 if (obj->symtab_sec_idx) {
645 err = -EOPNOTSUPP;
646 pr_warn("multiple SYMTAB sections found, not supported\n");
647 return err;
648 }
649 obj->symtab_sec_idx = sec_idx;
650 break;
651 case SHT_STRTAB:
652 /* we'll construct our own string table */
653 break;
654 case SHT_PROGBITS:
655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);
658 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 return err;
661 }
662 sec->skipped = true;
663 continue;
664 }
665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);
668 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675
676 /* data & code */
677 break;
678 case SHT_NOBITS:
679 /* BSS */
680 break;
681 case SHT_REL:
682 /* relocations */
683 break;
684 default:
685 pr_warn("unrecognized section #%zu (%s) in %s\n",
686 sec_idx, sec_name, filename);
687 err = -EINVAL;
688 return err;
689 }
690 }
691
692 err = err ?: linker_sanity_check_elf(obj);
693 err = err ?: linker_sanity_check_btf(obj);
694 err = err ?: linker_sanity_check_btf_ext(obj);
695 err = err ?: linker_fixup_btf(obj);
696
697 return err;
698}
699
700static bool is_pow_of_2(size_t x)
701{
702 return x && (x & (x - 1)) == 0;
703}
704
705static int linker_sanity_check_elf(struct src_obj *obj)
706{
707 struct src_sec *sec;
708 int i, err;
709
710 if (!obj->symtab_sec_idx) {
711 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
712 return -EINVAL;
713 }
714 if (!obj->shstrs_sec_idx) {
715 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
716 return -EINVAL;
717 }
718
719 for (i = 1; i < obj->sec_cnt; i++) {
720 sec = &obj->secs[i];
721
722 if (sec->sec_name[0] == '\0') {
723 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
724 return -EINVAL;
725 }
726
727 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
728 return -EINVAL;
729 if (sec->shdr->sh_addralign != sec->data->d_align)
730 return -EINVAL;
731
732 if (sec->shdr->sh_size != sec->data->d_size)
733 return -EINVAL;
734
735 switch (sec->shdr->sh_type) {
736 case SHT_SYMTAB:
737 err = linker_sanity_check_elf_symtab(obj, sec);
738 if (err)
739 return err;
740 break;
741 case SHT_STRTAB:
742 break;
743 case SHT_PROGBITS:
744 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
745 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
746 return -EINVAL;
747 }
748 break;
749 case SHT_NOBITS:
750 break;
751 case SHT_REL:
752 err = linker_sanity_check_elf_relos(obj, sec);
753 if (err)
754 return err;
755 break;
756 case SHT_LLVM_ADDRSIG:
757 break;
758 default:
759 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
760 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
761 return -EINVAL;
762 }
763 }
764
765 return 0;
766}
767
768static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
769{
770 struct src_sec *link_sec;
771 Elf64_Sym *sym;
772 int i, n;
773
774 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
775 return -EINVAL;
776 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
777 return -EINVAL;
778
779 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
780 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
781 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
782 return -EINVAL;
783 }
784 link_sec = &obj->secs[sec->shdr->sh_link];
785 if (link_sec->shdr->sh_type != SHT_STRTAB) {
786 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
787 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
788 return -EINVAL;
789 }
790
791 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
792 sym = sec->data->d_buf;
793 for (i = 0; i < n; i++, sym++) {
794 int sym_type = ELF64_ST_TYPE(sym->st_info);
795 int sym_bind = ELF64_ST_BIND(sym->st_info);
796 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
797
798 if (i == 0) {
799 if (sym->st_name != 0 || sym->st_info != 0
800 || sym->st_other != 0 || sym->st_shndx != 0
801 || sym->st_value != 0 || sym->st_size != 0) {
802 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
803 return -EINVAL;
804 }
805 continue;
806 }
807 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
808 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
809 i, sec->sec_idx, sym_bind);
810 return -EINVAL;
811 }
812 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
813 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
814 i, sec->sec_idx, sym_vis);
815 return -EINVAL;
816 }
817 if (sym->st_shndx == 0) {
818 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
819 || sym->st_value != 0 || sym->st_size != 0) {
820 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
821 i, obj->filename);
822
823 return -EINVAL;
824 }
825 continue;
826 }
827 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
828 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
829 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
830 return -EINVAL;
831 }
832 if (sym_type == STT_SECTION) {
833 if (sym->st_value != 0)
834 return -EINVAL;
835 continue;
836 }
837 }
838
839 return 0;
840}
841
842static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
843{
844 struct src_sec *link_sec, *sym_sec;
845 Elf64_Rel *relo;
846 int i, n;
847
848 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
849 return -EINVAL;
850 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
851 return -EINVAL;
852
853 /* SHT_REL's sh_link should point to SYMTAB */
854 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
855 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
856 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
857 return -EINVAL;
858 }
859
860 /* SHT_REL's sh_info points to relocated section */
861 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
862 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
863 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
864 return -EINVAL;
865 }
866 link_sec = &obj->secs[sec->shdr->sh_info];
867
868 /* .rel<secname> -> <secname> pattern is followed */
869 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
870 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
871 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
872 sec->sec_idx, obj->filename);
873 return -EINVAL;
874 }
875
876 /* don't further validate relocations for ignored sections */
877 if (link_sec->skipped)
878 return 0;
879
880 /* relocatable section is data or instructions */
881 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
882 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
883 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
884 return -EINVAL;
885 }
886
887 /* check sanity of each relocation */
888 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
889 relo = sec->data->d_buf;
890 sym_sec = &obj->secs[obj->symtab_sec_idx];
891 for (i = 0; i < n; i++, relo++) {
892 size_t sym_idx = ELF64_R_SYM(relo->r_info);
893 size_t sym_type = ELF64_R_TYPE(relo->r_info);
894
895 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
896 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
897 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
898 i, sec->sec_idx, sym_type, obj->filename);
899 return -EINVAL;
900 }
901
902 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
903 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
904 i, sec->sec_idx, sym_idx, obj->filename);
905 return -EINVAL;
906 }
907
908 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
909 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
910 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
911 i, sec->sec_idx, sym_idx, obj->filename);
912 return -EINVAL;
913 }
914 }
915 }
916
917 return 0;
918}
919
920static int check_btf_type_id(__u32 *type_id, void *ctx)
921{
922 struct btf *btf = ctx;
923
924 if (*type_id > btf__get_nr_types(btf))
925 return -EINVAL;
926
927 return 0;
928}
929
930static int check_btf_str_off(__u32 *str_off, void *ctx)
931{
932 struct btf *btf = ctx;
933 const char *s;
934
935 s = btf__str_by_offset(btf, *str_off);
936
937 if (!s)
938 return -EINVAL;
939
940 return 0;
941}
942
943static int linker_sanity_check_btf(struct src_obj *obj)
944{
945 struct btf_type *t;
946 int i, n, err = 0;
947
948 if (!obj->btf)
949 return 0;
950
951 n = btf__get_nr_types(obj->btf);
952 for (i = 1; i <= n; i++) {
953 t = btf_type_by_id(obj->btf, i);
954
955 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
956 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
957 if (err)
958 return err;
959 }
960
961 return 0;
962}
963
964static int linker_sanity_check_btf_ext(struct src_obj *obj)
965{
966 int err = 0;
967
968 if (!obj->btf_ext)
969 return 0;
970
971 /* can't use .BTF.ext without .BTF */
972 if (!obj->btf)
973 return -EINVAL;
974
975 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
976 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
977 if (err)
978 return err;
979
980 return 0;
981}
982
983static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
984{
985 Elf_Scn *scn;
986 Elf_Data *data;
987 Elf64_Shdr *shdr;
988 int name_off;
989
990 dst_sec->sec_sz = 0;
991 dst_sec->sec_idx = 0;
992 dst_sec->ephemeral = src_sec->ephemeral;
993
994 /* ephemeral sections are just thin section shells lacking most parts */
995 if (src_sec->ephemeral)
996 return 0;
997
998 scn = elf_newscn(linker->elf);
999 if (!scn)
1000 return -ENOMEM;
1001 data = elf_newdata(scn);
1002 if (!data)
1003 return -ENOMEM;
1004 shdr = elf64_getshdr(scn);
1005 if (!shdr)
1006 return -ENOMEM;
1007
1008 dst_sec->scn = scn;
1009 dst_sec->shdr = shdr;
1010 dst_sec->data = data;
1011 dst_sec->sec_idx = elf_ndxscn(scn);
1012
1013 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1014 if (name_off < 0)
1015 return name_off;
1016
1017 shdr->sh_name = name_off;
1018 shdr->sh_type = src_sec->shdr->sh_type;
1019 shdr->sh_flags = src_sec->shdr->sh_flags;
1020 shdr->sh_size = 0;
1021 /* sh_link and sh_info have different meaning for different types of
1022 * sections, so we leave it up to the caller code to fill them in, if
1023 * necessary
1024 */
1025 shdr->sh_link = 0;
1026 shdr->sh_info = 0;
1027 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1028 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1029
1030 data->d_type = src_sec->data->d_type;
1031 data->d_size = 0;
1032 data->d_buf = NULL;
1033 data->d_align = src_sec->data->d_align;
1034 data->d_off = 0;
1035
1036 return 0;
1037}
1038
1039static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1040{
1041 struct dst_sec *sec;
1042 int i;
1043
1044 for (i = 1; i < linker->sec_cnt; i++) {
1045 sec = &linker->secs[i];
1046
1047 if (strcmp(sec->sec_name, sec_name) == 0)
1048 return sec;
1049 }
1050
1051 return NULL;
1052}
1053
1054static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1055{
1056 if (dst->ephemeral || src->ephemeral)
1057 return true;
1058
1059 if (dst->shdr->sh_type != src->shdr->sh_type) {
1060 pr_warn("sec %s types mismatch\n", dst->sec_name);
1061 return false;
1062 }
1063 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1064 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1065 return false;
1066 }
1067 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1068 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1069 return false;
1070 }
1071
1072 return true;
1073}
1074
1075static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1076{
1077 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1078 return false;
1079 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1080 return false;
1081 return true;
1082}
1083
1084static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1085{
1086 void *tmp;
1087 size_t dst_align, src_align;
1088 size_t dst_align_sz, dst_final_sz;
1089 int err;
1090
1091 /* Ephemeral source section doesn't contribute anything to ELF
1092 * section data.
1093 */
1094 if (src->ephemeral)
1095 return 0;
1096
1097 /* Some sections (like .maps) can contain both externs (and thus be
1098 * ephemeral) and non-externs (map definitions). So it's possible that
1099 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1100 * first non-ephemeral entity appears. In such case, we add ELF
1101 * section, data, etc.
1102 */
1103 if (dst->ephemeral) {
1104 err = init_sec(linker, dst, src);
1105 if (err)
1106 return err;
1107 }
1108
1109 dst_align = dst->shdr->sh_addralign;
1110 src_align = src->shdr->sh_addralign;
1111 if (dst_align == 0)
1112 dst_align = 1;
1113 if (dst_align < src_align)
1114 dst_align = src_align;
1115
1116 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1117
1118 /* no need to re-align final size */
1119 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1120
1121 if (src->shdr->sh_type != SHT_NOBITS) {
1122 tmp = realloc(dst->raw_data, dst_final_sz);
1123 if (!tmp)
1124 return -ENOMEM;
1125 dst->raw_data = tmp;
1126
1127 /* pad dst section, if it's alignment forced size increase */
1128 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1129 /* now copy src data at a properly aligned offset */
1130 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1131 }
1132
1133 dst->sec_sz = dst_final_sz;
1134 dst->shdr->sh_size = dst_final_sz;
1135 dst->data->d_size = dst_final_sz;
1136
1137 dst->shdr->sh_addralign = dst_align;
1138 dst->data->d_align = dst_align;
1139
1140 src->dst_off = dst_align_sz;
1141
1142 return 0;
1143}
1144
1145static bool is_data_sec(struct src_sec *sec)
1146{
1147 if (!sec || sec->skipped)
1148 return false;
1149 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1150 if (sec->ephemeral)
1151 return true;
1152 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1153}
1154
1155static bool is_relo_sec(struct src_sec *sec)
1156{
1157 if (!sec || sec->skipped || sec->ephemeral)
1158 return false;
1159 return sec->shdr->sh_type == SHT_REL;
1160}
1161
1162static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1163{
1164 int i, err;
1165
1166 for (i = 1; i < obj->sec_cnt; i++) {
1167 struct src_sec *src_sec;
1168 struct dst_sec *dst_sec;
1169
1170 src_sec = &obj->secs[i];
1171 if (!is_data_sec(src_sec))
1172 continue;
1173
1174 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1175 if (!dst_sec) {
1176 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1177 if (!dst_sec)
1178 return -ENOMEM;
1179 err = init_sec(linker, dst_sec, src_sec);
1180 if (err) {
1181 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1182 return err;
1183 }
1184 } else {
1185 if (!secs_match(dst_sec, src_sec)) {
1186 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1187 return -1;
1188 }
1189
1190 /* "license" and "version" sections are deduped */
1191 if (strcmp(src_sec->sec_name, "license") == 0
1192 || strcmp(src_sec->sec_name, "version") == 0) {
1193 if (!sec_content_is_same(dst_sec, src_sec)) {
1194 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1195 return -EINVAL;
1196 }
1197 src_sec->skipped = true;
1198 src_sec->dst_id = dst_sec->id;
1199 continue;
1200 }
1201 }
1202
1203 /* record mapped section index */
1204 src_sec->dst_id = dst_sec->id;
1205
1206 err = extend_sec(linker, dst_sec, src_sec);
1207 if (err)
1208 return err;
1209 }
1210
1211 return 0;
1212}
1213
1214static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1215{
1216 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1217 Elf64_Sym *sym = symtab->data->d_buf;
1218 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1219 int str_sec_idx = symtab->shdr->sh_link;
1220 const char *sym_name;
1221
1222 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1223 if (!obj->sym_map)
1224 return -ENOMEM;
1225
1226 for (i = 0; i < n; i++, sym++) {
1227 /* We already validated all-zero symbol #0 and we already
1228 * appended it preventively to the final SYMTAB, so skip it.
1229 */
1230 if (i == 0)
1231 continue;
1232
1233 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1234 if (!sym_name) {
1235 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1236 return -EINVAL;
1237 }
1238
1239 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1240 if (err)
1241 return err;
1242 }
1243
1244 return 0;
1245}
1246
1247static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1248{
1249 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1250 Elf64_Sym *syms = symtab->raw_data;
1251
1252 return &syms[sym_idx];
1253}
1254
1255static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1256{
1257 struct glob_sym *glob_sym;
1258 const char *name;
1259 int i;
1260
1261 for (i = 0; i < linker->glob_sym_cnt; i++) {
1262 glob_sym = &linker->glob_syms[i];
1263 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1264
1265 if (strcmp(name, sym_name) == 0)
1266 return glob_sym;
1267 }
1268
1269 return NULL;
1270}
1271
1272static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1273{
1274 struct glob_sym *syms, *sym;
1275
1276 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1277 sizeof(*linker->glob_syms));
1278 if (!syms)
1279 return NULL;
1280
1281 sym = &syms[linker->glob_sym_cnt];
1282 memset(sym, 0, sizeof(*sym));
1283 sym->var_idx = -1;
1284
1285 linker->glob_syms = syms;
1286 linker->glob_sym_cnt++;
1287
1288 return sym;
1289}
1290
1291static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1292 const struct btf *btf1, __u32 id1,
1293 const struct btf *btf2, __u32 id2)
1294{
1295 const struct btf_type *t1, *t2;
1296 bool is_static1, is_static2;
1297 const char *n1, *n2;
1298 int i, n;
1299
1300recur:
1301 n1 = n2 = NULL;
1302 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1303 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1304
1305 /* check if only one side is FWD, otherwise handle with common logic */
1306 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1307 n1 = btf__str_by_offset(btf1, t1->name_off);
1308 n2 = btf__str_by_offset(btf2, t2->name_off);
1309 if (strcmp(n1, n2) != 0) {
1310 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1311 sym_name, n1, n2);
1312 return false;
1313 }
1314 /* validate if FWD kind matches concrete kind */
1315 if (btf_is_fwd(t1)) {
1316 if (btf_kflag(t1) && btf_is_union(t2))
1317 return true;
1318 if (!btf_kflag(t1) && btf_is_struct(t2))
1319 return true;
1320 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1321 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1322 } else {
1323 if (btf_kflag(t2) && btf_is_union(t1))
1324 return true;
1325 if (!btf_kflag(t2) && btf_is_struct(t1))
1326 return true;
1327 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1328 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1329 }
1330 return false;
1331 }
1332
1333 if (btf_kind(t1) != btf_kind(t2)) {
1334 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1335 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1336 return false;
1337 }
1338
1339 switch (btf_kind(t1)) {
1340 case BTF_KIND_STRUCT:
1341 case BTF_KIND_UNION:
1342 case BTF_KIND_ENUM:
1343 case BTF_KIND_FWD:
1344 case BTF_KIND_FUNC:
1345 case BTF_KIND_VAR:
1346 n1 = btf__str_by_offset(btf1, t1->name_off);
1347 n2 = btf__str_by_offset(btf2, t2->name_off);
1348 if (strcmp(n1, n2) != 0) {
1349 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1350 sym_name, btf_kind_str(t1), n1, n2);
1351 return false;
1352 }
1353 break;
1354 default:
1355 break;
1356 }
1357
1358 switch (btf_kind(t1)) {
1359 case BTF_KIND_UNKN: /* void */
1360 case BTF_KIND_FWD:
1361 return true;
1362 case BTF_KIND_INT:
1363 case BTF_KIND_FLOAT:
1364 case BTF_KIND_ENUM:
1365 /* ignore encoding for int and enum values for enum */
1366 if (t1->size != t2->size) {
1367 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1368 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1369 return false;
1370 }
1371 return true;
1372 case BTF_KIND_PTR:
1373 /* just validate overall shape of the referenced type, so no
1374 * contents comparison for struct/union, and allowd fwd vs
1375 * struct/union
1376 */
1377 exact = false;
1378 id1 = t1->type;
1379 id2 = t2->type;
1380 goto recur;
1381 case BTF_KIND_ARRAY:
1382 /* ignore index type and array size */
1383 id1 = btf_array(t1)->type;
1384 id2 = btf_array(t2)->type;
1385 goto recur;
1386 case BTF_KIND_FUNC:
1387 /* extern and global linkages are compatible */
1388 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1389 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1390 if (is_static1 != is_static2) {
1391 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1392 return false;
1393 }
1394
1395 id1 = t1->type;
1396 id2 = t2->type;
1397 goto recur;
1398 case BTF_KIND_VAR:
1399 /* extern and global linkages are compatible */
1400 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1401 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1402 if (is_static1 != is_static2) {
1403 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1404 return false;
1405 }
1406
1407 id1 = t1->type;
1408 id2 = t2->type;
1409 goto recur;
1410 case BTF_KIND_STRUCT:
1411 case BTF_KIND_UNION: {
1412 const struct btf_member *m1, *m2;
1413
1414 if (!exact)
1415 return true;
1416
1417 if (btf_vlen(t1) != btf_vlen(t2)) {
1418 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1419 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1420 return false;
1421 }
1422
1423 n = btf_vlen(t1);
1424 m1 = btf_members(t1);
1425 m2 = btf_members(t2);
1426 for (i = 0; i < n; i++, m1++, m2++) {
1427 n1 = btf__str_by_offset(btf1, m1->name_off);
1428 n2 = btf__str_by_offset(btf2, m2->name_off);
1429 if (strcmp(n1, n2) != 0) {
1430 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1431 sym_name, i, n1, n2);
1432 return false;
1433 }
1434 if (m1->offset != m2->offset) {
1435 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1436 sym_name, i, n1);
1437 return false;
1438 }
1439 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1440 return false;
1441 }
1442
1443 return true;
1444 }
1445 case BTF_KIND_FUNC_PROTO: {
1446 const struct btf_param *m1, *m2;
1447
1448 if (btf_vlen(t1) != btf_vlen(t2)) {
1449 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1450 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1451 return false;
1452 }
1453
1454 n = btf_vlen(t1);
1455 m1 = btf_params(t1);
1456 m2 = btf_params(t2);
1457 for (i = 0; i < n; i++, m1++, m2++) {
1458 /* ignore func arg names */
1459 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1460 return false;
1461 }
1462
1463 /* now check return type as well */
1464 id1 = t1->type;
1465 id2 = t2->type;
1466 goto recur;
1467 }
1468
1469 /* skip_mods_and_typedefs() make this impossible */
1470 case BTF_KIND_TYPEDEF:
1471 case BTF_KIND_VOLATILE:
1472 case BTF_KIND_CONST:
1473 case BTF_KIND_RESTRICT:
1474 /* DATASECs are never compared with each other */
1475 case BTF_KIND_DATASEC:
1476 default:
1477 pr_warn("global '%s': unsupported BTF kind %s\n",
1478 sym_name, btf_kind_str(t1));
1479 return false;
1480 }
1481}
1482
1483static bool map_defs_match(const char *sym_name,
1484 const struct btf *main_btf,
1485 const struct btf_map_def *main_def,
1486 const struct btf_map_def *main_inner_def,
1487 const struct btf *extra_btf,
1488 const struct btf_map_def *extra_def,
1489 const struct btf_map_def *extra_inner_def)
1490{
1491 const char *reason;
1492
1493 if (main_def->map_type != extra_def->map_type) {
1494 reason = "type";
1495 goto mismatch;
1496 }
1497
1498 /* check key type/size match */
1499 if (main_def->key_size != extra_def->key_size) {
1500 reason = "key_size";
1501 goto mismatch;
1502 }
1503 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1504 reason = "key type";
1505 goto mismatch;
1506 }
1507 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1508 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1509 main_btf, main_def->key_type_id,
1510 extra_btf, extra_def->key_type_id)) {
1511 reason = "key type";
1512 goto mismatch;
1513 }
1514
1515 /* validate value type/size match */
1516 if (main_def->value_size != extra_def->value_size) {
1517 reason = "value_size";
1518 goto mismatch;
1519 }
1520 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1521 reason = "value type";
1522 goto mismatch;
1523 }
1524 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1525 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1526 main_btf, main_def->value_type_id,
1527 extra_btf, extra_def->value_type_id)) {
1528 reason = "key type";
1529 goto mismatch;
1530 }
1531
1532 if (main_def->max_entries != extra_def->max_entries) {
1533 reason = "max_entries";
1534 goto mismatch;
1535 }
1536 if (main_def->map_flags != extra_def->map_flags) {
1537 reason = "map_flags";
1538 goto mismatch;
1539 }
1540 if (main_def->numa_node != extra_def->numa_node) {
1541 reason = "numa_node";
1542 goto mismatch;
1543 }
1544 if (main_def->pinning != extra_def->pinning) {
1545 reason = "pinning";
1546 goto mismatch;
1547 }
1548
1549 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1550 reason = "inner map";
1551 goto mismatch;
1552 }
1553
1554 if (main_def->parts & MAP_DEF_INNER_MAP) {
1555 char inner_map_name[128];
1556
1557 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1558
1559 return map_defs_match(inner_map_name,
1560 main_btf, main_inner_def, NULL,
1561 extra_btf, extra_inner_def, NULL);
1562 }
1563
1564 return true;
1565
1566mismatch:
1567 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1568 return false;
1569}
1570
1571static bool glob_map_defs_match(const char *sym_name,
1572 struct bpf_linker *linker, struct glob_sym *glob_sym,
1573 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1574{
1575 struct btf_map_def dst_def = {}, dst_inner_def = {};
1576 struct btf_map_def src_def = {}, src_inner_def = {};
1577 const struct btf_type *t;
1578 int err;
1579
1580 t = btf__type_by_id(obj->btf, btf_id);
1581 if (!btf_is_var(t)) {
1582 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1583 return false;
1584 }
1585 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1586
1587 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1588 if (err) {
1589 pr_warn("global '%s': invalid map definition\n", sym_name);
1590 return false;
1591 }
1592
1593 /* re-parse existing map definition */
1594 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1595 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1596 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1597 if (err) {
1598 /* this should not happen, because we already validated it */
1599 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1600 return false;
1601 }
1602
1603 /* Currently extern map definition has to be complete and match
1604 * concrete map definition exactly. This restriction might be lifted
1605 * in the future.
1606 */
1607 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1608 obj->btf, &src_def, &src_inner_def);
1609}
1610
1611static bool glob_syms_match(const char *sym_name,
1612 struct bpf_linker *linker, struct glob_sym *glob_sym,
1613 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1614{
1615 const struct btf_type *src_t;
1616
1617 /* if we are dealing with externs, BTF types describing both global
1618 * and extern VARs/FUNCs should be completely present in all files
1619 */
1620 if (!glob_sym->btf_id || !btf_id) {
1621 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1622 return false;
1623 }
1624
1625 src_t = btf__type_by_id(obj->btf, btf_id);
1626 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1627 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1628 btf_kind_str(src_t), sym_name);
1629 return false;
1630 }
1631
1632 /* deal with .maps definitions specially */
1633 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1634 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1635
1636 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1637 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1638 return false;
1639
1640 return true;
1641}
1642
1643static bool btf_is_non_static(const struct btf_type *t)
1644{
1645 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1646 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1647}
1648
1649static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1650 int *out_btf_sec_id, int *out_btf_id)
1651{
1652 int i, j, n, m, btf_id = 0;
1653 const struct btf_type *t;
1654 const struct btf_var_secinfo *vi;
1655 const char *name;
1656
1657 if (!obj->btf) {
1658 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1659 return -EINVAL;
1660 }
1661
1662 n = btf__get_nr_types(obj->btf);
1663 for (i = 1; i <= n; i++) {
1664 t = btf__type_by_id(obj->btf, i);
1665
1666 /* some global and extern FUNCs and VARs might not be associated with any
1667 * DATASEC, so try to detect them in the same pass
1668 */
1669 if (btf_is_non_static(t)) {
1670 name = btf__str_by_offset(obj->btf, t->name_off);
1671 if (strcmp(name, sym_name) != 0)
1672 continue;
1673
1674 /* remember and still try to find DATASEC */
1675 btf_id = i;
1676 continue;
1677 }
1678
1679 if (!btf_is_datasec(t))
1680 continue;
1681
1682 vi = btf_var_secinfos(t);
1683 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1684 t = btf__type_by_id(obj->btf, vi->type);
1685 name = btf__str_by_offset(obj->btf, t->name_off);
1686
1687 if (strcmp(name, sym_name) != 0)
1688 continue;
1689 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1690 continue;
1691 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1692 continue;
1693
1694 if (btf_id && btf_id != vi->type) {
1695 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1696 sym_name, btf_id, vi->type);
1697 return -EINVAL;
1698 }
1699
1700 *out_btf_sec_id = i;
1701 *out_btf_id = vi->type;
1702
1703 return 0;
1704 }
1705 }
1706
1707 /* free-floating extern or global FUNC */
1708 if (btf_id) {
1709 *out_btf_sec_id = 0;
1710 *out_btf_id = btf_id;
1711 return 0;
1712 }
1713
1714 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1715 return -ENOENT;
1716}
1717
1718static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1719{
1720 struct src_sec *sec;
1721 int i;
1722
1723 for (i = 1; i < obj->sec_cnt; i++) {
1724 sec = &obj->secs[i];
1725
1726 if (strcmp(sec->sec_name, sec_name) == 0)
1727 return sec;
1728 }
1729
1730 return NULL;
1731}
1732
1733static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1734 struct btf *src_btf, int src_id)
1735{
1736 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1737 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1738 struct btf_param *src_p, *dst_p;
1739 const char *s;
1740 int i, n, off;
1741
1742 /* We already made sure that source and destination types (FUNC or
1743 * VAR) match in terms of types and argument names.
1744 */
1745 if (btf_is_var(dst_t)) {
1746 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1747 return 0;
1748 }
1749
1750 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1751
1752 /* now onto FUNC_PROTO types */
1753 src_t = btf_type_by_id(src_btf, src_t->type);
1754 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1755
1756 /* Fill in all the argument names, which for extern FUNCs are missing.
1757 * We'll end up with two copies of FUNCs/VARs for externs, but that
1758 * will be taken care of by BTF dedup at the very end.
1759 * It might be that BTF types for extern in one file has less/more BTF
1760 * information (e.g., FWD instead of full STRUCT/UNION information),
1761 * but that should be (in most cases, subject to BTF dedup rules)
1762 * handled and resolved by BTF dedup algorithm as well, so we won't
1763 * worry about it. Our only job is to make sure that argument names
1764 * are populated on both sides, otherwise BTF dedup will pedantically
1765 * consider them different.
1766 */
1767 src_p = btf_params(src_t);
1768 dst_p = btf_params(dst_t);
1769 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1770 if (!src_p->name_off)
1771 continue;
1772
1773 /* src_btf has more complete info, so add name to dst_btf */
1774 s = btf__str_by_offset(src_btf, src_p->name_off);
1775 off = btf__add_str(dst_btf, s);
1776 if (off < 0)
1777 return off;
1778 dst_p->name_off = off;
1779 }
1780 return 0;
1781}
1782
1783static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1784{
1785 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1786}
1787
1788static void sym_update_type(Elf64_Sym *sym, int sym_type)
1789{
1790 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1791}
1792
1793static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1794{
1795 /* libelf doesn't provide setters for ST_VISIBILITY,
1796 * but it is stored in the lower 2 bits of st_other
1797 */
1798 sym->st_other &= ~0x03;
1799 sym->st_other |= sym_vis;
1800}
1801
1802static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1803 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1804{
1805 struct src_sec *src_sec = NULL;
1806 struct dst_sec *dst_sec = NULL;
1807 struct glob_sym *glob_sym = NULL;
1808 int name_off, sym_type, sym_bind, sym_vis, err;
1809 int btf_sec_id = 0, btf_id = 0;
1810 size_t dst_sym_idx;
1811 Elf64_Sym *dst_sym;
1812 bool sym_is_extern;
1813
1814 sym_type = ELF64_ST_TYPE(sym->st_info);
1815 sym_bind = ELF64_ST_BIND(sym->st_info);
1816 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1817 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1818
1819 if (sym_is_extern) {
1820 if (!obj->btf) {
1821 pr_warn("externs without BTF info are not supported\n");
1822 return -ENOTSUP;
1823 }
1824 } else if (sym->st_shndx < SHN_LORESERVE) {
1825 src_sec = &obj->secs[sym->st_shndx];
1826 if (src_sec->skipped)
1827 return 0;
1828 dst_sec = &linker->secs[src_sec->dst_id];
1829
1830 /* allow only one STT_SECTION symbol per section */
1831 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1832 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1833 return 0;
1834 }
1835 }
1836
1837 if (sym_bind == STB_LOCAL)
1838 goto add_sym;
1839
1840 /* find matching BTF info */
1841 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1842 if (err)
1843 return err;
1844
1845 if (sym_is_extern && btf_sec_id) {
1846 const char *sec_name = NULL;
1847 const struct btf_type *t;
1848
1849 t = btf__type_by_id(obj->btf, btf_sec_id);
1850 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1851
1852 /* Clang puts unannotated extern vars into
1853 * '.extern' BTF DATASEC. Treat them the same
1854 * as unannotated extern funcs (which are
1855 * currently not put into any DATASECs).
1856 * Those don't have associated src_sec/dst_sec.
1857 */
1858 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1859 src_sec = find_src_sec_by_name(obj, sec_name);
1860 if (!src_sec) {
1861 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1862 return -ENOENT;
1863 }
1864 dst_sec = &linker->secs[src_sec->dst_id];
1865 }
1866 }
1867
1868 glob_sym = find_glob_sym(linker, sym_name);
1869 if (glob_sym) {
1870 /* Preventively resolve to existing symbol. This is
1871 * needed for further relocation symbol remapping in
1872 * the next step of linking.
1873 */
1874 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1875
1876 /* If both symbols are non-externs, at least one of
1877 * them has to be STB_WEAK, otherwise they are in
1878 * a conflict with each other.
1879 */
1880 if (!sym_is_extern && !glob_sym->is_extern
1881 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1882 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1883 src_sym_idx, sym_name, obj->filename);
1884 return -EINVAL;
1885 }
1886
1887 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1888 return -EINVAL;
1889
1890 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1891
1892 /* If new symbol is strong, then force dst_sym to be strong as
1893 * well; this way a mix of weak and non-weak extern
1894 * definitions will end up being strong.
1895 */
1896 if (sym_bind == STB_GLOBAL) {
1897 /* We still need to preserve type (NOTYPE or
1898 * OBJECT/FUNC, depending on whether the symbol is
1899 * extern or not)
1900 */
1901 sym_update_bind(dst_sym, STB_GLOBAL);
1902 glob_sym->is_weak = false;
1903 }
1904
1905 /* Non-default visibility is "contaminating", with stricter
1906 * visibility overwriting more permissive ones, even if more
1907 * permissive visibility comes from just an extern definition.
1908 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1909 * ensured by ELF symbol sanity checks above.
1910 */
1911 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1912 sym_update_visibility(dst_sym, sym_vis);
1913
1914 /* If the new symbol is extern, then regardless if
1915 * existing symbol is extern or resolved global, just
1916 * keep the existing one untouched.
1917 */
1918 if (sym_is_extern)
1919 return 0;
1920
1921 /* If existing symbol is a strong resolved symbol, bail out,
1922 * because we lost resolution battle have nothing to
1923 * contribute. We already checked abover that there is no
1924 * strong-strong conflict. We also already tightened binding
1925 * and visibility, so nothing else to contribute at that point.
1926 */
1927 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1928 return 0;
1929
1930 /* At this point, new symbol is strong non-extern,
1931 * so overwrite glob_sym with new symbol information.
1932 * Preserve binding and visibility.
1933 */
1934 sym_update_type(dst_sym, sym_type);
1935 dst_sym->st_shndx = dst_sec->sec_idx;
1936 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1937 dst_sym->st_size = sym->st_size;
1938
1939 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1940 glob_sym->sec_id = dst_sec->id;
1941 glob_sym->is_extern = false;
1942
1943 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1944 obj->btf, btf_id))
1945 return -EINVAL;
1946
1947 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1948 glob_sym->underlying_btf_id = 0;
1949
1950 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1951 return 0;
1952 }
1953
1954add_sym:
1955 name_off = strset__add_str(linker->strtab_strs, sym_name);
1956 if (name_off < 0)
1957 return name_off;
1958
1959 dst_sym = add_new_sym(linker, &dst_sym_idx);
1960 if (!dst_sym)
1961 return -ENOMEM;
1962
1963 dst_sym->st_name = name_off;
1964 dst_sym->st_info = sym->st_info;
1965 dst_sym->st_other = sym->st_other;
1966 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1967 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1968 dst_sym->st_size = sym->st_size;
1969
1970 obj->sym_map[src_sym_idx] = dst_sym_idx;
1971
1972 if (sym_type == STT_SECTION && dst_sym) {
1973 dst_sec->sec_sym_idx = dst_sym_idx;
1974 dst_sym->st_value = 0;
1975 }
1976
1977 if (sym_bind != STB_LOCAL) {
1978 glob_sym = add_glob_sym(linker);
1979 if (!glob_sym)
1980 return -ENOMEM;
1981
1982 glob_sym->sym_idx = dst_sym_idx;
1983 /* we use dst_sec->id (and not dst_sec->sec_idx), because
1984 * ephemeral sections (.kconfig, .ksyms, etc) don't have
1985 * sec_idx (as they don't have corresponding ELF section), but
1986 * still have id. .extern doesn't have even ephemeral section
1987 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1988 */
1989 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1990 glob_sym->name_off = name_off;
1991 /* we will fill btf_id in during BTF merging step */
1992 glob_sym->btf_id = 0;
1993 glob_sym->is_extern = sym_is_extern;
1994 glob_sym->is_weak = sym_bind == STB_WEAK;
1995 }
1996
1997 return 0;
1998}
1999
2000static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2001{
2002 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2003 struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
2004 int i, err;
2005
2006 for (i = 1; i < obj->sec_cnt; i++) {
2007 struct src_sec *src_sec, *src_linked_sec;
2008 struct dst_sec *dst_sec, *dst_linked_sec;
2009 Elf64_Rel *src_rel, *dst_rel;
2010 int j, n;
2011
2012 src_sec = &obj->secs[i];
2013 if (!is_relo_sec(src_sec))
2014 continue;
2015
2016 /* shdr->sh_info points to relocatable section */
2017 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2018 if (src_linked_sec->skipped)
2019 continue;
2020
2021 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2022 if (!dst_sec) {
2023 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2024 if (!dst_sec)
2025 return -ENOMEM;
2026 err = init_sec(linker, dst_sec, src_sec);
2027 if (err) {
2028 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2029 return err;
2030 }
2031 } else if (!secs_match(dst_sec, src_sec)) {
2032 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2033 return -1;
2034 }
2035
2036 /* shdr->sh_link points to SYMTAB */
2037 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2038
2039 /* shdr->sh_info points to relocated section */
2040 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2041 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2042
2043 src_sec->dst_id = dst_sec->id;
2044 err = extend_sec(linker, dst_sec, src_sec);
2045 if (err)
2046 return err;
2047
2048 src_rel = src_sec->data->d_buf;
2049 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2050 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2051 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2052 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2053 size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
2054 Elf64_Sym *src_sym, *dst_sym;
2055 size_t dst_sym_idx;
2056
2057 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2058 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2059
2060 dst_sym_idx = obj->sym_map[src_sym_idx];
2061 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
2062 dst_rel->r_offset += src_linked_sec->dst_off;
2063 sym_type = ELF64_R_TYPE(src_rel->r_info);
2064 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2065
2066 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2067 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2068 struct bpf_insn *insn;
2069
2070 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2071 /* calls to the very first static function inside
2072 * .text section at offset 0 will
2073 * reference section symbol, not the
2074 * function symbol. Fix that up,
2075 * otherwise it won't be possible to
2076 * relocate calls to two different
2077 * static functions with the same name
2078 * (rom two different object files)
2079 */
2080 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2081 if (insn->code == (BPF_JMP | BPF_CALL))
2082 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2083 else
2084 insn->imm += sec->dst_off;
2085 } else {
2086 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2087 return -EINVAL;
2088 }
2089 }
2090
2091 }
2092 }
2093
2094 return 0;
2095}
2096
2097static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2098 int sym_type, const char *sym_name)
2099{
2100 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2101 Elf64_Sym *sym = symtab->data->d_buf;
2102 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2103 int str_sec_idx = symtab->shdr->sh_link;
2104 const char *name;
2105
2106 for (i = 0; i < n; i++, sym++) {
2107 if (sym->st_shndx != sec_idx)
2108 continue;
2109 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2110 continue;
2111
2112 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2113 if (!name)
2114 return NULL;
2115
2116 if (strcmp(sym_name, name) != 0)
2117 continue;
2118
2119 return sym;
2120 }
2121
2122 return NULL;
2123}
2124
2125static int linker_fixup_btf(struct src_obj *obj)
2126{
2127 const char *sec_name;
2128 struct src_sec *sec;
2129 int i, j, n, m;
2130
2131 if (!obj->btf)
2132 return 0;
2133
2134 n = btf__get_nr_types(obj->btf);
2135 for (i = 1; i <= n; i++) {
2136 struct btf_var_secinfo *vi;
2137 struct btf_type *t;
2138
2139 t = btf_type_by_id(obj->btf, i);
2140 if (btf_kind(t) != BTF_KIND_DATASEC)
2141 continue;
2142
2143 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2144 sec = find_src_sec_by_name(obj, sec_name);
2145 if (sec) {
2146 /* record actual section size, unless ephemeral */
2147 if (sec->shdr)
2148 t->size = sec->shdr->sh_size;
2149 } else {
2150 /* BTF can have some sections that are not represented
2151 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2152 * for special extern variables.
2153 *
2154 * For all but one such special (ephemeral)
2155 * sections, we pre-create "section shells" to be able
2156 * to keep track of extra per-section metadata later
2157 * (e.g., those BTF extern variables).
2158 *
2159 * .extern is even more special, though, because it
2160 * contains extern variables that need to be resolved
2161 * by static linker, not libbpf and kernel. When such
2162 * externs are resolved, we are going to remove them
2163 * from .extern BTF section and might end up not
2164 * needing it at all. Each resolved extern should have
2165 * matching non-extern VAR/FUNC in other sections.
2166 *
2167 * We do support leaving some of the externs
2168 * unresolved, though, to support cases of building
2169 * libraries, which will later be linked against final
2170 * BPF applications. So if at finalization we still
2171 * see unresolved externs, we'll create .extern
2172 * section on our own.
2173 */
2174 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2175 continue;
2176
2177 sec = add_src_sec(obj, sec_name);
2178 if (!sec)
2179 return -ENOMEM;
2180
2181 sec->ephemeral = true;
2182 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2183 }
2184
2185 /* remember ELF section and its BTF type ID match */
2186 sec->sec_type_id = i;
2187
2188 /* fix up variable offsets */
2189 vi = btf_var_secinfos(t);
2190 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2191 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2192 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2193 int var_linkage = btf_var(vt)->linkage;
2194 Elf64_Sym *sym;
2195
2196 /* no need to patch up static or extern vars */
2197 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2198 continue;
2199
2200 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2201 if (!sym) {
2202 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2203 return -ENOENT;
2204 }
2205
2206 vi->offset = sym->st_value;
2207 }
2208 }
2209
2210 return 0;
2211}
2212
2213static int remap_type_id(__u32 *type_id, void *ctx)
2214{
2215 int *id_map = ctx;
2216 int new_id = id_map[*type_id];
2217
2218 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2219 if (new_id == 0 && *type_id != 0) {
2220 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2221 return -EINVAL;
2222 }
2223
2224 *type_id = id_map[*type_id];
2225
2226 return 0;
2227}
2228
2229static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2230{
2231 const struct btf_type *t;
2232 int i, j, n, start_id, id;
2233 const char *name;
2234
2235 if (!obj->btf)
2236 return 0;
2237
2238 start_id = btf__get_nr_types(linker->btf) + 1;
2239 n = btf__get_nr_types(obj->btf);
2240
2241 obj->btf_type_map = calloc(n + 1, sizeof(int));
2242 if (!obj->btf_type_map)
2243 return -ENOMEM;
2244
2245 for (i = 1; i <= n; i++) {
2246 struct glob_sym *glob_sym = NULL;
2247
2248 t = btf__type_by_id(obj->btf, i);
2249
2250 /* DATASECs are handled specially below */
2251 if (btf_kind(t) == BTF_KIND_DATASEC)
2252 continue;
2253
2254 if (btf_is_non_static(t)) {
2255 /* there should be glob_sym already */
2256 name = btf__str_by_offset(obj->btf, t->name_off);
2257 glob_sym = find_glob_sym(linker, name);
2258
2259 /* VARs without corresponding glob_sym are those that
2260 * belong to skipped/deduplicated sections (i.e.,
2261 * license and version), so just skip them
2262 */
2263 if (!glob_sym)
2264 continue;
2265
2266 /* linker_append_elf_sym() might have requested
2267 * updating underlying type ID, if extern was resolved
2268 * to strong symbol or weak got upgraded to non-weak
2269 */
2270 if (glob_sym->underlying_btf_id == 0)
2271 glob_sym->underlying_btf_id = -t->type;
2272
2273 /* globals from previous object files that match our
2274 * VAR/FUNC already have a corresponding associated
2275 * BTF type, so just make sure to use it
2276 */
2277 if (glob_sym->btf_id) {
2278 /* reuse existing BTF type for global var/func */
2279 obj->btf_type_map[i] = glob_sym->btf_id;
2280 continue;
2281 }
2282 }
2283
2284 id = btf__add_type(linker->btf, obj->btf, t);
2285 if (id < 0) {
2286 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2287 return id;
2288 }
2289
2290 obj->btf_type_map[i] = id;
2291
2292 /* record just appended BTF type for var/func */
2293 if (glob_sym) {
2294 glob_sym->btf_id = id;
2295 glob_sym->underlying_btf_id = -t->type;
2296 }
2297 }
2298
2299 /* remap all the types except DATASECs */
2300 n = btf__get_nr_types(linker->btf);
2301 for (i = start_id; i <= n; i++) {
2302 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2303
2304 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2305 return -EINVAL;
2306 }
2307
2308 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2309 * actual type), if necessary
2310 */
2311 for (i = 0; i < linker->glob_sym_cnt; i++) {
2312 struct glob_sym *glob_sym = &linker->glob_syms[i];
2313 struct btf_type *glob_t;
2314
2315 if (glob_sym->underlying_btf_id >= 0)
2316 continue;
2317
2318 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2319
2320 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2321 glob_t->type = glob_sym->underlying_btf_id;
2322 }
2323
2324 /* append DATASEC info */
2325 for (i = 1; i < obj->sec_cnt; i++) {
2326 struct src_sec *src_sec;
2327 struct dst_sec *dst_sec;
2328 const struct btf_var_secinfo *src_var;
2329 struct btf_var_secinfo *dst_var;
2330
2331 src_sec = &obj->secs[i];
2332 if (!src_sec->sec_type_id || src_sec->skipped)
2333 continue;
2334 dst_sec = &linker->secs[src_sec->dst_id];
2335
2336 /* Mark section as having BTF regardless of the presence of
2337 * variables. In some cases compiler might generate empty BTF
2338 * with no variables information. E.g., when promoting local
2339 * array/structure variable initial values and BPF object
2340 * file otherwise has no read-only static variables in
2341 * .rodata. We need to preserve such empty BTF and just set
2342 * correct section size.
2343 */
2344 dst_sec->has_btf = true;
2345
2346 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2347 src_var = btf_var_secinfos(t);
2348 n = btf_vlen(t);
2349 for (j = 0; j < n; j++, src_var++) {
2350 void *sec_vars = dst_sec->sec_vars;
2351 int new_id = obj->btf_type_map[src_var->type];
2352 struct glob_sym *glob_sym = NULL;
2353
2354 t = btf_type_by_id(linker->btf, new_id);
2355 if (btf_is_non_static(t)) {
2356 name = btf__str_by_offset(linker->btf, t->name_off);
2357 glob_sym = find_glob_sym(linker, name);
2358 if (glob_sym->sec_id != dst_sec->id) {
2359 pr_warn("global '%s': section mismatch %d vs %d\n",
2360 name, glob_sym->sec_id, dst_sec->id);
2361 return -EINVAL;
2362 }
2363 }
2364
2365 /* If there is already a member (VAR or FUNC) mapped
2366 * to the same type, don't add a duplicate entry.
2367 * This will happen when multiple object files define
2368 * the same extern VARs/FUNCs.
2369 */
2370 if (glob_sym && glob_sym->var_idx >= 0) {
2371 __s64 sz;
2372
2373 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2374 /* Because underlying BTF type might have
2375 * changed, so might its size have changed, so
2376 * re-calculate and update it in sec_var.
2377 */
2378 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2379 if (sz < 0) {
2380 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2381 name, (int)sz);
2382 return -EINVAL;
2383 }
2384 dst_var->size = sz;
2385 continue;
2386 }
2387
2388 sec_vars = libbpf_reallocarray(sec_vars,
2389 dst_sec->sec_var_cnt + 1,
2390 sizeof(*dst_sec->sec_vars));
2391 if (!sec_vars)
2392 return -ENOMEM;
2393
2394 dst_sec->sec_vars = sec_vars;
2395 dst_sec->sec_var_cnt++;
2396
2397 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2398 dst_var->type = obj->btf_type_map[src_var->type];
2399 dst_var->size = src_var->size;
2400 dst_var->offset = src_sec->dst_off + src_var->offset;
2401
2402 if (glob_sym)
2403 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2404 }
2405 }
2406
2407 return 0;
2408}
2409
2410static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2411{
2412 void *tmp;
2413
2414 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2415 if (!tmp)
2416 return NULL;
2417 ext_data->recs = tmp;
2418
2419 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2420 memcpy(tmp, src_rec, ext_data->rec_sz);
2421
2422 ext_data->rec_cnt++;
2423
2424 return tmp;
2425}
2426
2427static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2428{
2429 const struct btf_ext_info_sec *ext_sec;
2430 const char *sec_name, *s;
2431 struct src_sec *src_sec;
2432 struct dst_sec *dst_sec;
2433 int rec_sz, str_off, i;
2434
2435 if (!obj->btf_ext)
2436 return 0;
2437
2438 rec_sz = obj->btf_ext->func_info.rec_size;
2439 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2440 struct bpf_func_info_min *src_rec, *dst_rec;
2441
2442 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2443 src_sec = find_src_sec_by_name(obj, sec_name);
2444 if (!src_sec) {
2445 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2446 return -EINVAL;
2447 }
2448 dst_sec = &linker->secs[src_sec->dst_id];
2449
2450 if (dst_sec->func_info.rec_sz == 0)
2451 dst_sec->func_info.rec_sz = rec_sz;
2452 if (dst_sec->func_info.rec_sz != rec_sz) {
2453 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2454 return -EINVAL;
2455 }
2456
2457 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2458 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2459 if (!dst_rec)
2460 return -ENOMEM;
2461
2462 dst_rec->insn_off += src_sec->dst_off;
2463 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2464 }
2465 }
2466
2467 rec_sz = obj->btf_ext->line_info.rec_size;
2468 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2469 struct bpf_line_info_min *src_rec, *dst_rec;
2470
2471 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2472 src_sec = find_src_sec_by_name(obj, sec_name);
2473 if (!src_sec) {
2474 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2475 return -EINVAL;
2476 }
2477 dst_sec = &linker->secs[src_sec->dst_id];
2478
2479 if (dst_sec->line_info.rec_sz == 0)
2480 dst_sec->line_info.rec_sz = rec_sz;
2481 if (dst_sec->line_info.rec_sz != rec_sz) {
2482 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2483 return -EINVAL;
2484 }
2485
2486 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2487 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2488 if (!dst_rec)
2489 return -ENOMEM;
2490
2491 dst_rec->insn_off += src_sec->dst_off;
2492
2493 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2494 str_off = btf__add_str(linker->btf, s);
2495 if (str_off < 0)
2496 return -ENOMEM;
2497 dst_rec->file_name_off = str_off;
2498
2499 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2500 str_off = btf__add_str(linker->btf, s);
2501 if (str_off < 0)
2502 return -ENOMEM;
2503 dst_rec->line_off = str_off;
2504
2505 /* dst_rec->line_col is fine */
2506 }
2507 }
2508
2509 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2510 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2511 struct bpf_core_relo *src_rec, *dst_rec;
2512
2513 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2514 src_sec = find_src_sec_by_name(obj, sec_name);
2515 if (!src_sec) {
2516 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2517 return -EINVAL;
2518 }
2519 dst_sec = &linker->secs[src_sec->dst_id];
2520
2521 if (dst_sec->core_relo_info.rec_sz == 0)
2522 dst_sec->core_relo_info.rec_sz = rec_sz;
2523 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2524 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2525 return -EINVAL;
2526 }
2527
2528 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2529 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2530 if (!dst_rec)
2531 return -ENOMEM;
2532
2533 dst_rec->insn_off += src_sec->dst_off;
2534 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2535
2536 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2537 str_off = btf__add_str(linker->btf, s);
2538 if (str_off < 0)
2539 return -ENOMEM;
2540 dst_rec->access_str_off = str_off;
2541
2542 /* dst_rec->kind is fine */
2543 }
2544 }
2545
2546 return 0;
2547}
2548
2549int bpf_linker__finalize(struct bpf_linker *linker)
2550{
2551 struct dst_sec *sec;
2552 size_t strs_sz;
2553 const void *strs;
2554 int err, i;
2555
2556 if (!linker->elf)
2557 return libbpf_err(-EINVAL);
2558
2559 err = finalize_btf(linker);
2560 if (err)
2561 return libbpf_err(err);
2562
2563 /* Finalize strings */
2564 strs_sz = strset__data_size(linker->strtab_strs);
2565 strs = strset__data(linker->strtab_strs);
2566
2567 sec = &linker->secs[linker->strtab_sec_idx];
2568 sec->data->d_align = 1;
2569 sec->data->d_off = 0LL;
2570 sec->data->d_buf = (void *)strs;
2571 sec->data->d_type = ELF_T_BYTE;
2572 sec->data->d_size = strs_sz;
2573 sec->shdr->sh_size = strs_sz;
2574
2575 for (i = 1; i < linker->sec_cnt; i++) {
2576 sec = &linker->secs[i];
2577
2578 /* STRTAB is handled specially above */
2579 if (sec->sec_idx == linker->strtab_sec_idx)
2580 continue;
2581
2582 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2583 if (!sec->scn)
2584 continue;
2585
2586 sec->data->d_buf = sec->raw_data;
2587 }
2588
2589 /* Finalize ELF layout */
2590 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2591 err = -errno;
2592 pr_warn_elf("failed to finalize ELF layout");
2593 return libbpf_err(err);
2594 }
2595
2596 /* Write out final ELF contents */
2597 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2598 err = -errno;
2599 pr_warn_elf("failed to write ELF contents");
2600 return libbpf_err(err);
2601 }
2602
2603 elf_end(linker->elf);
2604 close(linker->fd);
2605
2606 linker->elf = NULL;
2607 linker->fd = -1;
2608
2609 return 0;
2610}
2611
2612static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2613 size_t align, const void *raw_data, size_t raw_sz)
2614{
2615 Elf_Scn *scn;
2616 Elf_Data *data;
2617 Elf64_Shdr *shdr;
2618 int name_off;
2619
2620 name_off = strset__add_str(linker->strtab_strs, sec_name);
2621 if (name_off < 0)
2622 return name_off;
2623
2624 scn = elf_newscn(linker->elf);
2625 if (!scn)
2626 return -ENOMEM;
2627 data = elf_newdata(scn);
2628 if (!data)
2629 return -ENOMEM;
2630 shdr = elf64_getshdr(scn);
2631 if (!shdr)
2632 return -EINVAL;
2633
2634 shdr->sh_name = name_off;
2635 shdr->sh_type = SHT_PROGBITS;
2636 shdr->sh_flags = 0;
2637 shdr->sh_size = raw_sz;
2638 shdr->sh_link = 0;
2639 shdr->sh_info = 0;
2640 shdr->sh_addralign = align;
2641 shdr->sh_entsize = 0;
2642
2643 data->d_type = ELF_T_BYTE;
2644 data->d_size = raw_sz;
2645 data->d_buf = (void *)raw_data;
2646 data->d_align = align;
2647 data->d_off = 0;
2648
2649 return 0;
2650}
2651
2652static int finalize_btf(struct bpf_linker *linker)
2653{
2654 struct btf *btf = linker->btf;
2655 const void *raw_data;
2656 int i, j, id, err;
2657 __u32 raw_sz;
2658
2659 /* bail out if no BTF data was produced */
2660 if (btf__get_nr_types(linker->btf) == 0)
2661 return 0;
2662
2663 for (i = 1; i < linker->sec_cnt; i++) {
2664 struct dst_sec *sec = &linker->secs[i];
2665
2666 if (!sec->has_btf)
2667 continue;
2668
2669 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2670 if (id < 0) {
2671 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2672 sec->sec_name, id);
2673 return id;
2674 }
2675
2676 for (j = 0; j < sec->sec_var_cnt; j++) {
2677 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2678
2679 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2680 return -EINVAL;
2681 }
2682 }
2683
2684 err = finalize_btf_ext(linker);
2685 if (err) {
2686 pr_warn(".BTF.ext generation failed: %d\n", err);
2687 return err;
2688 }
2689
2690 err = btf__dedup(linker->btf, linker->btf_ext, NULL);
2691 if (err) {
2692 pr_warn("BTF dedup failed: %d\n", err);
2693 return err;
2694 }
2695
2696 /* Emit .BTF section */
2697 raw_data = btf__get_raw_data(linker->btf, &raw_sz);
2698 if (!raw_data)
2699 return -ENOMEM;
2700
2701 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2702 if (err) {
2703 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2704 return err;
2705 }
2706
2707 /* Emit .BTF.ext section */
2708 if (linker->btf_ext) {
2709 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2710 if (!raw_data)
2711 return -ENOMEM;
2712
2713 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2714 if (err) {
2715 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2716 return err;
2717 }
2718 }
2719
2720 return 0;
2721}
2722
2723static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2724 const char *sec_name, struct btf_ext_sec_data *sec_data)
2725{
2726 struct btf_ext_info_sec *sec_info;
2727 void *cur = output;
2728 int str_off;
2729 size_t sz;
2730
2731 if (!sec_data->rec_cnt)
2732 return 0;
2733
2734 str_off = btf__add_str(linker->btf, sec_name);
2735 if (str_off < 0)
2736 return -ENOMEM;
2737
2738 sec_info = cur;
2739 sec_info->sec_name_off = str_off;
2740 sec_info->num_info = sec_data->rec_cnt;
2741 cur += sizeof(struct btf_ext_info_sec);
2742
2743 sz = sec_data->rec_cnt * sec_data->rec_sz;
2744 memcpy(cur, sec_data->recs, sz);
2745 cur += sz;
2746
2747 return cur - output;
2748}
2749
2750static int finalize_btf_ext(struct bpf_linker *linker)
2751{
2752 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2753 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2754 struct btf_ext_header *hdr;
2755 void *data, *cur;
2756 int i, err, sz;
2757
2758 /* validate that all sections have the same .BTF.ext record sizes
2759 * and calculate total data size for each type of data (func info,
2760 * line info, core relos)
2761 */
2762 for (i = 1; i < linker->sec_cnt; i++) {
2763 struct dst_sec *sec = &linker->secs[i];
2764
2765 if (sec->func_info.rec_cnt) {
2766 if (func_rec_sz == 0)
2767 func_rec_sz = sec->func_info.rec_sz;
2768 if (func_rec_sz != sec->func_info.rec_sz) {
2769 pr_warn("mismatch in func_info record size %zu != %u\n",
2770 func_rec_sz, sec->func_info.rec_sz);
2771 return -EINVAL;
2772 }
2773
2774 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2775 }
2776 if (sec->line_info.rec_cnt) {
2777 if (line_rec_sz == 0)
2778 line_rec_sz = sec->line_info.rec_sz;
2779 if (line_rec_sz != sec->line_info.rec_sz) {
2780 pr_warn("mismatch in line_info record size %zu != %u\n",
2781 line_rec_sz, sec->line_info.rec_sz);
2782 return -EINVAL;
2783 }
2784
2785 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2786 }
2787 if (sec->core_relo_info.rec_cnt) {
2788 if (core_relo_rec_sz == 0)
2789 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2790 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2791 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2792 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2793 return -EINVAL;
2794 }
2795
2796 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2797 }
2798 }
2799
2800 if (!funcs_sz && !lines_sz && !core_relos_sz)
2801 return 0;
2802
2803 total_sz += sizeof(struct btf_ext_header);
2804 if (funcs_sz) {
2805 funcs_sz += sizeof(__u32); /* record size prefix */
2806 total_sz += funcs_sz;
2807 }
2808 if (lines_sz) {
2809 lines_sz += sizeof(__u32); /* record size prefix */
2810 total_sz += lines_sz;
2811 }
2812 if (core_relos_sz) {
2813 core_relos_sz += sizeof(__u32); /* record size prefix */
2814 total_sz += core_relos_sz;
2815 }
2816
2817 cur = data = calloc(1, total_sz);
2818 if (!data)
2819 return -ENOMEM;
2820
2821 hdr = cur;
2822 hdr->magic = BTF_MAGIC;
2823 hdr->version = BTF_VERSION;
2824 hdr->flags = 0;
2825 hdr->hdr_len = sizeof(struct btf_ext_header);
2826 cur += sizeof(struct btf_ext_header);
2827
2828 /* All offsets are in bytes relative to the end of this header */
2829 hdr->func_info_off = 0;
2830 hdr->func_info_len = funcs_sz;
2831 hdr->line_info_off = funcs_sz;
2832 hdr->line_info_len = lines_sz;
2833 hdr->core_relo_off = funcs_sz + lines_sz;
2834 hdr->core_relo_len = core_relos_sz;
2835
2836 if (funcs_sz) {
2837 *(__u32 *)cur = func_rec_sz;
2838 cur += sizeof(__u32);
2839
2840 for (i = 1; i < linker->sec_cnt; i++) {
2841 struct dst_sec *sec = &linker->secs[i];
2842
2843 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2844 if (sz < 0) {
2845 err = sz;
2846 goto out;
2847 }
2848
2849 cur += sz;
2850 }
2851 }
2852
2853 if (lines_sz) {
2854 *(__u32 *)cur = line_rec_sz;
2855 cur += sizeof(__u32);
2856
2857 for (i = 1; i < linker->sec_cnt; i++) {
2858 struct dst_sec *sec = &linker->secs[i];
2859
2860 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2861 if (sz < 0) {
2862 err = sz;
2863 goto out;
2864 }
2865
2866 cur += sz;
2867 }
2868 }
2869
2870 if (core_relos_sz) {
2871 *(__u32 *)cur = core_relo_rec_sz;
2872 cur += sizeof(__u32);
2873
2874 for (i = 1; i < linker->sec_cnt; i++) {
2875 struct dst_sec *sec = &linker->secs[i];
2876
2877 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2878 if (sz < 0) {
2879 err = sz;
2880 goto out;
2881 }
2882
2883 cur += sz;
2884 }
2885 }
2886
2887 linker->btf_ext = btf_ext__new(data, total_sz);
2888 err = libbpf_get_error(linker->btf_ext);
2889 if (err) {
2890 linker->btf_ext = NULL;
2891 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2892 goto out;
2893 }
2894
2895out:
2896 free(data);
2897 return err;
2898}