Loading...
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <errno.h>
14#include <linux/err.h>
15#include <linux/btf.h>
16#include <elf.h>
17#include <libelf.h>
18#include <fcntl.h>
19#include "libbpf.h"
20#include "btf.h"
21#include "libbpf_internal.h"
22#include "strset.h"
23
24#define BTF_EXTERN_SEC ".extern"
25
26struct src_sec {
27 const char *sec_name;
28 /* positional (not necessarily ELF) index in an array of sections */
29 int id;
30 /* positional (not necessarily ELF) index of a matching section in a final object file */
31 int dst_id;
32 /* section data offset in a matching output section */
33 int dst_off;
34 /* whether section is omitted from the final ELF file */
35 bool skipped;
36 /* whether section is an ephemeral section, not mapped to an ELF section */
37 bool ephemeral;
38
39 /* ELF info */
40 size_t sec_idx;
41 Elf_Scn *scn;
42 Elf64_Shdr *shdr;
43 Elf_Data *data;
44
45 /* corresponding BTF DATASEC type ID */
46 int sec_type_id;
47};
48
49struct src_obj {
50 const char *filename;
51 int fd;
52 Elf *elf;
53 /* Section header strings section index */
54 size_t shstrs_sec_idx;
55 /* SYMTAB section index */
56 size_t symtab_sec_idx;
57
58 struct btf *btf;
59 struct btf_ext *btf_ext;
60
61 /* List of sections (including ephemeral). Slot zero is unused. */
62 struct src_sec *secs;
63 int sec_cnt;
64
65 /* mapping of symbol indices from src to dst ELF */
66 int *sym_map;
67 /* mapping from the src BTF type IDs to dst ones */
68 int *btf_type_map;
69};
70
71/* single .BTF.ext data section */
72struct btf_ext_sec_data {
73 size_t rec_cnt;
74 __u32 rec_sz;
75 void *recs;
76};
77
78struct glob_sym {
79 /* ELF symbol index */
80 int sym_idx;
81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
82 int sec_id;
83 /* extern name offset in STRTAB */
84 int name_off;
85 /* optional associated BTF type ID */
86 int btf_id;
87 /* BTF type ID to which VAR/FUNC type is pointing to; used for
88 * rewriting types when extern VAR/FUNC is resolved to a concrete
89 * definition
90 */
91 int underlying_btf_id;
92 /* sec_var index in the corresponding dst_sec, if exists */
93 int var_idx;
94
95 /* extern or resolved/global symbol */
96 bool is_extern;
97 /* weak or strong symbol, never goes back from strong to weak */
98 bool is_weak;
99};
100
101struct dst_sec {
102 char *sec_name;
103 /* positional (not necessarily ELF) index in an array of sections */
104 int id;
105
106 bool ephemeral;
107
108 /* ELF info */
109 size_t sec_idx;
110 Elf_Scn *scn;
111 Elf64_Shdr *shdr;
112 Elf_Data *data;
113
114 /* final output section size */
115 int sec_sz;
116 /* final output contents of the section */
117 void *raw_data;
118
119 /* corresponding STT_SECTION symbol index in SYMTAB */
120 int sec_sym_idx;
121
122 /* section's DATASEC variable info, emitted on BTF finalization */
123 bool has_btf;
124 int sec_var_cnt;
125 struct btf_var_secinfo *sec_vars;
126
127 /* section's .BTF.ext data */
128 struct btf_ext_sec_data func_info;
129 struct btf_ext_sec_data line_info;
130 struct btf_ext_sec_data core_relo_info;
131};
132
133struct bpf_linker {
134 char *filename;
135 int fd;
136 Elf *elf;
137 Elf64_Ehdr *elf_hdr;
138
139 /* Output sections metadata */
140 struct dst_sec *secs;
141 int sec_cnt;
142
143 struct strset *strtab_strs; /* STRTAB unique strings */
144 size_t strtab_sec_idx; /* STRTAB section index */
145 size_t symtab_sec_idx; /* SYMTAB section index */
146
147 struct btf *btf;
148 struct btf_ext *btf_ext;
149
150 /* global (including extern) ELF symbols */
151 int glob_sym_cnt;
152 struct glob_sym *glob_syms;
153};
154
155#define pr_warn_elf(fmt, ...) \
156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
157
158static int init_output_elf(struct bpf_linker *linker, const char *file);
159
160static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
161 const struct bpf_linker_file_opts *opts,
162 struct src_obj *obj);
163static int linker_sanity_check_elf(struct src_obj *obj);
164static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
165static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
166static int linker_sanity_check_btf(struct src_obj *obj);
167static int linker_sanity_check_btf_ext(struct src_obj *obj);
168static int linker_fixup_btf(struct src_obj *obj);
169static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
170static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
171static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
172 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
173static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
174static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
175static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
176
177static int finalize_btf(struct bpf_linker *linker);
178static int finalize_btf_ext(struct bpf_linker *linker);
179
180void bpf_linker__free(struct bpf_linker *linker)
181{
182 int i;
183
184 if (!linker)
185 return;
186
187 free(linker->filename);
188
189 if (linker->elf)
190 elf_end(linker->elf);
191
192 if (linker->fd >= 0)
193 close(linker->fd);
194
195 strset__free(linker->strtab_strs);
196
197 btf__free(linker->btf);
198 btf_ext__free(linker->btf_ext);
199
200 for (i = 1; i < linker->sec_cnt; i++) {
201 struct dst_sec *sec = &linker->secs[i];
202
203 free(sec->sec_name);
204 free(sec->raw_data);
205 free(sec->sec_vars);
206
207 free(sec->func_info.recs);
208 free(sec->line_info.recs);
209 free(sec->core_relo_info.recs);
210 }
211 free(linker->secs);
212
213 free(linker->glob_syms);
214 free(linker);
215}
216
217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218{
219 struct bpf_linker *linker;
220 int err;
221
222 if (!OPTS_VALID(opts, bpf_linker_opts))
223 return errno = EINVAL, NULL;
224
225 if (elf_version(EV_CURRENT) == EV_NONE) {
226 pr_warn_elf("libelf initialization failed");
227 return errno = EINVAL, NULL;
228 }
229
230 linker = calloc(1, sizeof(*linker));
231 if (!linker)
232 return errno = ENOMEM, NULL;
233
234 linker->fd = -1;
235
236 err = init_output_elf(linker, filename);
237 if (err)
238 goto err_out;
239
240 return linker;
241
242err_out:
243 bpf_linker__free(linker);
244 return errno = -err, NULL;
245}
246
247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248{
249 struct dst_sec *secs = linker->secs, *sec;
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251
252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 if (!secs)
254 return NULL;
255
256 /* zero out newly allocated memory */
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258
259 linker->secs = secs;
260 linker->sec_cnt = new_cnt;
261
262 sec = &linker->secs[new_cnt - 1];
263 sec->id = new_cnt - 1;
264 sec->sec_name = strdup(sec_name);
265 if (!sec->sec_name)
266 return NULL;
267
268 return sec;
269}
270
271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272{
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 Elf64_Sym *syms, *sym;
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 if (!syms)
279 return NULL;
280
281 sym = &syms[sym_cnt];
282 memset(sym, 0, sizeof(*sym));
283
284 symtab->raw_data = syms;
285 symtab->sec_sz += sizeof(*sym);
286 symtab->shdr->sh_size += sizeof(*sym);
287 symtab->data->d_size += sizeof(*sym);
288
289 if (sym_idx)
290 *sym_idx = sym_cnt;
291
292 return sym;
293}
294
295static int init_output_elf(struct bpf_linker *linker, const char *file)
296{
297 int err, str_off;
298 Elf64_Sym *init_sym;
299 struct dst_sec *sec;
300
301 linker->filename = strdup(file);
302 if (!linker->filename)
303 return -ENOMEM;
304
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306 if (linker->fd < 0) {
307 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);
309 return err;
310 }
311
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 if (!linker->elf) {
314 pr_warn_elf("failed to create ELF object");
315 return -EINVAL;
316 }
317
318 /* ELF header */
319 linker->elf_hdr = elf64_newehdr(linker->elf);
320 if (!linker->elf_hdr) {
321 pr_warn_elf("failed to create ELF header");
322 return -EINVAL;
323 }
324
325 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;
327#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331#else
332#error "Unknown __BYTE_ORDER__"
333#endif
334
335 /* STRTAB */
336 /* initialize strset with an empty string to conform to ELF */
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 if (libbpf_get_error(linker->strtab_strs))
339 return libbpf_get_error(linker->strtab_strs);
340
341 sec = add_dst_sec(linker, ".strtab");
342 if (!sec)
343 return -ENOMEM;
344
345 sec->scn = elf_newscn(linker->elf);
346 if (!sec->scn) {
347 pr_warn_elf("failed to create STRTAB section");
348 return -EINVAL;
349 }
350
351 sec->shdr = elf64_getshdr(sec->scn);
352 if (!sec->shdr)
353 return -EINVAL;
354
355 sec->data = elf_newdata(sec->scn);
356 if (!sec->data) {
357 pr_warn_elf("failed to create STRTAB data");
358 return -EINVAL;
359 }
360
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 if (str_off < 0)
363 return str_off;
364
365 sec->sec_idx = elf_ndxscn(sec->scn);
366 linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 linker->strtab_sec_idx = sec->sec_idx;
368
369 sec->shdr->sh_name = str_off;
370 sec->shdr->sh_type = SHT_STRTAB;
371 sec->shdr->sh_flags = SHF_STRINGS;
372 sec->shdr->sh_offset = 0;
373 sec->shdr->sh_link = 0;
374 sec->shdr->sh_info = 0;
375 sec->shdr->sh_addralign = 1;
376 sec->shdr->sh_size = sec->sec_sz = 0;
377 sec->shdr->sh_entsize = 0;
378
379 /* SYMTAB */
380 sec = add_dst_sec(linker, ".symtab");
381 if (!sec)
382 return -ENOMEM;
383
384 sec->scn = elf_newscn(linker->elf);
385 if (!sec->scn) {
386 pr_warn_elf("failed to create SYMTAB section");
387 return -EINVAL;
388 }
389
390 sec->shdr = elf64_getshdr(sec->scn);
391 if (!sec->shdr)
392 return -EINVAL;
393
394 sec->data = elf_newdata(sec->scn);
395 if (!sec->data) {
396 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;
398 }
399
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)
402 return str_off;
403
404 sec->sec_idx = elf_ndxscn(sec->scn);
405 linker->symtab_sec_idx = sec->sec_idx;
406
407 sec->shdr->sh_name = str_off;
408 sec->shdr->sh_type = SHT_SYMTAB;
409 sec->shdr->sh_flags = 0;
410 sec->shdr->sh_offset = 0;
411 sec->shdr->sh_link = linker->strtab_sec_idx;
412 /* sh_info should be one greater than the index of the last local
413 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
414 */
415 sec->shdr->sh_info = 0;
416 sec->shdr->sh_addralign = 8;
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418
419 /* .BTF */
420 linker->btf = btf__new_empty();
421 err = libbpf_get_error(linker->btf);
422 if (err)
423 return err;
424
425 /* add the special all-zero symbol */
426 init_sym = add_new_sym(linker, NULL);
427 if (!init_sym)
428 return -EINVAL;
429
430 init_sym->st_name = 0;
431 init_sym->st_info = 0;
432 init_sym->st_other = 0;
433 init_sym->st_shndx = SHN_UNDEF;
434 init_sym->st_value = 0;
435 init_sym->st_size = 0;
436
437 return 0;
438}
439
440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 const struct bpf_linker_file_opts *opts)
442{
443 struct src_obj obj = {};
444 int err = 0;
445
446 if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 return libbpf_err(-EINVAL);
448
449 if (!linker->elf)
450 return libbpf_err(-EINVAL);
451
452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 err = err ?: linker_append_sec_data(linker, &obj);
454 err = err ?: linker_append_elf_syms(linker, &obj);
455 err = err ?: linker_append_elf_relos(linker, &obj);
456 err = err ?: linker_append_btf(linker, &obj);
457 err = err ?: linker_append_btf_ext(linker, &obj);
458
459 /* free up src_obj resources */
460 free(obj.btf_type_map);
461 btf__free(obj.btf);
462 btf_ext__free(obj.btf_ext);
463 free(obj.secs);
464 free(obj.sym_map);
465 if (obj.elf)
466 elf_end(obj.elf);
467 if (obj.fd >= 0)
468 close(obj.fd);
469
470 return libbpf_err(err);
471}
472
473static bool is_dwarf_sec_name(const char *name)
474{
475 /* approximation, but the actual list is too long */
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477}
478
479static bool is_ignored_sec(struct src_sec *sec)
480{
481 Elf64_Shdr *shdr = sec->shdr;
482 const char *name = sec->sec_name;
483
484 /* no special handling of .strtab */
485 if (shdr->sh_type == SHT_STRTAB)
486 return true;
487
488 /* ignore .llvm_addrsig section as well */
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 return true;
491
492 /* no subprograms will lead to an empty .text section, ignore it */
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 strcmp(sec->sec_name, ".text") == 0)
495 return true;
496
497 /* DWARF sections */
498 if (is_dwarf_sec_name(sec->sec_name))
499 return true;
500
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 name += sizeof(".rel") - 1;
503 /* DWARF section relocations */
504 if (is_dwarf_sec_name(name))
505 return true;
506
507 /* .BTF and .BTF.ext don't need relocations */
508 if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 return true;
511 }
512
513 return false;
514}
515
516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517{
518 struct src_sec *secs = obj->secs, *sec;
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520
521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 if (!secs)
523 return NULL;
524
525 /* zero out newly allocated memory */
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527
528 obj->secs = secs;
529 obj->sec_cnt = new_cnt;
530
531 sec = &obj->secs[new_cnt - 1];
532 sec->id = new_cnt - 1;
533 sec->sec_name = sec_name;
534
535 return sec;
536}
537
538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)
541{
542#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543 const int host_endianness = ELFDATA2LSB;
544#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545 const int host_endianness = ELFDATA2MSB;
546#else
547#error "Unknown __BYTE_ORDER__"
548#endif
549 int err = 0;
550 Elf_Scn *scn;
551 Elf_Data *data;
552 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;
554 struct src_sec *sec;
555
556 pr_debug("linker: adding object file '%s'...\n", filename);
557
558 obj->filename = filename;
559
560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561 if (obj->fd < 0) {
562 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);
564 return err;
565 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 if (!obj->elf) {
568 err = -errno;
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return err;
571 }
572
573 /* Sanity check ELF file high-level properties */
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 err = -errno;
577 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;
579 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 return err;
584 }
585 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 err = -EOPNOTSUPP;
589 pr_warn_elf("unsupported kind of ELF file %s", filename);
590 return err;
591 }
592
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 err = -errno;
595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 return err;
597 }
598
599 scn = NULL;
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 size_t sec_idx = elf_ndxscn(scn);
602 const char *sec_name;
603
604 shdr = elf64_getshdr(scn);
605 if (!shdr) {
606 err = -errno;
607 pr_warn_elf("failed to get section #%zu header for %s",
608 sec_idx, filename);
609 return err;
610 }
611
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 if (!sec_name) {
614 err = -errno;
615 pr_warn_elf("failed to get section #%zu name for %s",
616 sec_idx, filename);
617 return err;
618 }
619
620 data = elf_getdata(scn, 0);
621 if (!data) {
622 err = -errno;
623 pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 sec_idx, sec_name, filename);
625 return err;
626 }
627
628 sec = add_src_sec(obj, sec_name);
629 if (!sec)
630 return -ENOMEM;
631
632 sec->scn = scn;
633 sec->shdr = shdr;
634 sec->data = data;
635 sec->sec_idx = elf_ndxscn(scn);
636
637 if (is_ignored_sec(sec)) {
638 sec->skipped = true;
639 continue;
640 }
641
642 switch (shdr->sh_type) {
643 case SHT_SYMTAB:
644 if (obj->symtab_sec_idx) {
645 err = -EOPNOTSUPP;
646 pr_warn("multiple SYMTAB sections found, not supported\n");
647 return err;
648 }
649 obj->symtab_sec_idx = sec_idx;
650 break;
651 case SHT_STRTAB:
652 /* we'll construct our own string table */
653 break;
654 case SHT_PROGBITS:
655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);
658 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 return err;
661 }
662 sec->skipped = true;
663 continue;
664 }
665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);
668 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675
676 /* data & code */
677 break;
678 case SHT_NOBITS:
679 /* BSS */
680 break;
681 case SHT_REL:
682 /* relocations */
683 break;
684 default:
685 pr_warn("unrecognized section #%zu (%s) in %s\n",
686 sec_idx, sec_name, filename);
687 err = -EINVAL;
688 return err;
689 }
690 }
691
692 err = err ?: linker_sanity_check_elf(obj);
693 err = err ?: linker_sanity_check_btf(obj);
694 err = err ?: linker_sanity_check_btf_ext(obj);
695 err = err ?: linker_fixup_btf(obj);
696
697 return err;
698}
699
700static int linker_sanity_check_elf(struct src_obj *obj)
701{
702 struct src_sec *sec;
703 int i, err;
704
705 if (!obj->symtab_sec_idx) {
706 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
707 return -EINVAL;
708 }
709 if (!obj->shstrs_sec_idx) {
710 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
711 return -EINVAL;
712 }
713
714 for (i = 1; i < obj->sec_cnt; i++) {
715 sec = &obj->secs[i];
716
717 if (sec->sec_name[0] == '\0') {
718 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
719 return -EINVAL;
720 }
721
722 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
723 return -EINVAL;
724 if (sec->shdr->sh_addralign != sec->data->d_align)
725 return -EINVAL;
726
727 if (sec->shdr->sh_size != sec->data->d_size)
728 return -EINVAL;
729
730 switch (sec->shdr->sh_type) {
731 case SHT_SYMTAB:
732 err = linker_sanity_check_elf_symtab(obj, sec);
733 if (err)
734 return err;
735 break;
736 case SHT_STRTAB:
737 break;
738 case SHT_PROGBITS:
739 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
740 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
741 return -EINVAL;
742 }
743 break;
744 case SHT_NOBITS:
745 break;
746 case SHT_REL:
747 err = linker_sanity_check_elf_relos(obj, sec);
748 if (err)
749 return err;
750 break;
751 case SHT_LLVM_ADDRSIG:
752 break;
753 default:
754 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
755 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
756 return -EINVAL;
757 }
758 }
759
760 return 0;
761}
762
763static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
764{
765 struct src_sec *link_sec;
766 Elf64_Sym *sym;
767 int i, n;
768
769 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
770 return -EINVAL;
771 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
772 return -EINVAL;
773
774 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
775 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
776 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
777 return -EINVAL;
778 }
779 link_sec = &obj->secs[sec->shdr->sh_link];
780 if (link_sec->shdr->sh_type != SHT_STRTAB) {
781 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
782 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
783 return -EINVAL;
784 }
785
786 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
787 sym = sec->data->d_buf;
788 for (i = 0; i < n; i++, sym++) {
789 int sym_type = ELF64_ST_TYPE(sym->st_info);
790 int sym_bind = ELF64_ST_BIND(sym->st_info);
791 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
792
793 if (i == 0) {
794 if (sym->st_name != 0 || sym->st_info != 0
795 || sym->st_other != 0 || sym->st_shndx != 0
796 || sym->st_value != 0 || sym->st_size != 0) {
797 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
798 return -EINVAL;
799 }
800 continue;
801 }
802 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
803 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
804 i, sec->sec_idx, sym_bind);
805 return -EINVAL;
806 }
807 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
808 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
809 i, sec->sec_idx, sym_vis);
810 return -EINVAL;
811 }
812 if (sym->st_shndx == 0) {
813 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
814 || sym->st_value != 0 || sym->st_size != 0) {
815 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
816 i, obj->filename);
817
818 return -EINVAL;
819 }
820 continue;
821 }
822 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
823 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
824 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
825 return -EINVAL;
826 }
827 if (sym_type == STT_SECTION) {
828 if (sym->st_value != 0)
829 return -EINVAL;
830 continue;
831 }
832 }
833
834 return 0;
835}
836
837static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
838{
839 struct src_sec *link_sec, *sym_sec;
840 Elf64_Rel *relo;
841 int i, n;
842
843 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
844 return -EINVAL;
845 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
846 return -EINVAL;
847
848 /* SHT_REL's sh_link should point to SYMTAB */
849 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
850 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
851 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
852 return -EINVAL;
853 }
854
855 /* SHT_REL's sh_info points to relocated section */
856 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
857 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
858 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
859 return -EINVAL;
860 }
861 link_sec = &obj->secs[sec->shdr->sh_info];
862
863 /* .rel<secname> -> <secname> pattern is followed */
864 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
865 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
866 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
867 sec->sec_idx, obj->filename);
868 return -EINVAL;
869 }
870
871 /* don't further validate relocations for ignored sections */
872 if (link_sec->skipped)
873 return 0;
874
875 /* relocatable section is data or instructions */
876 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
877 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
878 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
879 return -EINVAL;
880 }
881
882 /* check sanity of each relocation */
883 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
884 relo = sec->data->d_buf;
885 sym_sec = &obj->secs[obj->symtab_sec_idx];
886 for (i = 0; i < n; i++, relo++) {
887 size_t sym_idx = ELF64_R_SYM(relo->r_info);
888 size_t sym_type = ELF64_R_TYPE(relo->r_info);
889
890 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
891 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
892 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
893 i, sec->sec_idx, sym_type, obj->filename);
894 return -EINVAL;
895 }
896
897 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
898 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
899 i, sec->sec_idx, sym_idx, obj->filename);
900 return -EINVAL;
901 }
902
903 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
904 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
905 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
906 i, sec->sec_idx, sym_idx, obj->filename);
907 return -EINVAL;
908 }
909 }
910 }
911
912 return 0;
913}
914
915static int check_btf_type_id(__u32 *type_id, void *ctx)
916{
917 struct btf *btf = ctx;
918
919 if (*type_id >= btf__type_cnt(btf))
920 return -EINVAL;
921
922 return 0;
923}
924
925static int check_btf_str_off(__u32 *str_off, void *ctx)
926{
927 struct btf *btf = ctx;
928 const char *s;
929
930 s = btf__str_by_offset(btf, *str_off);
931
932 if (!s)
933 return -EINVAL;
934
935 return 0;
936}
937
938static int linker_sanity_check_btf(struct src_obj *obj)
939{
940 struct btf_type *t;
941 int i, n, err = 0;
942
943 if (!obj->btf)
944 return 0;
945
946 n = btf__type_cnt(obj->btf);
947 for (i = 1; i < n; i++) {
948 t = btf_type_by_id(obj->btf, i);
949
950 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
951 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
952 if (err)
953 return err;
954 }
955
956 return 0;
957}
958
959static int linker_sanity_check_btf_ext(struct src_obj *obj)
960{
961 int err = 0;
962
963 if (!obj->btf_ext)
964 return 0;
965
966 /* can't use .BTF.ext without .BTF */
967 if (!obj->btf)
968 return -EINVAL;
969
970 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
971 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
972 if (err)
973 return err;
974
975 return 0;
976}
977
978static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
979{
980 Elf_Scn *scn;
981 Elf_Data *data;
982 Elf64_Shdr *shdr;
983 int name_off;
984
985 dst_sec->sec_sz = 0;
986 dst_sec->sec_idx = 0;
987 dst_sec->ephemeral = src_sec->ephemeral;
988
989 /* ephemeral sections are just thin section shells lacking most parts */
990 if (src_sec->ephemeral)
991 return 0;
992
993 scn = elf_newscn(linker->elf);
994 if (!scn)
995 return -ENOMEM;
996 data = elf_newdata(scn);
997 if (!data)
998 return -ENOMEM;
999 shdr = elf64_getshdr(scn);
1000 if (!shdr)
1001 return -ENOMEM;
1002
1003 dst_sec->scn = scn;
1004 dst_sec->shdr = shdr;
1005 dst_sec->data = data;
1006 dst_sec->sec_idx = elf_ndxscn(scn);
1007
1008 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1009 if (name_off < 0)
1010 return name_off;
1011
1012 shdr->sh_name = name_off;
1013 shdr->sh_type = src_sec->shdr->sh_type;
1014 shdr->sh_flags = src_sec->shdr->sh_flags;
1015 shdr->sh_size = 0;
1016 /* sh_link and sh_info have different meaning for different types of
1017 * sections, so we leave it up to the caller code to fill them in, if
1018 * necessary
1019 */
1020 shdr->sh_link = 0;
1021 shdr->sh_info = 0;
1022 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1023 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1024
1025 data->d_type = src_sec->data->d_type;
1026 data->d_size = 0;
1027 data->d_buf = NULL;
1028 data->d_align = src_sec->data->d_align;
1029 data->d_off = 0;
1030
1031 return 0;
1032}
1033
1034static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1035{
1036 struct dst_sec *sec;
1037 int i;
1038
1039 for (i = 1; i < linker->sec_cnt; i++) {
1040 sec = &linker->secs[i];
1041
1042 if (strcmp(sec->sec_name, sec_name) == 0)
1043 return sec;
1044 }
1045
1046 return NULL;
1047}
1048
1049static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1050{
1051 if (dst->ephemeral || src->ephemeral)
1052 return true;
1053
1054 if (dst->shdr->sh_type != src->shdr->sh_type) {
1055 pr_warn("sec %s types mismatch\n", dst->sec_name);
1056 return false;
1057 }
1058 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1059 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1060 return false;
1061 }
1062 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1063 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1064 return false;
1065 }
1066
1067 return true;
1068}
1069
1070static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1071{
1072 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1073 return false;
1074 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1075 return false;
1076 return true;
1077}
1078
1079static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1080{
1081 void *tmp;
1082 size_t dst_align, src_align;
1083 size_t dst_align_sz, dst_final_sz;
1084 int err;
1085
1086 /* Ephemeral source section doesn't contribute anything to ELF
1087 * section data.
1088 */
1089 if (src->ephemeral)
1090 return 0;
1091
1092 /* Some sections (like .maps) can contain both externs (and thus be
1093 * ephemeral) and non-externs (map definitions). So it's possible that
1094 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1095 * first non-ephemeral entity appears. In such case, we add ELF
1096 * section, data, etc.
1097 */
1098 if (dst->ephemeral) {
1099 err = init_sec(linker, dst, src);
1100 if (err)
1101 return err;
1102 }
1103
1104 dst_align = dst->shdr->sh_addralign;
1105 src_align = src->shdr->sh_addralign;
1106 if (dst_align == 0)
1107 dst_align = 1;
1108 if (dst_align < src_align)
1109 dst_align = src_align;
1110
1111 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1112
1113 /* no need to re-align final size */
1114 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1115
1116 if (src->shdr->sh_type != SHT_NOBITS) {
1117 tmp = realloc(dst->raw_data, dst_final_sz);
1118 if (!tmp)
1119 return -ENOMEM;
1120 dst->raw_data = tmp;
1121
1122 /* pad dst section, if it's alignment forced size increase */
1123 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1124 /* now copy src data at a properly aligned offset */
1125 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1126 }
1127
1128 dst->sec_sz = dst_final_sz;
1129 dst->shdr->sh_size = dst_final_sz;
1130 dst->data->d_size = dst_final_sz;
1131
1132 dst->shdr->sh_addralign = dst_align;
1133 dst->data->d_align = dst_align;
1134
1135 src->dst_off = dst_align_sz;
1136
1137 return 0;
1138}
1139
1140static bool is_data_sec(struct src_sec *sec)
1141{
1142 if (!sec || sec->skipped)
1143 return false;
1144 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1145 if (sec->ephemeral)
1146 return true;
1147 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1148}
1149
1150static bool is_relo_sec(struct src_sec *sec)
1151{
1152 if (!sec || sec->skipped || sec->ephemeral)
1153 return false;
1154 return sec->shdr->sh_type == SHT_REL;
1155}
1156
1157static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1158{
1159 int i, err;
1160
1161 for (i = 1; i < obj->sec_cnt; i++) {
1162 struct src_sec *src_sec;
1163 struct dst_sec *dst_sec;
1164
1165 src_sec = &obj->secs[i];
1166 if (!is_data_sec(src_sec))
1167 continue;
1168
1169 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1170 if (!dst_sec) {
1171 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1172 if (!dst_sec)
1173 return -ENOMEM;
1174 err = init_sec(linker, dst_sec, src_sec);
1175 if (err) {
1176 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1177 return err;
1178 }
1179 } else {
1180 if (!secs_match(dst_sec, src_sec)) {
1181 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1182 return -1;
1183 }
1184
1185 /* "license" and "version" sections are deduped */
1186 if (strcmp(src_sec->sec_name, "license") == 0
1187 || strcmp(src_sec->sec_name, "version") == 0) {
1188 if (!sec_content_is_same(dst_sec, src_sec)) {
1189 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1190 return -EINVAL;
1191 }
1192 src_sec->skipped = true;
1193 src_sec->dst_id = dst_sec->id;
1194 continue;
1195 }
1196 }
1197
1198 /* record mapped section index */
1199 src_sec->dst_id = dst_sec->id;
1200
1201 err = extend_sec(linker, dst_sec, src_sec);
1202 if (err)
1203 return err;
1204 }
1205
1206 return 0;
1207}
1208
1209static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1210{
1211 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1212 Elf64_Sym *sym = symtab->data->d_buf;
1213 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1214 int str_sec_idx = symtab->shdr->sh_link;
1215 const char *sym_name;
1216
1217 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1218 if (!obj->sym_map)
1219 return -ENOMEM;
1220
1221 for (i = 0; i < n; i++, sym++) {
1222 /* We already validated all-zero symbol #0 and we already
1223 * appended it preventively to the final SYMTAB, so skip it.
1224 */
1225 if (i == 0)
1226 continue;
1227
1228 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1229 if (!sym_name) {
1230 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1231 return -EINVAL;
1232 }
1233
1234 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1235 if (err)
1236 return err;
1237 }
1238
1239 return 0;
1240}
1241
1242static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1243{
1244 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1245 Elf64_Sym *syms = symtab->raw_data;
1246
1247 return &syms[sym_idx];
1248}
1249
1250static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1251{
1252 struct glob_sym *glob_sym;
1253 const char *name;
1254 int i;
1255
1256 for (i = 0; i < linker->glob_sym_cnt; i++) {
1257 glob_sym = &linker->glob_syms[i];
1258 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1259
1260 if (strcmp(name, sym_name) == 0)
1261 return glob_sym;
1262 }
1263
1264 return NULL;
1265}
1266
1267static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1268{
1269 struct glob_sym *syms, *sym;
1270
1271 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1272 sizeof(*linker->glob_syms));
1273 if (!syms)
1274 return NULL;
1275
1276 sym = &syms[linker->glob_sym_cnt];
1277 memset(sym, 0, sizeof(*sym));
1278 sym->var_idx = -1;
1279
1280 linker->glob_syms = syms;
1281 linker->glob_sym_cnt++;
1282
1283 return sym;
1284}
1285
1286static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1287 const struct btf *btf1, __u32 id1,
1288 const struct btf *btf2, __u32 id2)
1289{
1290 const struct btf_type *t1, *t2;
1291 bool is_static1, is_static2;
1292 const char *n1, *n2;
1293 int i, n;
1294
1295recur:
1296 n1 = n2 = NULL;
1297 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1298 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1299
1300 /* check if only one side is FWD, otherwise handle with common logic */
1301 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1302 n1 = btf__str_by_offset(btf1, t1->name_off);
1303 n2 = btf__str_by_offset(btf2, t2->name_off);
1304 if (strcmp(n1, n2) != 0) {
1305 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1306 sym_name, n1, n2);
1307 return false;
1308 }
1309 /* validate if FWD kind matches concrete kind */
1310 if (btf_is_fwd(t1)) {
1311 if (btf_kflag(t1) && btf_is_union(t2))
1312 return true;
1313 if (!btf_kflag(t1) && btf_is_struct(t2))
1314 return true;
1315 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1316 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1317 } else {
1318 if (btf_kflag(t2) && btf_is_union(t1))
1319 return true;
1320 if (!btf_kflag(t2) && btf_is_struct(t1))
1321 return true;
1322 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1323 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1324 }
1325 return false;
1326 }
1327
1328 if (btf_kind(t1) != btf_kind(t2)) {
1329 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1330 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1331 return false;
1332 }
1333
1334 switch (btf_kind(t1)) {
1335 case BTF_KIND_STRUCT:
1336 case BTF_KIND_UNION:
1337 case BTF_KIND_ENUM:
1338 case BTF_KIND_ENUM64:
1339 case BTF_KIND_FWD:
1340 case BTF_KIND_FUNC:
1341 case BTF_KIND_VAR:
1342 n1 = btf__str_by_offset(btf1, t1->name_off);
1343 n2 = btf__str_by_offset(btf2, t2->name_off);
1344 if (strcmp(n1, n2) != 0) {
1345 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1346 sym_name, btf_kind_str(t1), n1, n2);
1347 return false;
1348 }
1349 break;
1350 default:
1351 break;
1352 }
1353
1354 switch (btf_kind(t1)) {
1355 case BTF_KIND_UNKN: /* void */
1356 case BTF_KIND_FWD:
1357 return true;
1358 case BTF_KIND_INT:
1359 case BTF_KIND_FLOAT:
1360 case BTF_KIND_ENUM:
1361 case BTF_KIND_ENUM64:
1362 /* ignore encoding for int and enum values for enum */
1363 if (t1->size != t2->size) {
1364 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1365 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1366 return false;
1367 }
1368 return true;
1369 case BTF_KIND_PTR:
1370 /* just validate overall shape of the referenced type, so no
1371 * contents comparison for struct/union, and allowd fwd vs
1372 * struct/union
1373 */
1374 exact = false;
1375 id1 = t1->type;
1376 id2 = t2->type;
1377 goto recur;
1378 case BTF_KIND_ARRAY:
1379 /* ignore index type and array size */
1380 id1 = btf_array(t1)->type;
1381 id2 = btf_array(t2)->type;
1382 goto recur;
1383 case BTF_KIND_FUNC:
1384 /* extern and global linkages are compatible */
1385 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1386 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1387 if (is_static1 != is_static2) {
1388 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1389 return false;
1390 }
1391
1392 id1 = t1->type;
1393 id2 = t2->type;
1394 goto recur;
1395 case BTF_KIND_VAR:
1396 /* extern and global linkages are compatible */
1397 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1398 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1399 if (is_static1 != is_static2) {
1400 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1401 return false;
1402 }
1403
1404 id1 = t1->type;
1405 id2 = t2->type;
1406 goto recur;
1407 case BTF_KIND_STRUCT:
1408 case BTF_KIND_UNION: {
1409 const struct btf_member *m1, *m2;
1410
1411 if (!exact)
1412 return true;
1413
1414 if (btf_vlen(t1) != btf_vlen(t2)) {
1415 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1416 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1417 return false;
1418 }
1419
1420 n = btf_vlen(t1);
1421 m1 = btf_members(t1);
1422 m2 = btf_members(t2);
1423 for (i = 0; i < n; i++, m1++, m2++) {
1424 n1 = btf__str_by_offset(btf1, m1->name_off);
1425 n2 = btf__str_by_offset(btf2, m2->name_off);
1426 if (strcmp(n1, n2) != 0) {
1427 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1428 sym_name, i, n1, n2);
1429 return false;
1430 }
1431 if (m1->offset != m2->offset) {
1432 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1433 sym_name, i, n1);
1434 return false;
1435 }
1436 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1437 return false;
1438 }
1439
1440 return true;
1441 }
1442 case BTF_KIND_FUNC_PROTO: {
1443 const struct btf_param *m1, *m2;
1444
1445 if (btf_vlen(t1) != btf_vlen(t2)) {
1446 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1447 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1448 return false;
1449 }
1450
1451 n = btf_vlen(t1);
1452 m1 = btf_params(t1);
1453 m2 = btf_params(t2);
1454 for (i = 0; i < n; i++, m1++, m2++) {
1455 /* ignore func arg names */
1456 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1457 return false;
1458 }
1459
1460 /* now check return type as well */
1461 id1 = t1->type;
1462 id2 = t2->type;
1463 goto recur;
1464 }
1465
1466 /* skip_mods_and_typedefs() make this impossible */
1467 case BTF_KIND_TYPEDEF:
1468 case BTF_KIND_VOLATILE:
1469 case BTF_KIND_CONST:
1470 case BTF_KIND_RESTRICT:
1471 /* DATASECs are never compared with each other */
1472 case BTF_KIND_DATASEC:
1473 default:
1474 pr_warn("global '%s': unsupported BTF kind %s\n",
1475 sym_name, btf_kind_str(t1));
1476 return false;
1477 }
1478}
1479
1480static bool map_defs_match(const char *sym_name,
1481 const struct btf *main_btf,
1482 const struct btf_map_def *main_def,
1483 const struct btf_map_def *main_inner_def,
1484 const struct btf *extra_btf,
1485 const struct btf_map_def *extra_def,
1486 const struct btf_map_def *extra_inner_def)
1487{
1488 const char *reason;
1489
1490 if (main_def->map_type != extra_def->map_type) {
1491 reason = "type";
1492 goto mismatch;
1493 }
1494
1495 /* check key type/size match */
1496 if (main_def->key_size != extra_def->key_size) {
1497 reason = "key_size";
1498 goto mismatch;
1499 }
1500 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1501 reason = "key type";
1502 goto mismatch;
1503 }
1504 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1505 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1506 main_btf, main_def->key_type_id,
1507 extra_btf, extra_def->key_type_id)) {
1508 reason = "key type";
1509 goto mismatch;
1510 }
1511
1512 /* validate value type/size match */
1513 if (main_def->value_size != extra_def->value_size) {
1514 reason = "value_size";
1515 goto mismatch;
1516 }
1517 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1518 reason = "value type";
1519 goto mismatch;
1520 }
1521 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1522 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1523 main_btf, main_def->value_type_id,
1524 extra_btf, extra_def->value_type_id)) {
1525 reason = "key type";
1526 goto mismatch;
1527 }
1528
1529 if (main_def->max_entries != extra_def->max_entries) {
1530 reason = "max_entries";
1531 goto mismatch;
1532 }
1533 if (main_def->map_flags != extra_def->map_flags) {
1534 reason = "map_flags";
1535 goto mismatch;
1536 }
1537 if (main_def->numa_node != extra_def->numa_node) {
1538 reason = "numa_node";
1539 goto mismatch;
1540 }
1541 if (main_def->pinning != extra_def->pinning) {
1542 reason = "pinning";
1543 goto mismatch;
1544 }
1545
1546 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1547 reason = "inner map";
1548 goto mismatch;
1549 }
1550
1551 if (main_def->parts & MAP_DEF_INNER_MAP) {
1552 char inner_map_name[128];
1553
1554 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1555
1556 return map_defs_match(inner_map_name,
1557 main_btf, main_inner_def, NULL,
1558 extra_btf, extra_inner_def, NULL);
1559 }
1560
1561 return true;
1562
1563mismatch:
1564 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1565 return false;
1566}
1567
1568static bool glob_map_defs_match(const char *sym_name,
1569 struct bpf_linker *linker, struct glob_sym *glob_sym,
1570 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1571{
1572 struct btf_map_def dst_def = {}, dst_inner_def = {};
1573 struct btf_map_def src_def = {}, src_inner_def = {};
1574 const struct btf_type *t;
1575 int err;
1576
1577 t = btf__type_by_id(obj->btf, btf_id);
1578 if (!btf_is_var(t)) {
1579 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1580 return false;
1581 }
1582 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1583
1584 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1585 if (err) {
1586 pr_warn("global '%s': invalid map definition\n", sym_name);
1587 return false;
1588 }
1589
1590 /* re-parse existing map definition */
1591 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1592 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1593 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1594 if (err) {
1595 /* this should not happen, because we already validated it */
1596 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1597 return false;
1598 }
1599
1600 /* Currently extern map definition has to be complete and match
1601 * concrete map definition exactly. This restriction might be lifted
1602 * in the future.
1603 */
1604 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1605 obj->btf, &src_def, &src_inner_def);
1606}
1607
1608static bool glob_syms_match(const char *sym_name,
1609 struct bpf_linker *linker, struct glob_sym *glob_sym,
1610 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1611{
1612 const struct btf_type *src_t;
1613
1614 /* if we are dealing with externs, BTF types describing both global
1615 * and extern VARs/FUNCs should be completely present in all files
1616 */
1617 if (!glob_sym->btf_id || !btf_id) {
1618 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1619 return false;
1620 }
1621
1622 src_t = btf__type_by_id(obj->btf, btf_id);
1623 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1624 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1625 btf_kind_str(src_t), sym_name);
1626 return false;
1627 }
1628
1629 /* deal with .maps definitions specially */
1630 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1631 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1632
1633 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1634 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1635 return false;
1636
1637 return true;
1638}
1639
1640static bool btf_is_non_static(const struct btf_type *t)
1641{
1642 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1643 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1644}
1645
1646static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1647 int *out_btf_sec_id, int *out_btf_id)
1648{
1649 int i, j, n, m, btf_id = 0;
1650 const struct btf_type *t;
1651 const struct btf_var_secinfo *vi;
1652 const char *name;
1653
1654 if (!obj->btf) {
1655 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1656 return -EINVAL;
1657 }
1658
1659 n = btf__type_cnt(obj->btf);
1660 for (i = 1; i < n; i++) {
1661 t = btf__type_by_id(obj->btf, i);
1662
1663 /* some global and extern FUNCs and VARs might not be associated with any
1664 * DATASEC, so try to detect them in the same pass
1665 */
1666 if (btf_is_non_static(t)) {
1667 name = btf__str_by_offset(obj->btf, t->name_off);
1668 if (strcmp(name, sym_name) != 0)
1669 continue;
1670
1671 /* remember and still try to find DATASEC */
1672 btf_id = i;
1673 continue;
1674 }
1675
1676 if (!btf_is_datasec(t))
1677 continue;
1678
1679 vi = btf_var_secinfos(t);
1680 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1681 t = btf__type_by_id(obj->btf, vi->type);
1682 name = btf__str_by_offset(obj->btf, t->name_off);
1683
1684 if (strcmp(name, sym_name) != 0)
1685 continue;
1686 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1687 continue;
1688 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1689 continue;
1690
1691 if (btf_id && btf_id != vi->type) {
1692 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1693 sym_name, btf_id, vi->type);
1694 return -EINVAL;
1695 }
1696
1697 *out_btf_sec_id = i;
1698 *out_btf_id = vi->type;
1699
1700 return 0;
1701 }
1702 }
1703
1704 /* free-floating extern or global FUNC */
1705 if (btf_id) {
1706 *out_btf_sec_id = 0;
1707 *out_btf_id = btf_id;
1708 return 0;
1709 }
1710
1711 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1712 return -ENOENT;
1713}
1714
1715static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1716{
1717 struct src_sec *sec;
1718 int i;
1719
1720 for (i = 1; i < obj->sec_cnt; i++) {
1721 sec = &obj->secs[i];
1722
1723 if (strcmp(sec->sec_name, sec_name) == 0)
1724 return sec;
1725 }
1726
1727 return NULL;
1728}
1729
1730static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1731 struct btf *src_btf, int src_id)
1732{
1733 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1734 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1735 struct btf_param *src_p, *dst_p;
1736 const char *s;
1737 int i, n, off;
1738
1739 /* We already made sure that source and destination types (FUNC or
1740 * VAR) match in terms of types and argument names.
1741 */
1742 if (btf_is_var(dst_t)) {
1743 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1744 return 0;
1745 }
1746
1747 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1748
1749 /* now onto FUNC_PROTO types */
1750 src_t = btf_type_by_id(src_btf, src_t->type);
1751 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1752
1753 /* Fill in all the argument names, which for extern FUNCs are missing.
1754 * We'll end up with two copies of FUNCs/VARs for externs, but that
1755 * will be taken care of by BTF dedup at the very end.
1756 * It might be that BTF types for extern in one file has less/more BTF
1757 * information (e.g., FWD instead of full STRUCT/UNION information),
1758 * but that should be (in most cases, subject to BTF dedup rules)
1759 * handled and resolved by BTF dedup algorithm as well, so we won't
1760 * worry about it. Our only job is to make sure that argument names
1761 * are populated on both sides, otherwise BTF dedup will pedantically
1762 * consider them different.
1763 */
1764 src_p = btf_params(src_t);
1765 dst_p = btf_params(dst_t);
1766 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1767 if (!src_p->name_off)
1768 continue;
1769
1770 /* src_btf has more complete info, so add name to dst_btf */
1771 s = btf__str_by_offset(src_btf, src_p->name_off);
1772 off = btf__add_str(dst_btf, s);
1773 if (off < 0)
1774 return off;
1775 dst_p->name_off = off;
1776 }
1777 return 0;
1778}
1779
1780static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1781{
1782 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1783}
1784
1785static void sym_update_type(Elf64_Sym *sym, int sym_type)
1786{
1787 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1788}
1789
1790static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1791{
1792 /* libelf doesn't provide setters for ST_VISIBILITY,
1793 * but it is stored in the lower 2 bits of st_other
1794 */
1795 sym->st_other &= ~0x03;
1796 sym->st_other |= sym_vis;
1797}
1798
1799static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1800 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1801{
1802 struct src_sec *src_sec = NULL;
1803 struct dst_sec *dst_sec = NULL;
1804 struct glob_sym *glob_sym = NULL;
1805 int name_off, sym_type, sym_bind, sym_vis, err;
1806 int btf_sec_id = 0, btf_id = 0;
1807 size_t dst_sym_idx;
1808 Elf64_Sym *dst_sym;
1809 bool sym_is_extern;
1810
1811 sym_type = ELF64_ST_TYPE(sym->st_info);
1812 sym_bind = ELF64_ST_BIND(sym->st_info);
1813 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1814 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1815
1816 if (sym_is_extern) {
1817 if (!obj->btf) {
1818 pr_warn("externs without BTF info are not supported\n");
1819 return -ENOTSUP;
1820 }
1821 } else if (sym->st_shndx < SHN_LORESERVE) {
1822 src_sec = &obj->secs[sym->st_shndx];
1823 if (src_sec->skipped)
1824 return 0;
1825 dst_sec = &linker->secs[src_sec->dst_id];
1826
1827 /* allow only one STT_SECTION symbol per section */
1828 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1829 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1830 return 0;
1831 }
1832 }
1833
1834 if (sym_bind == STB_LOCAL)
1835 goto add_sym;
1836
1837 /* find matching BTF info */
1838 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1839 if (err)
1840 return err;
1841
1842 if (sym_is_extern && btf_sec_id) {
1843 const char *sec_name = NULL;
1844 const struct btf_type *t;
1845
1846 t = btf__type_by_id(obj->btf, btf_sec_id);
1847 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1848
1849 /* Clang puts unannotated extern vars into
1850 * '.extern' BTF DATASEC. Treat them the same
1851 * as unannotated extern funcs (which are
1852 * currently not put into any DATASECs).
1853 * Those don't have associated src_sec/dst_sec.
1854 */
1855 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1856 src_sec = find_src_sec_by_name(obj, sec_name);
1857 if (!src_sec) {
1858 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1859 return -ENOENT;
1860 }
1861 dst_sec = &linker->secs[src_sec->dst_id];
1862 }
1863 }
1864
1865 glob_sym = find_glob_sym(linker, sym_name);
1866 if (glob_sym) {
1867 /* Preventively resolve to existing symbol. This is
1868 * needed for further relocation symbol remapping in
1869 * the next step of linking.
1870 */
1871 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1872
1873 /* If both symbols are non-externs, at least one of
1874 * them has to be STB_WEAK, otherwise they are in
1875 * a conflict with each other.
1876 */
1877 if (!sym_is_extern && !glob_sym->is_extern
1878 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1879 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1880 src_sym_idx, sym_name, obj->filename);
1881 return -EINVAL;
1882 }
1883
1884 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1885 return -EINVAL;
1886
1887 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1888
1889 /* If new symbol is strong, then force dst_sym to be strong as
1890 * well; this way a mix of weak and non-weak extern
1891 * definitions will end up being strong.
1892 */
1893 if (sym_bind == STB_GLOBAL) {
1894 /* We still need to preserve type (NOTYPE or
1895 * OBJECT/FUNC, depending on whether the symbol is
1896 * extern or not)
1897 */
1898 sym_update_bind(dst_sym, STB_GLOBAL);
1899 glob_sym->is_weak = false;
1900 }
1901
1902 /* Non-default visibility is "contaminating", with stricter
1903 * visibility overwriting more permissive ones, even if more
1904 * permissive visibility comes from just an extern definition.
1905 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1906 * ensured by ELF symbol sanity checks above.
1907 */
1908 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1909 sym_update_visibility(dst_sym, sym_vis);
1910
1911 /* If the new symbol is extern, then regardless if
1912 * existing symbol is extern or resolved global, just
1913 * keep the existing one untouched.
1914 */
1915 if (sym_is_extern)
1916 return 0;
1917
1918 /* If existing symbol is a strong resolved symbol, bail out,
1919 * because we lost resolution battle have nothing to
1920 * contribute. We already checked abover that there is no
1921 * strong-strong conflict. We also already tightened binding
1922 * and visibility, so nothing else to contribute at that point.
1923 */
1924 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1925 return 0;
1926
1927 /* At this point, new symbol is strong non-extern,
1928 * so overwrite glob_sym with new symbol information.
1929 * Preserve binding and visibility.
1930 */
1931 sym_update_type(dst_sym, sym_type);
1932 dst_sym->st_shndx = dst_sec->sec_idx;
1933 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1934 dst_sym->st_size = sym->st_size;
1935
1936 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1937 glob_sym->sec_id = dst_sec->id;
1938 glob_sym->is_extern = false;
1939
1940 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1941 obj->btf, btf_id))
1942 return -EINVAL;
1943
1944 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1945 glob_sym->underlying_btf_id = 0;
1946
1947 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1948 return 0;
1949 }
1950
1951add_sym:
1952 name_off = strset__add_str(linker->strtab_strs, sym_name);
1953 if (name_off < 0)
1954 return name_off;
1955
1956 dst_sym = add_new_sym(linker, &dst_sym_idx);
1957 if (!dst_sym)
1958 return -ENOMEM;
1959
1960 dst_sym->st_name = name_off;
1961 dst_sym->st_info = sym->st_info;
1962 dst_sym->st_other = sym->st_other;
1963 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1964 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1965 dst_sym->st_size = sym->st_size;
1966
1967 obj->sym_map[src_sym_idx] = dst_sym_idx;
1968
1969 if (sym_type == STT_SECTION && dst_sym) {
1970 dst_sec->sec_sym_idx = dst_sym_idx;
1971 dst_sym->st_value = 0;
1972 }
1973
1974 if (sym_bind != STB_LOCAL) {
1975 glob_sym = add_glob_sym(linker);
1976 if (!glob_sym)
1977 return -ENOMEM;
1978
1979 glob_sym->sym_idx = dst_sym_idx;
1980 /* we use dst_sec->id (and not dst_sec->sec_idx), because
1981 * ephemeral sections (.kconfig, .ksyms, etc) don't have
1982 * sec_idx (as they don't have corresponding ELF section), but
1983 * still have id. .extern doesn't have even ephemeral section
1984 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1985 */
1986 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1987 glob_sym->name_off = name_off;
1988 /* we will fill btf_id in during BTF merging step */
1989 glob_sym->btf_id = 0;
1990 glob_sym->is_extern = sym_is_extern;
1991 glob_sym->is_weak = sym_bind == STB_WEAK;
1992 }
1993
1994 return 0;
1995}
1996
1997static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
1998{
1999 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2000 struct dst_sec *dst_symtab;
2001 int i, err;
2002
2003 for (i = 1; i < obj->sec_cnt; i++) {
2004 struct src_sec *src_sec, *src_linked_sec;
2005 struct dst_sec *dst_sec, *dst_linked_sec;
2006 Elf64_Rel *src_rel, *dst_rel;
2007 int j, n;
2008
2009 src_sec = &obj->secs[i];
2010 if (!is_relo_sec(src_sec))
2011 continue;
2012
2013 /* shdr->sh_info points to relocatable section */
2014 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2015 if (src_linked_sec->skipped)
2016 continue;
2017
2018 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2019 if (!dst_sec) {
2020 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2021 if (!dst_sec)
2022 return -ENOMEM;
2023 err = init_sec(linker, dst_sec, src_sec);
2024 if (err) {
2025 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2026 return err;
2027 }
2028 } else if (!secs_match(dst_sec, src_sec)) {
2029 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2030 return -1;
2031 }
2032
2033 /* add_dst_sec() above could have invalidated linker->secs */
2034 dst_symtab = &linker->secs[linker->symtab_sec_idx];
2035
2036 /* shdr->sh_link points to SYMTAB */
2037 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2038
2039 /* shdr->sh_info points to relocated section */
2040 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2041 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2042
2043 src_sec->dst_id = dst_sec->id;
2044 err = extend_sec(linker, dst_sec, src_sec);
2045 if (err)
2046 return err;
2047
2048 src_rel = src_sec->data->d_buf;
2049 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2050 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2051 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2052 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2053 size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
2054 Elf64_Sym *src_sym, *dst_sym;
2055 size_t dst_sym_idx;
2056
2057 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2058 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2059
2060 dst_sym_idx = obj->sym_map[src_sym_idx];
2061 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
2062 dst_rel->r_offset += src_linked_sec->dst_off;
2063 sym_type = ELF64_R_TYPE(src_rel->r_info);
2064 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2065
2066 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2067 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2068 struct bpf_insn *insn;
2069
2070 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2071 /* calls to the very first static function inside
2072 * .text section at offset 0 will
2073 * reference section symbol, not the
2074 * function symbol. Fix that up,
2075 * otherwise it won't be possible to
2076 * relocate calls to two different
2077 * static functions with the same name
2078 * (rom two different object files)
2079 */
2080 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2081 if (insn->code == (BPF_JMP | BPF_CALL))
2082 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2083 else
2084 insn->imm += sec->dst_off;
2085 } else {
2086 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2087 return -EINVAL;
2088 }
2089 }
2090
2091 }
2092 }
2093
2094 return 0;
2095}
2096
2097static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2098 int sym_type, const char *sym_name)
2099{
2100 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2101 Elf64_Sym *sym = symtab->data->d_buf;
2102 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2103 int str_sec_idx = symtab->shdr->sh_link;
2104 const char *name;
2105
2106 for (i = 0; i < n; i++, sym++) {
2107 if (sym->st_shndx != sec_idx)
2108 continue;
2109 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2110 continue;
2111
2112 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2113 if (!name)
2114 return NULL;
2115
2116 if (strcmp(sym_name, name) != 0)
2117 continue;
2118
2119 return sym;
2120 }
2121
2122 return NULL;
2123}
2124
2125static int linker_fixup_btf(struct src_obj *obj)
2126{
2127 const char *sec_name;
2128 struct src_sec *sec;
2129 int i, j, n, m;
2130
2131 if (!obj->btf)
2132 return 0;
2133
2134 n = btf__type_cnt(obj->btf);
2135 for (i = 1; i < n; i++) {
2136 struct btf_var_secinfo *vi;
2137 struct btf_type *t;
2138
2139 t = btf_type_by_id(obj->btf, i);
2140 if (btf_kind(t) != BTF_KIND_DATASEC)
2141 continue;
2142
2143 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2144 sec = find_src_sec_by_name(obj, sec_name);
2145 if (sec) {
2146 /* record actual section size, unless ephemeral */
2147 if (sec->shdr)
2148 t->size = sec->shdr->sh_size;
2149 } else {
2150 /* BTF can have some sections that are not represented
2151 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2152 * for special extern variables.
2153 *
2154 * For all but one such special (ephemeral)
2155 * sections, we pre-create "section shells" to be able
2156 * to keep track of extra per-section metadata later
2157 * (e.g., those BTF extern variables).
2158 *
2159 * .extern is even more special, though, because it
2160 * contains extern variables that need to be resolved
2161 * by static linker, not libbpf and kernel. When such
2162 * externs are resolved, we are going to remove them
2163 * from .extern BTF section and might end up not
2164 * needing it at all. Each resolved extern should have
2165 * matching non-extern VAR/FUNC in other sections.
2166 *
2167 * We do support leaving some of the externs
2168 * unresolved, though, to support cases of building
2169 * libraries, which will later be linked against final
2170 * BPF applications. So if at finalization we still
2171 * see unresolved externs, we'll create .extern
2172 * section on our own.
2173 */
2174 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2175 continue;
2176
2177 sec = add_src_sec(obj, sec_name);
2178 if (!sec)
2179 return -ENOMEM;
2180
2181 sec->ephemeral = true;
2182 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2183 }
2184
2185 /* remember ELF section and its BTF type ID match */
2186 sec->sec_type_id = i;
2187
2188 /* fix up variable offsets */
2189 vi = btf_var_secinfos(t);
2190 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2191 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2192 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2193 int var_linkage = btf_var(vt)->linkage;
2194 Elf64_Sym *sym;
2195
2196 /* no need to patch up static or extern vars */
2197 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2198 continue;
2199
2200 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2201 if (!sym) {
2202 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2203 return -ENOENT;
2204 }
2205
2206 vi->offset = sym->st_value;
2207 }
2208 }
2209
2210 return 0;
2211}
2212
2213static int remap_type_id(__u32 *type_id, void *ctx)
2214{
2215 int *id_map = ctx;
2216 int new_id = id_map[*type_id];
2217
2218 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2219 if (new_id == 0 && *type_id != 0) {
2220 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2221 return -EINVAL;
2222 }
2223
2224 *type_id = id_map[*type_id];
2225
2226 return 0;
2227}
2228
2229static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2230{
2231 const struct btf_type *t;
2232 int i, j, n, start_id, id;
2233 const char *name;
2234
2235 if (!obj->btf)
2236 return 0;
2237
2238 start_id = btf__type_cnt(linker->btf);
2239 n = btf__type_cnt(obj->btf);
2240
2241 obj->btf_type_map = calloc(n + 1, sizeof(int));
2242 if (!obj->btf_type_map)
2243 return -ENOMEM;
2244
2245 for (i = 1; i < n; i++) {
2246 struct glob_sym *glob_sym = NULL;
2247
2248 t = btf__type_by_id(obj->btf, i);
2249
2250 /* DATASECs are handled specially below */
2251 if (btf_kind(t) == BTF_KIND_DATASEC)
2252 continue;
2253
2254 if (btf_is_non_static(t)) {
2255 /* there should be glob_sym already */
2256 name = btf__str_by_offset(obj->btf, t->name_off);
2257 glob_sym = find_glob_sym(linker, name);
2258
2259 /* VARs without corresponding glob_sym are those that
2260 * belong to skipped/deduplicated sections (i.e.,
2261 * license and version), so just skip them
2262 */
2263 if (!glob_sym)
2264 continue;
2265
2266 /* linker_append_elf_sym() might have requested
2267 * updating underlying type ID, if extern was resolved
2268 * to strong symbol or weak got upgraded to non-weak
2269 */
2270 if (glob_sym->underlying_btf_id == 0)
2271 glob_sym->underlying_btf_id = -t->type;
2272
2273 /* globals from previous object files that match our
2274 * VAR/FUNC already have a corresponding associated
2275 * BTF type, so just make sure to use it
2276 */
2277 if (glob_sym->btf_id) {
2278 /* reuse existing BTF type for global var/func */
2279 obj->btf_type_map[i] = glob_sym->btf_id;
2280 continue;
2281 }
2282 }
2283
2284 id = btf__add_type(linker->btf, obj->btf, t);
2285 if (id < 0) {
2286 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2287 return id;
2288 }
2289
2290 obj->btf_type_map[i] = id;
2291
2292 /* record just appended BTF type for var/func */
2293 if (glob_sym) {
2294 glob_sym->btf_id = id;
2295 glob_sym->underlying_btf_id = -t->type;
2296 }
2297 }
2298
2299 /* remap all the types except DATASECs */
2300 n = btf__type_cnt(linker->btf);
2301 for (i = start_id; i < n; i++) {
2302 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2303
2304 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2305 return -EINVAL;
2306 }
2307
2308 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2309 * actual type), if necessary
2310 */
2311 for (i = 0; i < linker->glob_sym_cnt; i++) {
2312 struct glob_sym *glob_sym = &linker->glob_syms[i];
2313 struct btf_type *glob_t;
2314
2315 if (glob_sym->underlying_btf_id >= 0)
2316 continue;
2317
2318 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2319
2320 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2321 glob_t->type = glob_sym->underlying_btf_id;
2322 }
2323
2324 /* append DATASEC info */
2325 for (i = 1; i < obj->sec_cnt; i++) {
2326 struct src_sec *src_sec;
2327 struct dst_sec *dst_sec;
2328 const struct btf_var_secinfo *src_var;
2329 struct btf_var_secinfo *dst_var;
2330
2331 src_sec = &obj->secs[i];
2332 if (!src_sec->sec_type_id || src_sec->skipped)
2333 continue;
2334 dst_sec = &linker->secs[src_sec->dst_id];
2335
2336 /* Mark section as having BTF regardless of the presence of
2337 * variables. In some cases compiler might generate empty BTF
2338 * with no variables information. E.g., when promoting local
2339 * array/structure variable initial values and BPF object
2340 * file otherwise has no read-only static variables in
2341 * .rodata. We need to preserve such empty BTF and just set
2342 * correct section size.
2343 */
2344 dst_sec->has_btf = true;
2345
2346 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2347 src_var = btf_var_secinfos(t);
2348 n = btf_vlen(t);
2349 for (j = 0; j < n; j++, src_var++) {
2350 void *sec_vars = dst_sec->sec_vars;
2351 int new_id = obj->btf_type_map[src_var->type];
2352 struct glob_sym *glob_sym = NULL;
2353
2354 t = btf_type_by_id(linker->btf, new_id);
2355 if (btf_is_non_static(t)) {
2356 name = btf__str_by_offset(linker->btf, t->name_off);
2357 glob_sym = find_glob_sym(linker, name);
2358 if (glob_sym->sec_id != dst_sec->id) {
2359 pr_warn("global '%s': section mismatch %d vs %d\n",
2360 name, glob_sym->sec_id, dst_sec->id);
2361 return -EINVAL;
2362 }
2363 }
2364
2365 /* If there is already a member (VAR or FUNC) mapped
2366 * to the same type, don't add a duplicate entry.
2367 * This will happen when multiple object files define
2368 * the same extern VARs/FUNCs.
2369 */
2370 if (glob_sym && glob_sym->var_idx >= 0) {
2371 __s64 sz;
2372
2373 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2374 /* Because underlying BTF type might have
2375 * changed, so might its size have changed, so
2376 * re-calculate and update it in sec_var.
2377 */
2378 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2379 if (sz < 0) {
2380 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2381 name, (int)sz);
2382 return -EINVAL;
2383 }
2384 dst_var->size = sz;
2385 continue;
2386 }
2387
2388 sec_vars = libbpf_reallocarray(sec_vars,
2389 dst_sec->sec_var_cnt + 1,
2390 sizeof(*dst_sec->sec_vars));
2391 if (!sec_vars)
2392 return -ENOMEM;
2393
2394 dst_sec->sec_vars = sec_vars;
2395 dst_sec->sec_var_cnt++;
2396
2397 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2398 dst_var->type = obj->btf_type_map[src_var->type];
2399 dst_var->size = src_var->size;
2400 dst_var->offset = src_sec->dst_off + src_var->offset;
2401
2402 if (glob_sym)
2403 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2404 }
2405 }
2406
2407 return 0;
2408}
2409
2410static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2411{
2412 void *tmp;
2413
2414 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2415 if (!tmp)
2416 return NULL;
2417 ext_data->recs = tmp;
2418
2419 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2420 memcpy(tmp, src_rec, ext_data->rec_sz);
2421
2422 ext_data->rec_cnt++;
2423
2424 return tmp;
2425}
2426
2427static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2428{
2429 const struct btf_ext_info_sec *ext_sec;
2430 const char *sec_name, *s;
2431 struct src_sec *src_sec;
2432 struct dst_sec *dst_sec;
2433 int rec_sz, str_off, i;
2434
2435 if (!obj->btf_ext)
2436 return 0;
2437
2438 rec_sz = obj->btf_ext->func_info.rec_size;
2439 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2440 struct bpf_func_info_min *src_rec, *dst_rec;
2441
2442 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2443 src_sec = find_src_sec_by_name(obj, sec_name);
2444 if (!src_sec) {
2445 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2446 return -EINVAL;
2447 }
2448 dst_sec = &linker->secs[src_sec->dst_id];
2449
2450 if (dst_sec->func_info.rec_sz == 0)
2451 dst_sec->func_info.rec_sz = rec_sz;
2452 if (dst_sec->func_info.rec_sz != rec_sz) {
2453 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2454 return -EINVAL;
2455 }
2456
2457 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2458 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2459 if (!dst_rec)
2460 return -ENOMEM;
2461
2462 dst_rec->insn_off += src_sec->dst_off;
2463 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2464 }
2465 }
2466
2467 rec_sz = obj->btf_ext->line_info.rec_size;
2468 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2469 struct bpf_line_info_min *src_rec, *dst_rec;
2470
2471 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2472 src_sec = find_src_sec_by_name(obj, sec_name);
2473 if (!src_sec) {
2474 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2475 return -EINVAL;
2476 }
2477 dst_sec = &linker->secs[src_sec->dst_id];
2478
2479 if (dst_sec->line_info.rec_sz == 0)
2480 dst_sec->line_info.rec_sz = rec_sz;
2481 if (dst_sec->line_info.rec_sz != rec_sz) {
2482 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2483 return -EINVAL;
2484 }
2485
2486 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2487 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2488 if (!dst_rec)
2489 return -ENOMEM;
2490
2491 dst_rec->insn_off += src_sec->dst_off;
2492
2493 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2494 str_off = btf__add_str(linker->btf, s);
2495 if (str_off < 0)
2496 return -ENOMEM;
2497 dst_rec->file_name_off = str_off;
2498
2499 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2500 str_off = btf__add_str(linker->btf, s);
2501 if (str_off < 0)
2502 return -ENOMEM;
2503 dst_rec->line_off = str_off;
2504
2505 /* dst_rec->line_col is fine */
2506 }
2507 }
2508
2509 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2510 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2511 struct bpf_core_relo *src_rec, *dst_rec;
2512
2513 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2514 src_sec = find_src_sec_by_name(obj, sec_name);
2515 if (!src_sec) {
2516 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2517 return -EINVAL;
2518 }
2519 dst_sec = &linker->secs[src_sec->dst_id];
2520
2521 if (dst_sec->core_relo_info.rec_sz == 0)
2522 dst_sec->core_relo_info.rec_sz = rec_sz;
2523 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2524 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2525 return -EINVAL;
2526 }
2527
2528 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2529 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2530 if (!dst_rec)
2531 return -ENOMEM;
2532
2533 dst_rec->insn_off += src_sec->dst_off;
2534 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2535
2536 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2537 str_off = btf__add_str(linker->btf, s);
2538 if (str_off < 0)
2539 return -ENOMEM;
2540 dst_rec->access_str_off = str_off;
2541
2542 /* dst_rec->kind is fine */
2543 }
2544 }
2545
2546 return 0;
2547}
2548
2549int bpf_linker__finalize(struct bpf_linker *linker)
2550{
2551 struct dst_sec *sec;
2552 size_t strs_sz;
2553 const void *strs;
2554 int err, i;
2555
2556 if (!linker->elf)
2557 return libbpf_err(-EINVAL);
2558
2559 err = finalize_btf(linker);
2560 if (err)
2561 return libbpf_err(err);
2562
2563 /* Finalize strings */
2564 strs_sz = strset__data_size(linker->strtab_strs);
2565 strs = strset__data(linker->strtab_strs);
2566
2567 sec = &linker->secs[linker->strtab_sec_idx];
2568 sec->data->d_align = 1;
2569 sec->data->d_off = 0LL;
2570 sec->data->d_buf = (void *)strs;
2571 sec->data->d_type = ELF_T_BYTE;
2572 sec->data->d_size = strs_sz;
2573 sec->shdr->sh_size = strs_sz;
2574
2575 for (i = 1; i < linker->sec_cnt; i++) {
2576 sec = &linker->secs[i];
2577
2578 /* STRTAB is handled specially above */
2579 if (sec->sec_idx == linker->strtab_sec_idx)
2580 continue;
2581
2582 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2583 if (!sec->scn)
2584 continue;
2585
2586 sec->data->d_buf = sec->raw_data;
2587 }
2588
2589 /* Finalize ELF layout */
2590 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2591 err = -errno;
2592 pr_warn_elf("failed to finalize ELF layout");
2593 return libbpf_err(err);
2594 }
2595
2596 /* Write out final ELF contents */
2597 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2598 err = -errno;
2599 pr_warn_elf("failed to write ELF contents");
2600 return libbpf_err(err);
2601 }
2602
2603 elf_end(linker->elf);
2604 close(linker->fd);
2605
2606 linker->elf = NULL;
2607 linker->fd = -1;
2608
2609 return 0;
2610}
2611
2612static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2613 size_t align, const void *raw_data, size_t raw_sz)
2614{
2615 Elf_Scn *scn;
2616 Elf_Data *data;
2617 Elf64_Shdr *shdr;
2618 int name_off;
2619
2620 name_off = strset__add_str(linker->strtab_strs, sec_name);
2621 if (name_off < 0)
2622 return name_off;
2623
2624 scn = elf_newscn(linker->elf);
2625 if (!scn)
2626 return -ENOMEM;
2627 data = elf_newdata(scn);
2628 if (!data)
2629 return -ENOMEM;
2630 shdr = elf64_getshdr(scn);
2631 if (!shdr)
2632 return -EINVAL;
2633
2634 shdr->sh_name = name_off;
2635 shdr->sh_type = SHT_PROGBITS;
2636 shdr->sh_flags = 0;
2637 shdr->sh_size = raw_sz;
2638 shdr->sh_link = 0;
2639 shdr->sh_info = 0;
2640 shdr->sh_addralign = align;
2641 shdr->sh_entsize = 0;
2642
2643 data->d_type = ELF_T_BYTE;
2644 data->d_size = raw_sz;
2645 data->d_buf = (void *)raw_data;
2646 data->d_align = align;
2647 data->d_off = 0;
2648
2649 return 0;
2650}
2651
2652static int finalize_btf(struct bpf_linker *linker)
2653{
2654 LIBBPF_OPTS(btf_dedup_opts, opts);
2655 struct btf *btf = linker->btf;
2656 const void *raw_data;
2657 int i, j, id, err;
2658 __u32 raw_sz;
2659
2660 /* bail out if no BTF data was produced */
2661 if (btf__type_cnt(linker->btf) == 1)
2662 return 0;
2663
2664 for (i = 1; i < linker->sec_cnt; i++) {
2665 struct dst_sec *sec = &linker->secs[i];
2666
2667 if (!sec->has_btf)
2668 continue;
2669
2670 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2671 if (id < 0) {
2672 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2673 sec->sec_name, id);
2674 return id;
2675 }
2676
2677 for (j = 0; j < sec->sec_var_cnt; j++) {
2678 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2679
2680 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2681 return -EINVAL;
2682 }
2683 }
2684
2685 err = finalize_btf_ext(linker);
2686 if (err) {
2687 pr_warn(".BTF.ext generation failed: %d\n", err);
2688 return err;
2689 }
2690
2691 opts.btf_ext = linker->btf_ext;
2692 err = btf__dedup(linker->btf, &opts);
2693 if (err) {
2694 pr_warn("BTF dedup failed: %d\n", err);
2695 return err;
2696 }
2697
2698 /* Emit .BTF section */
2699 raw_data = btf__raw_data(linker->btf, &raw_sz);
2700 if (!raw_data)
2701 return -ENOMEM;
2702
2703 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2704 if (err) {
2705 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2706 return err;
2707 }
2708
2709 /* Emit .BTF.ext section */
2710 if (linker->btf_ext) {
2711 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2712 if (!raw_data)
2713 return -ENOMEM;
2714
2715 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2716 if (err) {
2717 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2718 return err;
2719 }
2720 }
2721
2722 return 0;
2723}
2724
2725static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2726 const char *sec_name, struct btf_ext_sec_data *sec_data)
2727{
2728 struct btf_ext_info_sec *sec_info;
2729 void *cur = output;
2730 int str_off;
2731 size_t sz;
2732
2733 if (!sec_data->rec_cnt)
2734 return 0;
2735
2736 str_off = btf__add_str(linker->btf, sec_name);
2737 if (str_off < 0)
2738 return -ENOMEM;
2739
2740 sec_info = cur;
2741 sec_info->sec_name_off = str_off;
2742 sec_info->num_info = sec_data->rec_cnt;
2743 cur += sizeof(struct btf_ext_info_sec);
2744
2745 sz = sec_data->rec_cnt * sec_data->rec_sz;
2746 memcpy(cur, sec_data->recs, sz);
2747 cur += sz;
2748
2749 return cur - output;
2750}
2751
2752static int finalize_btf_ext(struct bpf_linker *linker)
2753{
2754 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2755 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2756 struct btf_ext_header *hdr;
2757 void *data, *cur;
2758 int i, err, sz;
2759
2760 /* validate that all sections have the same .BTF.ext record sizes
2761 * and calculate total data size for each type of data (func info,
2762 * line info, core relos)
2763 */
2764 for (i = 1; i < linker->sec_cnt; i++) {
2765 struct dst_sec *sec = &linker->secs[i];
2766
2767 if (sec->func_info.rec_cnt) {
2768 if (func_rec_sz == 0)
2769 func_rec_sz = sec->func_info.rec_sz;
2770 if (func_rec_sz != sec->func_info.rec_sz) {
2771 pr_warn("mismatch in func_info record size %zu != %u\n",
2772 func_rec_sz, sec->func_info.rec_sz);
2773 return -EINVAL;
2774 }
2775
2776 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2777 }
2778 if (sec->line_info.rec_cnt) {
2779 if (line_rec_sz == 0)
2780 line_rec_sz = sec->line_info.rec_sz;
2781 if (line_rec_sz != sec->line_info.rec_sz) {
2782 pr_warn("mismatch in line_info record size %zu != %u\n",
2783 line_rec_sz, sec->line_info.rec_sz);
2784 return -EINVAL;
2785 }
2786
2787 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2788 }
2789 if (sec->core_relo_info.rec_cnt) {
2790 if (core_relo_rec_sz == 0)
2791 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2792 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2793 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2794 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2795 return -EINVAL;
2796 }
2797
2798 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2799 }
2800 }
2801
2802 if (!funcs_sz && !lines_sz && !core_relos_sz)
2803 return 0;
2804
2805 total_sz += sizeof(struct btf_ext_header);
2806 if (funcs_sz) {
2807 funcs_sz += sizeof(__u32); /* record size prefix */
2808 total_sz += funcs_sz;
2809 }
2810 if (lines_sz) {
2811 lines_sz += sizeof(__u32); /* record size prefix */
2812 total_sz += lines_sz;
2813 }
2814 if (core_relos_sz) {
2815 core_relos_sz += sizeof(__u32); /* record size prefix */
2816 total_sz += core_relos_sz;
2817 }
2818
2819 cur = data = calloc(1, total_sz);
2820 if (!data)
2821 return -ENOMEM;
2822
2823 hdr = cur;
2824 hdr->magic = BTF_MAGIC;
2825 hdr->version = BTF_VERSION;
2826 hdr->flags = 0;
2827 hdr->hdr_len = sizeof(struct btf_ext_header);
2828 cur += sizeof(struct btf_ext_header);
2829
2830 /* All offsets are in bytes relative to the end of this header */
2831 hdr->func_info_off = 0;
2832 hdr->func_info_len = funcs_sz;
2833 hdr->line_info_off = funcs_sz;
2834 hdr->line_info_len = lines_sz;
2835 hdr->core_relo_off = funcs_sz + lines_sz;
2836 hdr->core_relo_len = core_relos_sz;
2837
2838 if (funcs_sz) {
2839 *(__u32 *)cur = func_rec_sz;
2840 cur += sizeof(__u32);
2841
2842 for (i = 1; i < linker->sec_cnt; i++) {
2843 struct dst_sec *sec = &linker->secs[i];
2844
2845 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2846 if (sz < 0) {
2847 err = sz;
2848 goto out;
2849 }
2850
2851 cur += sz;
2852 }
2853 }
2854
2855 if (lines_sz) {
2856 *(__u32 *)cur = line_rec_sz;
2857 cur += sizeof(__u32);
2858
2859 for (i = 1; i < linker->sec_cnt; i++) {
2860 struct dst_sec *sec = &linker->secs[i];
2861
2862 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2863 if (sz < 0) {
2864 err = sz;
2865 goto out;
2866 }
2867
2868 cur += sz;
2869 }
2870 }
2871
2872 if (core_relos_sz) {
2873 *(__u32 *)cur = core_relo_rec_sz;
2874 cur += sizeof(__u32);
2875
2876 for (i = 1; i < linker->sec_cnt; i++) {
2877 struct dst_sec *sec = &linker->secs[i];
2878
2879 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2880 if (sz < 0) {
2881 err = sz;
2882 goto out;
2883 }
2884
2885 cur += sz;
2886 }
2887 }
2888
2889 linker->btf_ext = btf_ext__new(data, total_sz);
2890 err = libbpf_get_error(linker->btf_ext);
2891 if (err) {
2892 linker->btf_ext = NULL;
2893 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2894 goto out;
2895 }
2896
2897out:
2898 free(data);
2899 return err;
2900}
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <errno.h>
14#include <linux/err.h>
15#include <linux/btf.h>
16#include <elf.h>
17#include <libelf.h>
18#include <fcntl.h>
19#include "libbpf.h"
20#include "btf.h"
21#include "libbpf_internal.h"
22#include "strset.h"
23
24#define BTF_EXTERN_SEC ".extern"
25
26struct src_sec {
27 const char *sec_name;
28 /* positional (not necessarily ELF) index in an array of sections */
29 int id;
30 /* positional (not necessarily ELF) index of a matching section in a final object file */
31 int dst_id;
32 /* section data offset in a matching output section */
33 int dst_off;
34 /* whether section is omitted from the final ELF file */
35 bool skipped;
36 /* whether section is an ephemeral section, not mapped to an ELF section */
37 bool ephemeral;
38
39 /* ELF info */
40 size_t sec_idx;
41 Elf_Scn *scn;
42 Elf64_Shdr *shdr;
43 Elf_Data *data;
44
45 /* corresponding BTF DATASEC type ID */
46 int sec_type_id;
47};
48
49struct src_obj {
50 const char *filename;
51 int fd;
52 Elf *elf;
53 /* Section header strings section index */
54 size_t shstrs_sec_idx;
55 /* SYMTAB section index */
56 size_t symtab_sec_idx;
57
58 struct btf *btf;
59 struct btf_ext *btf_ext;
60
61 /* List of sections (including ephemeral). Slot zero is unused. */
62 struct src_sec *secs;
63 int sec_cnt;
64
65 /* mapping of symbol indices from src to dst ELF */
66 int *sym_map;
67 /* mapping from the src BTF type IDs to dst ones */
68 int *btf_type_map;
69};
70
71/* single .BTF.ext data section */
72struct btf_ext_sec_data {
73 size_t rec_cnt;
74 __u32 rec_sz;
75 void *recs;
76};
77
78struct glob_sym {
79 /* ELF symbol index */
80 int sym_idx;
81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
82 int sec_id;
83 /* extern name offset in STRTAB */
84 int name_off;
85 /* optional associated BTF type ID */
86 int btf_id;
87 /* BTF type ID to which VAR/FUNC type is pointing to; used for
88 * rewriting types when extern VAR/FUNC is resolved to a concrete
89 * definition
90 */
91 int underlying_btf_id;
92 /* sec_var index in the corresponding dst_sec, if exists */
93 int var_idx;
94
95 /* extern or resolved/global symbol */
96 bool is_extern;
97 /* weak or strong symbol, never goes back from strong to weak */
98 bool is_weak;
99};
100
101struct dst_sec {
102 char *sec_name;
103 /* positional (not necessarily ELF) index in an array of sections */
104 int id;
105
106 bool ephemeral;
107
108 /* ELF info */
109 size_t sec_idx;
110 Elf_Scn *scn;
111 Elf64_Shdr *shdr;
112 Elf_Data *data;
113
114 /* final output section size */
115 int sec_sz;
116 /* final output contents of the section */
117 void *raw_data;
118
119 /* corresponding STT_SECTION symbol index in SYMTAB */
120 int sec_sym_idx;
121
122 /* section's DATASEC variable info, emitted on BTF finalization */
123 bool has_btf;
124 int sec_var_cnt;
125 struct btf_var_secinfo *sec_vars;
126
127 /* section's .BTF.ext data */
128 struct btf_ext_sec_data func_info;
129 struct btf_ext_sec_data line_info;
130 struct btf_ext_sec_data core_relo_info;
131};
132
133struct bpf_linker {
134 char *filename;
135 int fd;
136 Elf *elf;
137 Elf64_Ehdr *elf_hdr;
138
139 /* Output sections metadata */
140 struct dst_sec *secs;
141 int sec_cnt;
142
143 struct strset *strtab_strs; /* STRTAB unique strings */
144 size_t strtab_sec_idx; /* STRTAB section index */
145 size_t symtab_sec_idx; /* SYMTAB section index */
146
147 struct btf *btf;
148 struct btf_ext *btf_ext;
149
150 /* global (including extern) ELF symbols */
151 int glob_sym_cnt;
152 struct glob_sym *glob_syms;
153};
154
155#define pr_warn_elf(fmt, ...) \
156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
157
158static int init_output_elf(struct bpf_linker *linker, const char *file);
159
160static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
161 const struct bpf_linker_file_opts *opts,
162 struct src_obj *obj);
163static int linker_sanity_check_elf(struct src_obj *obj);
164static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
165static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
166static int linker_sanity_check_btf(struct src_obj *obj);
167static int linker_sanity_check_btf_ext(struct src_obj *obj);
168static int linker_fixup_btf(struct src_obj *obj);
169static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
170static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
171static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
172 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
173static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
174static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
175static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
176
177static int finalize_btf(struct bpf_linker *linker);
178static int finalize_btf_ext(struct bpf_linker *linker);
179
180void bpf_linker__free(struct bpf_linker *linker)
181{
182 int i;
183
184 if (!linker)
185 return;
186
187 free(linker->filename);
188
189 if (linker->elf)
190 elf_end(linker->elf);
191
192 if (linker->fd >= 0)
193 close(linker->fd);
194
195 strset__free(linker->strtab_strs);
196
197 btf__free(linker->btf);
198 btf_ext__free(linker->btf_ext);
199
200 for (i = 1; i < linker->sec_cnt; i++) {
201 struct dst_sec *sec = &linker->secs[i];
202
203 free(sec->sec_name);
204 free(sec->raw_data);
205 free(sec->sec_vars);
206
207 free(sec->func_info.recs);
208 free(sec->line_info.recs);
209 free(sec->core_relo_info.recs);
210 }
211 free(linker->secs);
212
213 free(linker->glob_syms);
214 free(linker);
215}
216
217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218{
219 struct bpf_linker *linker;
220 int err;
221
222 if (!OPTS_VALID(opts, bpf_linker_opts))
223 return errno = EINVAL, NULL;
224
225 if (elf_version(EV_CURRENT) == EV_NONE) {
226 pr_warn_elf("libelf initialization failed");
227 return errno = EINVAL, NULL;
228 }
229
230 linker = calloc(1, sizeof(*linker));
231 if (!linker)
232 return errno = ENOMEM, NULL;
233
234 linker->fd = -1;
235
236 err = init_output_elf(linker, filename);
237 if (err)
238 goto err_out;
239
240 return linker;
241
242err_out:
243 bpf_linker__free(linker);
244 return errno = -err, NULL;
245}
246
247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248{
249 struct dst_sec *secs = linker->secs, *sec;
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251
252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 if (!secs)
254 return NULL;
255
256 /* zero out newly allocated memory */
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258
259 linker->secs = secs;
260 linker->sec_cnt = new_cnt;
261
262 sec = &linker->secs[new_cnt - 1];
263 sec->id = new_cnt - 1;
264 sec->sec_name = strdup(sec_name);
265 if (!sec->sec_name)
266 return NULL;
267
268 return sec;
269}
270
271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272{
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 Elf64_Sym *syms, *sym;
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 if (!syms)
279 return NULL;
280
281 sym = &syms[sym_cnt];
282 memset(sym, 0, sizeof(*sym));
283
284 symtab->raw_data = syms;
285 symtab->sec_sz += sizeof(*sym);
286 symtab->shdr->sh_size += sizeof(*sym);
287 symtab->data->d_size += sizeof(*sym);
288
289 if (sym_idx)
290 *sym_idx = sym_cnt;
291
292 return sym;
293}
294
295static int init_output_elf(struct bpf_linker *linker, const char *file)
296{
297 int err, str_off;
298 Elf64_Sym *init_sym;
299 struct dst_sec *sec;
300
301 linker->filename = strdup(file);
302 if (!linker->filename)
303 return -ENOMEM;
304
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306 if (linker->fd < 0) {
307 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);
309 return err;
310 }
311
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 if (!linker->elf) {
314 pr_warn_elf("failed to create ELF object");
315 return -EINVAL;
316 }
317
318 /* ELF header */
319 linker->elf_hdr = elf64_newehdr(linker->elf);
320 if (!linker->elf_hdr) {
321 pr_warn_elf("failed to create ELF header");
322 return -EINVAL;
323 }
324
325 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;
327#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331#else
332#error "Unknown __BYTE_ORDER__"
333#endif
334
335 /* STRTAB */
336 /* initialize strset with an empty string to conform to ELF */
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 if (libbpf_get_error(linker->strtab_strs))
339 return libbpf_get_error(linker->strtab_strs);
340
341 sec = add_dst_sec(linker, ".strtab");
342 if (!sec)
343 return -ENOMEM;
344
345 sec->scn = elf_newscn(linker->elf);
346 if (!sec->scn) {
347 pr_warn_elf("failed to create STRTAB section");
348 return -EINVAL;
349 }
350
351 sec->shdr = elf64_getshdr(sec->scn);
352 if (!sec->shdr)
353 return -EINVAL;
354
355 sec->data = elf_newdata(sec->scn);
356 if (!sec->data) {
357 pr_warn_elf("failed to create STRTAB data");
358 return -EINVAL;
359 }
360
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 if (str_off < 0)
363 return str_off;
364
365 sec->sec_idx = elf_ndxscn(sec->scn);
366 linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 linker->strtab_sec_idx = sec->sec_idx;
368
369 sec->shdr->sh_name = str_off;
370 sec->shdr->sh_type = SHT_STRTAB;
371 sec->shdr->sh_flags = SHF_STRINGS;
372 sec->shdr->sh_offset = 0;
373 sec->shdr->sh_link = 0;
374 sec->shdr->sh_info = 0;
375 sec->shdr->sh_addralign = 1;
376 sec->shdr->sh_size = sec->sec_sz = 0;
377 sec->shdr->sh_entsize = 0;
378
379 /* SYMTAB */
380 sec = add_dst_sec(linker, ".symtab");
381 if (!sec)
382 return -ENOMEM;
383
384 sec->scn = elf_newscn(linker->elf);
385 if (!sec->scn) {
386 pr_warn_elf("failed to create SYMTAB section");
387 return -EINVAL;
388 }
389
390 sec->shdr = elf64_getshdr(sec->scn);
391 if (!sec->shdr)
392 return -EINVAL;
393
394 sec->data = elf_newdata(sec->scn);
395 if (!sec->data) {
396 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;
398 }
399
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)
402 return str_off;
403
404 sec->sec_idx = elf_ndxscn(sec->scn);
405 linker->symtab_sec_idx = sec->sec_idx;
406
407 sec->shdr->sh_name = str_off;
408 sec->shdr->sh_type = SHT_SYMTAB;
409 sec->shdr->sh_flags = 0;
410 sec->shdr->sh_offset = 0;
411 sec->shdr->sh_link = linker->strtab_sec_idx;
412 /* sh_info should be one greater than the index of the last local
413 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
414 */
415 sec->shdr->sh_info = 0;
416 sec->shdr->sh_addralign = 8;
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418
419 /* .BTF */
420 linker->btf = btf__new_empty();
421 err = libbpf_get_error(linker->btf);
422 if (err)
423 return err;
424
425 /* add the special all-zero symbol */
426 init_sym = add_new_sym(linker, NULL);
427 if (!init_sym)
428 return -EINVAL;
429
430 init_sym->st_name = 0;
431 init_sym->st_info = 0;
432 init_sym->st_other = 0;
433 init_sym->st_shndx = SHN_UNDEF;
434 init_sym->st_value = 0;
435 init_sym->st_size = 0;
436
437 return 0;
438}
439
440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 const struct bpf_linker_file_opts *opts)
442{
443 struct src_obj obj = {};
444 int err = 0;
445
446 if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 return libbpf_err(-EINVAL);
448
449 if (!linker->elf)
450 return libbpf_err(-EINVAL);
451
452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 err = err ?: linker_append_sec_data(linker, &obj);
454 err = err ?: linker_append_elf_syms(linker, &obj);
455 err = err ?: linker_append_elf_relos(linker, &obj);
456 err = err ?: linker_append_btf(linker, &obj);
457 err = err ?: linker_append_btf_ext(linker, &obj);
458
459 /* free up src_obj resources */
460 free(obj.btf_type_map);
461 btf__free(obj.btf);
462 btf_ext__free(obj.btf_ext);
463 free(obj.secs);
464 free(obj.sym_map);
465 if (obj.elf)
466 elf_end(obj.elf);
467 if (obj.fd >= 0)
468 close(obj.fd);
469
470 return libbpf_err(err);
471}
472
473static bool is_dwarf_sec_name(const char *name)
474{
475 /* approximation, but the actual list is too long */
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477}
478
479static bool is_ignored_sec(struct src_sec *sec)
480{
481 Elf64_Shdr *shdr = sec->shdr;
482 const char *name = sec->sec_name;
483
484 /* no special handling of .strtab */
485 if (shdr->sh_type == SHT_STRTAB)
486 return true;
487
488 /* ignore .llvm_addrsig section as well */
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 return true;
491
492 /* no subprograms will lead to an empty .text section, ignore it */
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 strcmp(sec->sec_name, ".text") == 0)
495 return true;
496
497 /* DWARF sections */
498 if (is_dwarf_sec_name(sec->sec_name))
499 return true;
500
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 name += sizeof(".rel") - 1;
503 /* DWARF section relocations */
504 if (is_dwarf_sec_name(name))
505 return true;
506
507 /* .BTF and .BTF.ext don't need relocations */
508 if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 return true;
511 }
512
513 return false;
514}
515
516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517{
518 struct src_sec *secs = obj->secs, *sec;
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520
521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 if (!secs)
523 return NULL;
524
525 /* zero out newly allocated memory */
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527
528 obj->secs = secs;
529 obj->sec_cnt = new_cnt;
530
531 sec = &obj->secs[new_cnt - 1];
532 sec->id = new_cnt - 1;
533 sec->sec_name = sec_name;
534
535 return sec;
536}
537
538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)
541{
542#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543 const int host_endianness = ELFDATA2LSB;
544#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545 const int host_endianness = ELFDATA2MSB;
546#else
547#error "Unknown __BYTE_ORDER__"
548#endif
549 int err = 0;
550 Elf_Scn *scn;
551 Elf_Data *data;
552 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;
554 struct src_sec *sec;
555
556 pr_debug("linker: adding object file '%s'...\n", filename);
557
558 obj->filename = filename;
559
560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561 if (obj->fd < 0) {
562 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);
564 return err;
565 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 if (!obj->elf) {
568 err = -errno;
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return err;
571 }
572
573 /* Sanity check ELF file high-level properties */
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 err = -errno;
577 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;
579 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 return err;
584 }
585 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 err = -EOPNOTSUPP;
589 pr_warn_elf("unsupported kind of ELF file %s", filename);
590 return err;
591 }
592
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 err = -errno;
595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 return err;
597 }
598
599 scn = NULL;
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 size_t sec_idx = elf_ndxscn(scn);
602 const char *sec_name;
603
604 shdr = elf64_getshdr(scn);
605 if (!shdr) {
606 err = -errno;
607 pr_warn_elf("failed to get section #%zu header for %s",
608 sec_idx, filename);
609 return err;
610 }
611
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 if (!sec_name) {
614 err = -errno;
615 pr_warn_elf("failed to get section #%zu name for %s",
616 sec_idx, filename);
617 return err;
618 }
619
620 data = elf_getdata(scn, 0);
621 if (!data) {
622 err = -errno;
623 pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 sec_idx, sec_name, filename);
625 return err;
626 }
627
628 sec = add_src_sec(obj, sec_name);
629 if (!sec)
630 return -ENOMEM;
631
632 sec->scn = scn;
633 sec->shdr = shdr;
634 sec->data = data;
635 sec->sec_idx = elf_ndxscn(scn);
636
637 if (is_ignored_sec(sec)) {
638 sec->skipped = true;
639 continue;
640 }
641
642 switch (shdr->sh_type) {
643 case SHT_SYMTAB:
644 if (obj->symtab_sec_idx) {
645 err = -EOPNOTSUPP;
646 pr_warn("multiple SYMTAB sections found, not supported\n");
647 return err;
648 }
649 obj->symtab_sec_idx = sec_idx;
650 break;
651 case SHT_STRTAB:
652 /* we'll construct our own string table */
653 break;
654 case SHT_PROGBITS:
655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);
658 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 return err;
661 }
662 sec->skipped = true;
663 continue;
664 }
665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);
668 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675
676 /* data & code */
677 break;
678 case SHT_NOBITS:
679 /* BSS */
680 break;
681 case SHT_REL:
682 /* relocations */
683 break;
684 default:
685 pr_warn("unrecognized section #%zu (%s) in %s\n",
686 sec_idx, sec_name, filename);
687 err = -EINVAL;
688 return err;
689 }
690 }
691
692 err = err ?: linker_sanity_check_elf(obj);
693 err = err ?: linker_sanity_check_btf(obj);
694 err = err ?: linker_sanity_check_btf_ext(obj);
695 err = err ?: linker_fixup_btf(obj);
696
697 return err;
698}
699
700static int linker_sanity_check_elf(struct src_obj *obj)
701{
702 struct src_sec *sec;
703 int i, err;
704
705 if (!obj->symtab_sec_idx) {
706 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
707 return -EINVAL;
708 }
709 if (!obj->shstrs_sec_idx) {
710 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
711 return -EINVAL;
712 }
713
714 for (i = 1; i < obj->sec_cnt; i++) {
715 sec = &obj->secs[i];
716
717 if (sec->sec_name[0] == '\0') {
718 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
719 return -EINVAL;
720 }
721
722 if (is_dwarf_sec_name(sec->sec_name))
723 continue;
724
725 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
726 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
727 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
728 obj->filename);
729 return -EINVAL;
730 }
731 if (sec->shdr->sh_addralign != sec->data->d_align) {
732 pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
733 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
734 (long long unsigned)sec->data->d_align, obj->filename);
735 return -EINVAL;
736 }
737
738 if (sec->shdr->sh_size != sec->data->d_size) {
739 pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
740 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
741 (long long unsigned)sec->data->d_size, obj->filename);
742 return -EINVAL;
743 }
744
745 switch (sec->shdr->sh_type) {
746 case SHT_SYMTAB:
747 err = linker_sanity_check_elf_symtab(obj, sec);
748 if (err)
749 return err;
750 break;
751 case SHT_STRTAB:
752 break;
753 case SHT_PROGBITS:
754 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
755 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
756 pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
757 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
758 obj->filename);
759 return -EINVAL;
760 }
761 }
762 break;
763 case SHT_NOBITS:
764 break;
765 case SHT_REL:
766 err = linker_sanity_check_elf_relos(obj, sec);
767 if (err)
768 return err;
769 break;
770 case SHT_LLVM_ADDRSIG:
771 break;
772 default:
773 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
774 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
775 return -EINVAL;
776 }
777 }
778
779 return 0;
780}
781
782static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
783{
784 struct src_sec *link_sec;
785 Elf64_Sym *sym;
786 int i, n;
787
788 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
789 return -EINVAL;
790 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
791 return -EINVAL;
792
793 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
794 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
795 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
796 return -EINVAL;
797 }
798 link_sec = &obj->secs[sec->shdr->sh_link];
799 if (link_sec->shdr->sh_type != SHT_STRTAB) {
800 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
801 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
802 return -EINVAL;
803 }
804
805 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
806 sym = sec->data->d_buf;
807 for (i = 0; i < n; i++, sym++) {
808 int sym_type = ELF64_ST_TYPE(sym->st_info);
809 int sym_bind = ELF64_ST_BIND(sym->st_info);
810 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
811
812 if (i == 0) {
813 if (sym->st_name != 0 || sym->st_info != 0
814 || sym->st_other != 0 || sym->st_shndx != 0
815 || sym->st_value != 0 || sym->st_size != 0) {
816 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
817 return -EINVAL;
818 }
819 continue;
820 }
821 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
822 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
823 i, sec->sec_idx, sym_bind);
824 return -EINVAL;
825 }
826 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
827 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
828 i, sec->sec_idx, sym_vis);
829 return -EINVAL;
830 }
831 if (sym->st_shndx == 0) {
832 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
833 || sym->st_value != 0 || sym->st_size != 0) {
834 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
835 i, obj->filename);
836
837 return -EINVAL;
838 }
839 continue;
840 }
841 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
842 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
843 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
844 return -EINVAL;
845 }
846 if (sym_type == STT_SECTION) {
847 if (sym->st_value != 0)
848 return -EINVAL;
849 continue;
850 }
851 }
852
853 return 0;
854}
855
856static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
857{
858 struct src_sec *link_sec, *sym_sec;
859 Elf64_Rel *relo;
860 int i, n;
861
862 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
863 return -EINVAL;
864 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
865 return -EINVAL;
866
867 /* SHT_REL's sh_link should point to SYMTAB */
868 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
869 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
870 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
871 return -EINVAL;
872 }
873
874 /* SHT_REL's sh_info points to relocated section */
875 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
876 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
877 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
878 return -EINVAL;
879 }
880 link_sec = &obj->secs[sec->shdr->sh_info];
881
882 /* .rel<secname> -> <secname> pattern is followed */
883 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
884 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
885 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
886 sec->sec_idx, obj->filename);
887 return -EINVAL;
888 }
889
890 /* don't further validate relocations for ignored sections */
891 if (link_sec->skipped)
892 return 0;
893
894 /* relocatable section is data or instructions */
895 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
896 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
897 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
898 return -EINVAL;
899 }
900
901 /* check sanity of each relocation */
902 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
903 relo = sec->data->d_buf;
904 sym_sec = &obj->secs[obj->symtab_sec_idx];
905 for (i = 0; i < n; i++, relo++) {
906 size_t sym_idx = ELF64_R_SYM(relo->r_info);
907 size_t sym_type = ELF64_R_TYPE(relo->r_info);
908
909 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
910 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
911 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
912 i, sec->sec_idx, sym_type, obj->filename);
913 return -EINVAL;
914 }
915
916 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
917 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
918 i, sec->sec_idx, sym_idx, obj->filename);
919 return -EINVAL;
920 }
921
922 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
923 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
924 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
925 i, sec->sec_idx, sym_idx, obj->filename);
926 return -EINVAL;
927 }
928 }
929 }
930
931 return 0;
932}
933
934static int check_btf_type_id(__u32 *type_id, void *ctx)
935{
936 struct btf *btf = ctx;
937
938 if (*type_id >= btf__type_cnt(btf))
939 return -EINVAL;
940
941 return 0;
942}
943
944static int check_btf_str_off(__u32 *str_off, void *ctx)
945{
946 struct btf *btf = ctx;
947 const char *s;
948
949 s = btf__str_by_offset(btf, *str_off);
950
951 if (!s)
952 return -EINVAL;
953
954 return 0;
955}
956
957static int linker_sanity_check_btf(struct src_obj *obj)
958{
959 struct btf_type *t;
960 int i, n, err = 0;
961
962 if (!obj->btf)
963 return 0;
964
965 n = btf__type_cnt(obj->btf);
966 for (i = 1; i < n; i++) {
967 t = btf_type_by_id(obj->btf, i);
968
969 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
970 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
971 if (err)
972 return err;
973 }
974
975 return 0;
976}
977
978static int linker_sanity_check_btf_ext(struct src_obj *obj)
979{
980 int err = 0;
981
982 if (!obj->btf_ext)
983 return 0;
984
985 /* can't use .BTF.ext without .BTF */
986 if (!obj->btf)
987 return -EINVAL;
988
989 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
990 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
991 if (err)
992 return err;
993
994 return 0;
995}
996
997static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
998{
999 Elf_Scn *scn;
1000 Elf_Data *data;
1001 Elf64_Shdr *shdr;
1002 int name_off;
1003
1004 dst_sec->sec_sz = 0;
1005 dst_sec->sec_idx = 0;
1006 dst_sec->ephemeral = src_sec->ephemeral;
1007
1008 /* ephemeral sections are just thin section shells lacking most parts */
1009 if (src_sec->ephemeral)
1010 return 0;
1011
1012 scn = elf_newscn(linker->elf);
1013 if (!scn)
1014 return -ENOMEM;
1015 data = elf_newdata(scn);
1016 if (!data)
1017 return -ENOMEM;
1018 shdr = elf64_getshdr(scn);
1019 if (!shdr)
1020 return -ENOMEM;
1021
1022 dst_sec->scn = scn;
1023 dst_sec->shdr = shdr;
1024 dst_sec->data = data;
1025 dst_sec->sec_idx = elf_ndxscn(scn);
1026
1027 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1028 if (name_off < 0)
1029 return name_off;
1030
1031 shdr->sh_name = name_off;
1032 shdr->sh_type = src_sec->shdr->sh_type;
1033 shdr->sh_flags = src_sec->shdr->sh_flags;
1034 shdr->sh_size = 0;
1035 /* sh_link and sh_info have different meaning for different types of
1036 * sections, so we leave it up to the caller code to fill them in, if
1037 * necessary
1038 */
1039 shdr->sh_link = 0;
1040 shdr->sh_info = 0;
1041 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1042 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1043
1044 data->d_type = src_sec->data->d_type;
1045 data->d_size = 0;
1046 data->d_buf = NULL;
1047 data->d_align = src_sec->data->d_align;
1048 data->d_off = 0;
1049
1050 return 0;
1051}
1052
1053static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1054{
1055 struct dst_sec *sec;
1056 int i;
1057
1058 for (i = 1; i < linker->sec_cnt; i++) {
1059 sec = &linker->secs[i];
1060
1061 if (strcmp(sec->sec_name, sec_name) == 0)
1062 return sec;
1063 }
1064
1065 return NULL;
1066}
1067
1068static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1069{
1070 if (dst->ephemeral || src->ephemeral)
1071 return true;
1072
1073 if (dst->shdr->sh_type != src->shdr->sh_type) {
1074 pr_warn("sec %s types mismatch\n", dst->sec_name);
1075 return false;
1076 }
1077 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1078 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1079 return false;
1080 }
1081 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1082 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1083 return false;
1084 }
1085
1086 return true;
1087}
1088
1089static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1090{
1091 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1092 return false;
1093 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1094 return false;
1095 return true;
1096}
1097
1098static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1099{
1100 void *tmp;
1101 size_t dst_align, src_align;
1102 size_t dst_align_sz, dst_final_sz;
1103 int err;
1104
1105 /* Ephemeral source section doesn't contribute anything to ELF
1106 * section data.
1107 */
1108 if (src->ephemeral)
1109 return 0;
1110
1111 /* Some sections (like .maps) can contain both externs (and thus be
1112 * ephemeral) and non-externs (map definitions). So it's possible that
1113 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1114 * first non-ephemeral entity appears. In such case, we add ELF
1115 * section, data, etc.
1116 */
1117 if (dst->ephemeral) {
1118 err = init_sec(linker, dst, src);
1119 if (err)
1120 return err;
1121 }
1122
1123 dst_align = dst->shdr->sh_addralign;
1124 src_align = src->shdr->sh_addralign;
1125 if (dst_align == 0)
1126 dst_align = 1;
1127 if (dst_align < src_align)
1128 dst_align = src_align;
1129
1130 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1131
1132 /* no need to re-align final size */
1133 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1134
1135 if (src->shdr->sh_type != SHT_NOBITS) {
1136 tmp = realloc(dst->raw_data, dst_final_sz);
1137 /* If dst_align_sz == 0, realloc() behaves in a special way:
1138 * 1. When dst->raw_data is NULL it returns:
1139 * "either NULL or a pointer suitable to be passed to free()" [1].
1140 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1141 * thus invalidating any "pointer suitable to be passed to free()" obtained
1142 * at step (1).
1143 *
1144 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1145 * dst->raw_data would be freed again in bpf_linker__free().
1146 *
1147 * [1] man 3 realloc
1148 */
1149 if (!tmp && dst_align_sz > 0)
1150 return -ENOMEM;
1151 dst->raw_data = tmp;
1152
1153 /* pad dst section, if it's alignment forced size increase */
1154 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1155 /* now copy src data at a properly aligned offset */
1156 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1157 }
1158
1159 dst->sec_sz = dst_final_sz;
1160 dst->shdr->sh_size = dst_final_sz;
1161 dst->data->d_size = dst_final_sz;
1162
1163 dst->shdr->sh_addralign = dst_align;
1164 dst->data->d_align = dst_align;
1165
1166 src->dst_off = dst_align_sz;
1167
1168 return 0;
1169}
1170
1171static bool is_data_sec(struct src_sec *sec)
1172{
1173 if (!sec || sec->skipped)
1174 return false;
1175 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1176 if (sec->ephemeral)
1177 return true;
1178 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1179}
1180
1181static bool is_relo_sec(struct src_sec *sec)
1182{
1183 if (!sec || sec->skipped || sec->ephemeral)
1184 return false;
1185 return sec->shdr->sh_type == SHT_REL;
1186}
1187
1188static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1189{
1190 int i, err;
1191
1192 for (i = 1; i < obj->sec_cnt; i++) {
1193 struct src_sec *src_sec;
1194 struct dst_sec *dst_sec;
1195
1196 src_sec = &obj->secs[i];
1197 if (!is_data_sec(src_sec))
1198 continue;
1199
1200 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1201 if (!dst_sec) {
1202 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1203 if (!dst_sec)
1204 return -ENOMEM;
1205 err = init_sec(linker, dst_sec, src_sec);
1206 if (err) {
1207 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1208 return err;
1209 }
1210 } else {
1211 if (!secs_match(dst_sec, src_sec)) {
1212 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1213 return -1;
1214 }
1215
1216 /* "license" and "version" sections are deduped */
1217 if (strcmp(src_sec->sec_name, "license") == 0
1218 || strcmp(src_sec->sec_name, "version") == 0) {
1219 if (!sec_content_is_same(dst_sec, src_sec)) {
1220 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1221 return -EINVAL;
1222 }
1223 src_sec->skipped = true;
1224 src_sec->dst_id = dst_sec->id;
1225 continue;
1226 }
1227 }
1228
1229 /* record mapped section index */
1230 src_sec->dst_id = dst_sec->id;
1231
1232 err = extend_sec(linker, dst_sec, src_sec);
1233 if (err)
1234 return err;
1235 }
1236
1237 return 0;
1238}
1239
1240static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1241{
1242 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1243 Elf64_Sym *sym = symtab->data->d_buf;
1244 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1245 int str_sec_idx = symtab->shdr->sh_link;
1246 const char *sym_name;
1247
1248 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1249 if (!obj->sym_map)
1250 return -ENOMEM;
1251
1252 for (i = 0; i < n; i++, sym++) {
1253 /* We already validated all-zero symbol #0 and we already
1254 * appended it preventively to the final SYMTAB, so skip it.
1255 */
1256 if (i == 0)
1257 continue;
1258
1259 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1260 if (!sym_name) {
1261 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1262 return -EINVAL;
1263 }
1264
1265 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1266 if (err)
1267 return err;
1268 }
1269
1270 return 0;
1271}
1272
1273static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1274{
1275 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1276 Elf64_Sym *syms = symtab->raw_data;
1277
1278 return &syms[sym_idx];
1279}
1280
1281static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1282{
1283 struct glob_sym *glob_sym;
1284 const char *name;
1285 int i;
1286
1287 for (i = 0; i < linker->glob_sym_cnt; i++) {
1288 glob_sym = &linker->glob_syms[i];
1289 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1290
1291 if (strcmp(name, sym_name) == 0)
1292 return glob_sym;
1293 }
1294
1295 return NULL;
1296}
1297
1298static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1299{
1300 struct glob_sym *syms, *sym;
1301
1302 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1303 sizeof(*linker->glob_syms));
1304 if (!syms)
1305 return NULL;
1306
1307 sym = &syms[linker->glob_sym_cnt];
1308 memset(sym, 0, sizeof(*sym));
1309 sym->var_idx = -1;
1310
1311 linker->glob_syms = syms;
1312 linker->glob_sym_cnt++;
1313
1314 return sym;
1315}
1316
1317static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1318 const struct btf *btf1, __u32 id1,
1319 const struct btf *btf2, __u32 id2)
1320{
1321 const struct btf_type *t1, *t2;
1322 bool is_static1, is_static2;
1323 const char *n1, *n2;
1324 int i, n;
1325
1326recur:
1327 n1 = n2 = NULL;
1328 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1329 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1330
1331 /* check if only one side is FWD, otherwise handle with common logic */
1332 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1333 n1 = btf__str_by_offset(btf1, t1->name_off);
1334 n2 = btf__str_by_offset(btf2, t2->name_off);
1335 if (strcmp(n1, n2) != 0) {
1336 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1337 sym_name, n1, n2);
1338 return false;
1339 }
1340 /* validate if FWD kind matches concrete kind */
1341 if (btf_is_fwd(t1)) {
1342 if (btf_kflag(t1) && btf_is_union(t2))
1343 return true;
1344 if (!btf_kflag(t1) && btf_is_struct(t2))
1345 return true;
1346 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1347 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1348 } else {
1349 if (btf_kflag(t2) && btf_is_union(t1))
1350 return true;
1351 if (!btf_kflag(t2) && btf_is_struct(t1))
1352 return true;
1353 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1354 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1355 }
1356 return false;
1357 }
1358
1359 if (btf_kind(t1) != btf_kind(t2)) {
1360 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1361 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1362 return false;
1363 }
1364
1365 switch (btf_kind(t1)) {
1366 case BTF_KIND_STRUCT:
1367 case BTF_KIND_UNION:
1368 case BTF_KIND_ENUM:
1369 case BTF_KIND_ENUM64:
1370 case BTF_KIND_FWD:
1371 case BTF_KIND_FUNC:
1372 case BTF_KIND_VAR:
1373 n1 = btf__str_by_offset(btf1, t1->name_off);
1374 n2 = btf__str_by_offset(btf2, t2->name_off);
1375 if (strcmp(n1, n2) != 0) {
1376 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1377 sym_name, btf_kind_str(t1), n1, n2);
1378 return false;
1379 }
1380 break;
1381 default:
1382 break;
1383 }
1384
1385 switch (btf_kind(t1)) {
1386 case BTF_KIND_UNKN: /* void */
1387 case BTF_KIND_FWD:
1388 return true;
1389 case BTF_KIND_INT:
1390 case BTF_KIND_FLOAT:
1391 case BTF_KIND_ENUM:
1392 case BTF_KIND_ENUM64:
1393 /* ignore encoding for int and enum values for enum */
1394 if (t1->size != t2->size) {
1395 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1396 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1397 return false;
1398 }
1399 return true;
1400 case BTF_KIND_PTR:
1401 /* just validate overall shape of the referenced type, so no
1402 * contents comparison for struct/union, and allowd fwd vs
1403 * struct/union
1404 */
1405 exact = false;
1406 id1 = t1->type;
1407 id2 = t2->type;
1408 goto recur;
1409 case BTF_KIND_ARRAY:
1410 /* ignore index type and array size */
1411 id1 = btf_array(t1)->type;
1412 id2 = btf_array(t2)->type;
1413 goto recur;
1414 case BTF_KIND_FUNC:
1415 /* extern and global linkages are compatible */
1416 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1417 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1418 if (is_static1 != is_static2) {
1419 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1420 return false;
1421 }
1422
1423 id1 = t1->type;
1424 id2 = t2->type;
1425 goto recur;
1426 case BTF_KIND_VAR:
1427 /* extern and global linkages are compatible */
1428 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1429 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1430 if (is_static1 != is_static2) {
1431 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1432 return false;
1433 }
1434
1435 id1 = t1->type;
1436 id2 = t2->type;
1437 goto recur;
1438 case BTF_KIND_STRUCT:
1439 case BTF_KIND_UNION: {
1440 const struct btf_member *m1, *m2;
1441
1442 if (!exact)
1443 return true;
1444
1445 if (btf_vlen(t1) != btf_vlen(t2)) {
1446 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1447 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1448 return false;
1449 }
1450
1451 n = btf_vlen(t1);
1452 m1 = btf_members(t1);
1453 m2 = btf_members(t2);
1454 for (i = 0; i < n; i++, m1++, m2++) {
1455 n1 = btf__str_by_offset(btf1, m1->name_off);
1456 n2 = btf__str_by_offset(btf2, m2->name_off);
1457 if (strcmp(n1, n2) != 0) {
1458 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1459 sym_name, i, n1, n2);
1460 return false;
1461 }
1462 if (m1->offset != m2->offset) {
1463 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1464 sym_name, i, n1);
1465 return false;
1466 }
1467 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1468 return false;
1469 }
1470
1471 return true;
1472 }
1473 case BTF_KIND_FUNC_PROTO: {
1474 const struct btf_param *m1, *m2;
1475
1476 if (btf_vlen(t1) != btf_vlen(t2)) {
1477 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1478 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1479 return false;
1480 }
1481
1482 n = btf_vlen(t1);
1483 m1 = btf_params(t1);
1484 m2 = btf_params(t2);
1485 for (i = 0; i < n; i++, m1++, m2++) {
1486 /* ignore func arg names */
1487 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1488 return false;
1489 }
1490
1491 /* now check return type as well */
1492 id1 = t1->type;
1493 id2 = t2->type;
1494 goto recur;
1495 }
1496
1497 /* skip_mods_and_typedefs() make this impossible */
1498 case BTF_KIND_TYPEDEF:
1499 case BTF_KIND_VOLATILE:
1500 case BTF_KIND_CONST:
1501 case BTF_KIND_RESTRICT:
1502 /* DATASECs are never compared with each other */
1503 case BTF_KIND_DATASEC:
1504 default:
1505 pr_warn("global '%s': unsupported BTF kind %s\n",
1506 sym_name, btf_kind_str(t1));
1507 return false;
1508 }
1509}
1510
1511static bool map_defs_match(const char *sym_name,
1512 const struct btf *main_btf,
1513 const struct btf_map_def *main_def,
1514 const struct btf_map_def *main_inner_def,
1515 const struct btf *extra_btf,
1516 const struct btf_map_def *extra_def,
1517 const struct btf_map_def *extra_inner_def)
1518{
1519 const char *reason;
1520
1521 if (main_def->map_type != extra_def->map_type) {
1522 reason = "type";
1523 goto mismatch;
1524 }
1525
1526 /* check key type/size match */
1527 if (main_def->key_size != extra_def->key_size) {
1528 reason = "key_size";
1529 goto mismatch;
1530 }
1531 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1532 reason = "key type";
1533 goto mismatch;
1534 }
1535 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1536 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1537 main_btf, main_def->key_type_id,
1538 extra_btf, extra_def->key_type_id)) {
1539 reason = "key type";
1540 goto mismatch;
1541 }
1542
1543 /* validate value type/size match */
1544 if (main_def->value_size != extra_def->value_size) {
1545 reason = "value_size";
1546 goto mismatch;
1547 }
1548 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1549 reason = "value type";
1550 goto mismatch;
1551 }
1552 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1553 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1554 main_btf, main_def->value_type_id,
1555 extra_btf, extra_def->value_type_id)) {
1556 reason = "key type";
1557 goto mismatch;
1558 }
1559
1560 if (main_def->max_entries != extra_def->max_entries) {
1561 reason = "max_entries";
1562 goto mismatch;
1563 }
1564 if (main_def->map_flags != extra_def->map_flags) {
1565 reason = "map_flags";
1566 goto mismatch;
1567 }
1568 if (main_def->numa_node != extra_def->numa_node) {
1569 reason = "numa_node";
1570 goto mismatch;
1571 }
1572 if (main_def->pinning != extra_def->pinning) {
1573 reason = "pinning";
1574 goto mismatch;
1575 }
1576
1577 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1578 reason = "inner map";
1579 goto mismatch;
1580 }
1581
1582 if (main_def->parts & MAP_DEF_INNER_MAP) {
1583 char inner_map_name[128];
1584
1585 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1586
1587 return map_defs_match(inner_map_name,
1588 main_btf, main_inner_def, NULL,
1589 extra_btf, extra_inner_def, NULL);
1590 }
1591
1592 return true;
1593
1594mismatch:
1595 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1596 return false;
1597}
1598
1599static bool glob_map_defs_match(const char *sym_name,
1600 struct bpf_linker *linker, struct glob_sym *glob_sym,
1601 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1602{
1603 struct btf_map_def dst_def = {}, dst_inner_def = {};
1604 struct btf_map_def src_def = {}, src_inner_def = {};
1605 const struct btf_type *t;
1606 int err;
1607
1608 t = btf__type_by_id(obj->btf, btf_id);
1609 if (!btf_is_var(t)) {
1610 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1611 return false;
1612 }
1613 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1614
1615 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1616 if (err) {
1617 pr_warn("global '%s': invalid map definition\n", sym_name);
1618 return false;
1619 }
1620
1621 /* re-parse existing map definition */
1622 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1623 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1624 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1625 if (err) {
1626 /* this should not happen, because we already validated it */
1627 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1628 return false;
1629 }
1630
1631 /* Currently extern map definition has to be complete and match
1632 * concrete map definition exactly. This restriction might be lifted
1633 * in the future.
1634 */
1635 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1636 obj->btf, &src_def, &src_inner_def);
1637}
1638
1639static bool glob_syms_match(const char *sym_name,
1640 struct bpf_linker *linker, struct glob_sym *glob_sym,
1641 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1642{
1643 const struct btf_type *src_t;
1644
1645 /* if we are dealing with externs, BTF types describing both global
1646 * and extern VARs/FUNCs should be completely present in all files
1647 */
1648 if (!glob_sym->btf_id || !btf_id) {
1649 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1650 return false;
1651 }
1652
1653 src_t = btf__type_by_id(obj->btf, btf_id);
1654 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1655 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1656 btf_kind_str(src_t), sym_name);
1657 return false;
1658 }
1659
1660 /* deal with .maps definitions specially */
1661 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1662 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1663
1664 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1665 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1666 return false;
1667
1668 return true;
1669}
1670
1671static bool btf_is_non_static(const struct btf_type *t)
1672{
1673 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1674 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1675}
1676
1677static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1678 int *out_btf_sec_id, int *out_btf_id)
1679{
1680 int i, j, n, m, btf_id = 0;
1681 const struct btf_type *t;
1682 const struct btf_var_secinfo *vi;
1683 const char *name;
1684
1685 if (!obj->btf) {
1686 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1687 return -EINVAL;
1688 }
1689
1690 n = btf__type_cnt(obj->btf);
1691 for (i = 1; i < n; i++) {
1692 t = btf__type_by_id(obj->btf, i);
1693
1694 /* some global and extern FUNCs and VARs might not be associated with any
1695 * DATASEC, so try to detect them in the same pass
1696 */
1697 if (btf_is_non_static(t)) {
1698 name = btf__str_by_offset(obj->btf, t->name_off);
1699 if (strcmp(name, sym_name) != 0)
1700 continue;
1701
1702 /* remember and still try to find DATASEC */
1703 btf_id = i;
1704 continue;
1705 }
1706
1707 if (!btf_is_datasec(t))
1708 continue;
1709
1710 vi = btf_var_secinfos(t);
1711 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1712 t = btf__type_by_id(obj->btf, vi->type);
1713 name = btf__str_by_offset(obj->btf, t->name_off);
1714
1715 if (strcmp(name, sym_name) != 0)
1716 continue;
1717 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1718 continue;
1719 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1720 continue;
1721
1722 if (btf_id && btf_id != vi->type) {
1723 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1724 sym_name, btf_id, vi->type);
1725 return -EINVAL;
1726 }
1727
1728 *out_btf_sec_id = i;
1729 *out_btf_id = vi->type;
1730
1731 return 0;
1732 }
1733 }
1734
1735 /* free-floating extern or global FUNC */
1736 if (btf_id) {
1737 *out_btf_sec_id = 0;
1738 *out_btf_id = btf_id;
1739 return 0;
1740 }
1741
1742 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1743 return -ENOENT;
1744}
1745
1746static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1747{
1748 struct src_sec *sec;
1749 int i;
1750
1751 for (i = 1; i < obj->sec_cnt; i++) {
1752 sec = &obj->secs[i];
1753
1754 if (strcmp(sec->sec_name, sec_name) == 0)
1755 return sec;
1756 }
1757
1758 return NULL;
1759}
1760
1761static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1762 struct btf *src_btf, int src_id)
1763{
1764 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1765 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1766 struct btf_param *src_p, *dst_p;
1767 const char *s;
1768 int i, n, off;
1769
1770 /* We already made sure that source and destination types (FUNC or
1771 * VAR) match in terms of types and argument names.
1772 */
1773 if (btf_is_var(dst_t)) {
1774 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1775 return 0;
1776 }
1777
1778 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1779
1780 /* now onto FUNC_PROTO types */
1781 src_t = btf_type_by_id(src_btf, src_t->type);
1782 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1783
1784 /* Fill in all the argument names, which for extern FUNCs are missing.
1785 * We'll end up with two copies of FUNCs/VARs for externs, but that
1786 * will be taken care of by BTF dedup at the very end.
1787 * It might be that BTF types for extern in one file has less/more BTF
1788 * information (e.g., FWD instead of full STRUCT/UNION information),
1789 * but that should be (in most cases, subject to BTF dedup rules)
1790 * handled and resolved by BTF dedup algorithm as well, so we won't
1791 * worry about it. Our only job is to make sure that argument names
1792 * are populated on both sides, otherwise BTF dedup will pedantically
1793 * consider them different.
1794 */
1795 src_p = btf_params(src_t);
1796 dst_p = btf_params(dst_t);
1797 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1798 if (!src_p->name_off)
1799 continue;
1800
1801 /* src_btf has more complete info, so add name to dst_btf */
1802 s = btf__str_by_offset(src_btf, src_p->name_off);
1803 off = btf__add_str(dst_btf, s);
1804 if (off < 0)
1805 return off;
1806 dst_p->name_off = off;
1807 }
1808 return 0;
1809}
1810
1811static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1812{
1813 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1814}
1815
1816static void sym_update_type(Elf64_Sym *sym, int sym_type)
1817{
1818 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1819}
1820
1821static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1822{
1823 /* libelf doesn't provide setters for ST_VISIBILITY,
1824 * but it is stored in the lower 2 bits of st_other
1825 */
1826 sym->st_other &= ~0x03;
1827 sym->st_other |= sym_vis;
1828}
1829
1830static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1831 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1832{
1833 struct src_sec *src_sec = NULL;
1834 struct dst_sec *dst_sec = NULL;
1835 struct glob_sym *glob_sym = NULL;
1836 int name_off, sym_type, sym_bind, sym_vis, err;
1837 int btf_sec_id = 0, btf_id = 0;
1838 size_t dst_sym_idx;
1839 Elf64_Sym *dst_sym;
1840 bool sym_is_extern;
1841
1842 sym_type = ELF64_ST_TYPE(sym->st_info);
1843 sym_bind = ELF64_ST_BIND(sym->st_info);
1844 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1845 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1846
1847 if (sym_is_extern) {
1848 if (!obj->btf) {
1849 pr_warn("externs without BTF info are not supported\n");
1850 return -ENOTSUP;
1851 }
1852 } else if (sym->st_shndx < SHN_LORESERVE) {
1853 src_sec = &obj->secs[sym->st_shndx];
1854 if (src_sec->skipped)
1855 return 0;
1856 dst_sec = &linker->secs[src_sec->dst_id];
1857
1858 /* allow only one STT_SECTION symbol per section */
1859 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1860 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1861 return 0;
1862 }
1863 }
1864
1865 if (sym_bind == STB_LOCAL)
1866 goto add_sym;
1867
1868 /* find matching BTF info */
1869 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1870 if (err)
1871 return err;
1872
1873 if (sym_is_extern && btf_sec_id) {
1874 const char *sec_name = NULL;
1875 const struct btf_type *t;
1876
1877 t = btf__type_by_id(obj->btf, btf_sec_id);
1878 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1879
1880 /* Clang puts unannotated extern vars into
1881 * '.extern' BTF DATASEC. Treat them the same
1882 * as unannotated extern funcs (which are
1883 * currently not put into any DATASECs).
1884 * Those don't have associated src_sec/dst_sec.
1885 */
1886 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1887 src_sec = find_src_sec_by_name(obj, sec_name);
1888 if (!src_sec) {
1889 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1890 return -ENOENT;
1891 }
1892 dst_sec = &linker->secs[src_sec->dst_id];
1893 }
1894 }
1895
1896 glob_sym = find_glob_sym(linker, sym_name);
1897 if (glob_sym) {
1898 /* Preventively resolve to existing symbol. This is
1899 * needed for further relocation symbol remapping in
1900 * the next step of linking.
1901 */
1902 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1903
1904 /* If both symbols are non-externs, at least one of
1905 * them has to be STB_WEAK, otherwise they are in
1906 * a conflict with each other.
1907 */
1908 if (!sym_is_extern && !glob_sym->is_extern
1909 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1910 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1911 src_sym_idx, sym_name, obj->filename);
1912 return -EINVAL;
1913 }
1914
1915 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1916 return -EINVAL;
1917
1918 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1919
1920 /* If new symbol is strong, then force dst_sym to be strong as
1921 * well; this way a mix of weak and non-weak extern
1922 * definitions will end up being strong.
1923 */
1924 if (sym_bind == STB_GLOBAL) {
1925 /* We still need to preserve type (NOTYPE or
1926 * OBJECT/FUNC, depending on whether the symbol is
1927 * extern or not)
1928 */
1929 sym_update_bind(dst_sym, STB_GLOBAL);
1930 glob_sym->is_weak = false;
1931 }
1932
1933 /* Non-default visibility is "contaminating", with stricter
1934 * visibility overwriting more permissive ones, even if more
1935 * permissive visibility comes from just an extern definition.
1936 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1937 * ensured by ELF symbol sanity checks above.
1938 */
1939 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1940 sym_update_visibility(dst_sym, sym_vis);
1941
1942 /* If the new symbol is extern, then regardless if
1943 * existing symbol is extern or resolved global, just
1944 * keep the existing one untouched.
1945 */
1946 if (sym_is_extern)
1947 return 0;
1948
1949 /* If existing symbol is a strong resolved symbol, bail out,
1950 * because we lost resolution battle have nothing to
1951 * contribute. We already checked abover that there is no
1952 * strong-strong conflict. We also already tightened binding
1953 * and visibility, so nothing else to contribute at that point.
1954 */
1955 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1956 return 0;
1957
1958 /* At this point, new symbol is strong non-extern,
1959 * so overwrite glob_sym with new symbol information.
1960 * Preserve binding and visibility.
1961 */
1962 sym_update_type(dst_sym, sym_type);
1963 dst_sym->st_shndx = dst_sec->sec_idx;
1964 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1965 dst_sym->st_size = sym->st_size;
1966
1967 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1968 glob_sym->sec_id = dst_sec->id;
1969 glob_sym->is_extern = false;
1970
1971 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1972 obj->btf, btf_id))
1973 return -EINVAL;
1974
1975 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1976 glob_sym->underlying_btf_id = 0;
1977
1978 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1979 return 0;
1980 }
1981
1982add_sym:
1983 name_off = strset__add_str(linker->strtab_strs, sym_name);
1984 if (name_off < 0)
1985 return name_off;
1986
1987 dst_sym = add_new_sym(linker, &dst_sym_idx);
1988 if (!dst_sym)
1989 return -ENOMEM;
1990
1991 dst_sym->st_name = name_off;
1992 dst_sym->st_info = sym->st_info;
1993 dst_sym->st_other = sym->st_other;
1994 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1995 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1996 dst_sym->st_size = sym->st_size;
1997
1998 obj->sym_map[src_sym_idx] = dst_sym_idx;
1999
2000 if (sym_type == STT_SECTION && dst_sym) {
2001 dst_sec->sec_sym_idx = dst_sym_idx;
2002 dst_sym->st_value = 0;
2003 }
2004
2005 if (sym_bind != STB_LOCAL) {
2006 glob_sym = add_glob_sym(linker);
2007 if (!glob_sym)
2008 return -ENOMEM;
2009
2010 glob_sym->sym_idx = dst_sym_idx;
2011 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2012 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2013 * sec_idx (as they don't have corresponding ELF section), but
2014 * still have id. .extern doesn't have even ephemeral section
2015 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2016 */
2017 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2018 glob_sym->name_off = name_off;
2019 /* we will fill btf_id in during BTF merging step */
2020 glob_sym->btf_id = 0;
2021 glob_sym->is_extern = sym_is_extern;
2022 glob_sym->is_weak = sym_bind == STB_WEAK;
2023 }
2024
2025 return 0;
2026}
2027
2028static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2029{
2030 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2031 int i, err;
2032
2033 for (i = 1; i < obj->sec_cnt; i++) {
2034 struct src_sec *src_sec, *src_linked_sec;
2035 struct dst_sec *dst_sec, *dst_linked_sec;
2036 Elf64_Rel *src_rel, *dst_rel;
2037 int j, n;
2038
2039 src_sec = &obj->secs[i];
2040 if (!is_relo_sec(src_sec))
2041 continue;
2042
2043 /* shdr->sh_info points to relocatable section */
2044 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2045 if (src_linked_sec->skipped)
2046 continue;
2047
2048 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2049 if (!dst_sec) {
2050 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2051 if (!dst_sec)
2052 return -ENOMEM;
2053 err = init_sec(linker, dst_sec, src_sec);
2054 if (err) {
2055 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2056 return err;
2057 }
2058 } else if (!secs_match(dst_sec, src_sec)) {
2059 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2060 return -1;
2061 }
2062
2063 /* shdr->sh_link points to SYMTAB */
2064 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2065
2066 /* shdr->sh_info points to relocated section */
2067 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2068 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2069
2070 src_sec->dst_id = dst_sec->id;
2071 err = extend_sec(linker, dst_sec, src_sec);
2072 if (err)
2073 return err;
2074
2075 src_rel = src_sec->data->d_buf;
2076 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2077 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2078 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2079 size_t src_sym_idx, dst_sym_idx, sym_type;
2080 Elf64_Sym *src_sym;
2081
2082 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2083 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2084
2085 dst_sym_idx = obj->sym_map[src_sym_idx];
2086 dst_rel->r_offset += src_linked_sec->dst_off;
2087 sym_type = ELF64_R_TYPE(src_rel->r_info);
2088 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2089
2090 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2091 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2092 struct bpf_insn *insn;
2093
2094 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2095 /* calls to the very first static function inside
2096 * .text section at offset 0 will
2097 * reference section symbol, not the
2098 * function symbol. Fix that up,
2099 * otherwise it won't be possible to
2100 * relocate calls to two different
2101 * static functions with the same name
2102 * (rom two different object files)
2103 */
2104 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2105 if (insn->code == (BPF_JMP | BPF_CALL))
2106 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2107 else
2108 insn->imm += sec->dst_off;
2109 } else {
2110 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2111 return -EINVAL;
2112 }
2113 }
2114
2115 }
2116 }
2117
2118 return 0;
2119}
2120
2121static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2122 int sym_type, const char *sym_name)
2123{
2124 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2125 Elf64_Sym *sym = symtab->data->d_buf;
2126 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2127 int str_sec_idx = symtab->shdr->sh_link;
2128 const char *name;
2129
2130 for (i = 0; i < n; i++, sym++) {
2131 if (sym->st_shndx != sec_idx)
2132 continue;
2133 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2134 continue;
2135
2136 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2137 if (!name)
2138 return NULL;
2139
2140 if (strcmp(sym_name, name) != 0)
2141 continue;
2142
2143 return sym;
2144 }
2145
2146 return NULL;
2147}
2148
2149static int linker_fixup_btf(struct src_obj *obj)
2150{
2151 const char *sec_name;
2152 struct src_sec *sec;
2153 int i, j, n, m;
2154
2155 if (!obj->btf)
2156 return 0;
2157
2158 n = btf__type_cnt(obj->btf);
2159 for (i = 1; i < n; i++) {
2160 struct btf_var_secinfo *vi;
2161 struct btf_type *t;
2162
2163 t = btf_type_by_id(obj->btf, i);
2164 if (btf_kind(t) != BTF_KIND_DATASEC)
2165 continue;
2166
2167 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2168 sec = find_src_sec_by_name(obj, sec_name);
2169 if (sec) {
2170 /* record actual section size, unless ephemeral */
2171 if (sec->shdr)
2172 t->size = sec->shdr->sh_size;
2173 } else {
2174 /* BTF can have some sections that are not represented
2175 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2176 * for special extern variables.
2177 *
2178 * For all but one such special (ephemeral)
2179 * sections, we pre-create "section shells" to be able
2180 * to keep track of extra per-section metadata later
2181 * (e.g., those BTF extern variables).
2182 *
2183 * .extern is even more special, though, because it
2184 * contains extern variables that need to be resolved
2185 * by static linker, not libbpf and kernel. When such
2186 * externs are resolved, we are going to remove them
2187 * from .extern BTF section and might end up not
2188 * needing it at all. Each resolved extern should have
2189 * matching non-extern VAR/FUNC in other sections.
2190 *
2191 * We do support leaving some of the externs
2192 * unresolved, though, to support cases of building
2193 * libraries, which will later be linked against final
2194 * BPF applications. So if at finalization we still
2195 * see unresolved externs, we'll create .extern
2196 * section on our own.
2197 */
2198 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2199 continue;
2200
2201 sec = add_src_sec(obj, sec_name);
2202 if (!sec)
2203 return -ENOMEM;
2204
2205 sec->ephemeral = true;
2206 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2207 }
2208
2209 /* remember ELF section and its BTF type ID match */
2210 sec->sec_type_id = i;
2211
2212 /* fix up variable offsets */
2213 vi = btf_var_secinfos(t);
2214 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2215 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2216 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2217 int var_linkage = btf_var(vt)->linkage;
2218 Elf64_Sym *sym;
2219
2220 /* no need to patch up static or extern vars */
2221 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2222 continue;
2223
2224 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2225 if (!sym) {
2226 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2227 return -ENOENT;
2228 }
2229
2230 vi->offset = sym->st_value;
2231 }
2232 }
2233
2234 return 0;
2235}
2236
2237static int remap_type_id(__u32 *type_id, void *ctx)
2238{
2239 int *id_map = ctx;
2240 int new_id = id_map[*type_id];
2241
2242 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2243 if (new_id == 0 && *type_id != 0) {
2244 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2245 return -EINVAL;
2246 }
2247
2248 *type_id = id_map[*type_id];
2249
2250 return 0;
2251}
2252
2253static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2254{
2255 const struct btf_type *t;
2256 int i, j, n, start_id, id;
2257 const char *name;
2258
2259 if (!obj->btf)
2260 return 0;
2261
2262 start_id = btf__type_cnt(linker->btf);
2263 n = btf__type_cnt(obj->btf);
2264
2265 obj->btf_type_map = calloc(n + 1, sizeof(int));
2266 if (!obj->btf_type_map)
2267 return -ENOMEM;
2268
2269 for (i = 1; i < n; i++) {
2270 struct glob_sym *glob_sym = NULL;
2271
2272 t = btf__type_by_id(obj->btf, i);
2273
2274 /* DATASECs are handled specially below */
2275 if (btf_kind(t) == BTF_KIND_DATASEC)
2276 continue;
2277
2278 if (btf_is_non_static(t)) {
2279 /* there should be glob_sym already */
2280 name = btf__str_by_offset(obj->btf, t->name_off);
2281 glob_sym = find_glob_sym(linker, name);
2282
2283 /* VARs without corresponding glob_sym are those that
2284 * belong to skipped/deduplicated sections (i.e.,
2285 * license and version), so just skip them
2286 */
2287 if (!glob_sym)
2288 continue;
2289
2290 /* linker_append_elf_sym() might have requested
2291 * updating underlying type ID, if extern was resolved
2292 * to strong symbol or weak got upgraded to non-weak
2293 */
2294 if (glob_sym->underlying_btf_id == 0)
2295 glob_sym->underlying_btf_id = -t->type;
2296
2297 /* globals from previous object files that match our
2298 * VAR/FUNC already have a corresponding associated
2299 * BTF type, so just make sure to use it
2300 */
2301 if (glob_sym->btf_id) {
2302 /* reuse existing BTF type for global var/func */
2303 obj->btf_type_map[i] = glob_sym->btf_id;
2304 continue;
2305 }
2306 }
2307
2308 id = btf__add_type(linker->btf, obj->btf, t);
2309 if (id < 0) {
2310 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2311 return id;
2312 }
2313
2314 obj->btf_type_map[i] = id;
2315
2316 /* record just appended BTF type for var/func */
2317 if (glob_sym) {
2318 glob_sym->btf_id = id;
2319 glob_sym->underlying_btf_id = -t->type;
2320 }
2321 }
2322
2323 /* remap all the types except DATASECs */
2324 n = btf__type_cnt(linker->btf);
2325 for (i = start_id; i < n; i++) {
2326 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2327
2328 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2329 return -EINVAL;
2330 }
2331
2332 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2333 * actual type), if necessary
2334 */
2335 for (i = 0; i < linker->glob_sym_cnt; i++) {
2336 struct glob_sym *glob_sym = &linker->glob_syms[i];
2337 struct btf_type *glob_t;
2338
2339 if (glob_sym->underlying_btf_id >= 0)
2340 continue;
2341
2342 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2343
2344 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2345 glob_t->type = glob_sym->underlying_btf_id;
2346 }
2347
2348 /* append DATASEC info */
2349 for (i = 1; i < obj->sec_cnt; i++) {
2350 struct src_sec *src_sec;
2351 struct dst_sec *dst_sec;
2352 const struct btf_var_secinfo *src_var;
2353 struct btf_var_secinfo *dst_var;
2354
2355 src_sec = &obj->secs[i];
2356 if (!src_sec->sec_type_id || src_sec->skipped)
2357 continue;
2358 dst_sec = &linker->secs[src_sec->dst_id];
2359
2360 /* Mark section as having BTF regardless of the presence of
2361 * variables. In some cases compiler might generate empty BTF
2362 * with no variables information. E.g., when promoting local
2363 * array/structure variable initial values and BPF object
2364 * file otherwise has no read-only static variables in
2365 * .rodata. We need to preserve such empty BTF and just set
2366 * correct section size.
2367 */
2368 dst_sec->has_btf = true;
2369
2370 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2371 src_var = btf_var_secinfos(t);
2372 n = btf_vlen(t);
2373 for (j = 0; j < n; j++, src_var++) {
2374 void *sec_vars = dst_sec->sec_vars;
2375 int new_id = obj->btf_type_map[src_var->type];
2376 struct glob_sym *glob_sym = NULL;
2377
2378 t = btf_type_by_id(linker->btf, new_id);
2379 if (btf_is_non_static(t)) {
2380 name = btf__str_by_offset(linker->btf, t->name_off);
2381 glob_sym = find_glob_sym(linker, name);
2382 if (glob_sym->sec_id != dst_sec->id) {
2383 pr_warn("global '%s': section mismatch %d vs %d\n",
2384 name, glob_sym->sec_id, dst_sec->id);
2385 return -EINVAL;
2386 }
2387 }
2388
2389 /* If there is already a member (VAR or FUNC) mapped
2390 * to the same type, don't add a duplicate entry.
2391 * This will happen when multiple object files define
2392 * the same extern VARs/FUNCs.
2393 */
2394 if (glob_sym && glob_sym->var_idx >= 0) {
2395 __s64 sz;
2396
2397 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2398 /* Because underlying BTF type might have
2399 * changed, so might its size have changed, so
2400 * re-calculate and update it in sec_var.
2401 */
2402 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2403 if (sz < 0) {
2404 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2405 name, (int)sz);
2406 return -EINVAL;
2407 }
2408 dst_var->size = sz;
2409 continue;
2410 }
2411
2412 sec_vars = libbpf_reallocarray(sec_vars,
2413 dst_sec->sec_var_cnt + 1,
2414 sizeof(*dst_sec->sec_vars));
2415 if (!sec_vars)
2416 return -ENOMEM;
2417
2418 dst_sec->sec_vars = sec_vars;
2419 dst_sec->sec_var_cnt++;
2420
2421 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2422 dst_var->type = obj->btf_type_map[src_var->type];
2423 dst_var->size = src_var->size;
2424 dst_var->offset = src_sec->dst_off + src_var->offset;
2425
2426 if (glob_sym)
2427 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2428 }
2429 }
2430
2431 return 0;
2432}
2433
2434static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2435{
2436 void *tmp;
2437
2438 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2439 if (!tmp)
2440 return NULL;
2441 ext_data->recs = tmp;
2442
2443 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2444 memcpy(tmp, src_rec, ext_data->rec_sz);
2445
2446 ext_data->rec_cnt++;
2447
2448 return tmp;
2449}
2450
2451static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2452{
2453 const struct btf_ext_info_sec *ext_sec;
2454 const char *sec_name, *s;
2455 struct src_sec *src_sec;
2456 struct dst_sec *dst_sec;
2457 int rec_sz, str_off, i;
2458
2459 if (!obj->btf_ext)
2460 return 0;
2461
2462 rec_sz = obj->btf_ext->func_info.rec_size;
2463 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2464 struct bpf_func_info_min *src_rec, *dst_rec;
2465
2466 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2467 src_sec = find_src_sec_by_name(obj, sec_name);
2468 if (!src_sec) {
2469 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2470 return -EINVAL;
2471 }
2472 dst_sec = &linker->secs[src_sec->dst_id];
2473
2474 if (dst_sec->func_info.rec_sz == 0)
2475 dst_sec->func_info.rec_sz = rec_sz;
2476 if (dst_sec->func_info.rec_sz != rec_sz) {
2477 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2478 return -EINVAL;
2479 }
2480
2481 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2482 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2483 if (!dst_rec)
2484 return -ENOMEM;
2485
2486 dst_rec->insn_off += src_sec->dst_off;
2487 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2488 }
2489 }
2490
2491 rec_sz = obj->btf_ext->line_info.rec_size;
2492 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2493 struct bpf_line_info_min *src_rec, *dst_rec;
2494
2495 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2496 src_sec = find_src_sec_by_name(obj, sec_name);
2497 if (!src_sec) {
2498 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2499 return -EINVAL;
2500 }
2501 dst_sec = &linker->secs[src_sec->dst_id];
2502
2503 if (dst_sec->line_info.rec_sz == 0)
2504 dst_sec->line_info.rec_sz = rec_sz;
2505 if (dst_sec->line_info.rec_sz != rec_sz) {
2506 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2507 return -EINVAL;
2508 }
2509
2510 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2511 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2512 if (!dst_rec)
2513 return -ENOMEM;
2514
2515 dst_rec->insn_off += src_sec->dst_off;
2516
2517 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2518 str_off = btf__add_str(linker->btf, s);
2519 if (str_off < 0)
2520 return -ENOMEM;
2521 dst_rec->file_name_off = str_off;
2522
2523 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2524 str_off = btf__add_str(linker->btf, s);
2525 if (str_off < 0)
2526 return -ENOMEM;
2527 dst_rec->line_off = str_off;
2528
2529 /* dst_rec->line_col is fine */
2530 }
2531 }
2532
2533 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2534 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2535 struct bpf_core_relo *src_rec, *dst_rec;
2536
2537 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2538 src_sec = find_src_sec_by_name(obj, sec_name);
2539 if (!src_sec) {
2540 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2541 return -EINVAL;
2542 }
2543 dst_sec = &linker->secs[src_sec->dst_id];
2544
2545 if (dst_sec->core_relo_info.rec_sz == 0)
2546 dst_sec->core_relo_info.rec_sz = rec_sz;
2547 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2548 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2549 return -EINVAL;
2550 }
2551
2552 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2553 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2554 if (!dst_rec)
2555 return -ENOMEM;
2556
2557 dst_rec->insn_off += src_sec->dst_off;
2558 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2559
2560 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2561 str_off = btf__add_str(linker->btf, s);
2562 if (str_off < 0)
2563 return -ENOMEM;
2564 dst_rec->access_str_off = str_off;
2565
2566 /* dst_rec->kind is fine */
2567 }
2568 }
2569
2570 return 0;
2571}
2572
2573int bpf_linker__finalize(struct bpf_linker *linker)
2574{
2575 struct dst_sec *sec;
2576 size_t strs_sz;
2577 const void *strs;
2578 int err, i;
2579
2580 if (!linker->elf)
2581 return libbpf_err(-EINVAL);
2582
2583 err = finalize_btf(linker);
2584 if (err)
2585 return libbpf_err(err);
2586
2587 /* Finalize strings */
2588 strs_sz = strset__data_size(linker->strtab_strs);
2589 strs = strset__data(linker->strtab_strs);
2590
2591 sec = &linker->secs[linker->strtab_sec_idx];
2592 sec->data->d_align = 1;
2593 sec->data->d_off = 0LL;
2594 sec->data->d_buf = (void *)strs;
2595 sec->data->d_type = ELF_T_BYTE;
2596 sec->data->d_size = strs_sz;
2597 sec->shdr->sh_size = strs_sz;
2598
2599 for (i = 1; i < linker->sec_cnt; i++) {
2600 sec = &linker->secs[i];
2601
2602 /* STRTAB is handled specially above */
2603 if (sec->sec_idx == linker->strtab_sec_idx)
2604 continue;
2605
2606 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2607 if (!sec->scn)
2608 continue;
2609
2610 sec->data->d_buf = sec->raw_data;
2611 }
2612
2613 /* Finalize ELF layout */
2614 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2615 err = -errno;
2616 pr_warn_elf("failed to finalize ELF layout");
2617 return libbpf_err(err);
2618 }
2619
2620 /* Write out final ELF contents */
2621 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2622 err = -errno;
2623 pr_warn_elf("failed to write ELF contents");
2624 return libbpf_err(err);
2625 }
2626
2627 elf_end(linker->elf);
2628 close(linker->fd);
2629
2630 linker->elf = NULL;
2631 linker->fd = -1;
2632
2633 return 0;
2634}
2635
2636static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2637 size_t align, const void *raw_data, size_t raw_sz)
2638{
2639 Elf_Scn *scn;
2640 Elf_Data *data;
2641 Elf64_Shdr *shdr;
2642 int name_off;
2643
2644 name_off = strset__add_str(linker->strtab_strs, sec_name);
2645 if (name_off < 0)
2646 return name_off;
2647
2648 scn = elf_newscn(linker->elf);
2649 if (!scn)
2650 return -ENOMEM;
2651 data = elf_newdata(scn);
2652 if (!data)
2653 return -ENOMEM;
2654 shdr = elf64_getshdr(scn);
2655 if (!shdr)
2656 return -EINVAL;
2657
2658 shdr->sh_name = name_off;
2659 shdr->sh_type = SHT_PROGBITS;
2660 shdr->sh_flags = 0;
2661 shdr->sh_size = raw_sz;
2662 shdr->sh_link = 0;
2663 shdr->sh_info = 0;
2664 shdr->sh_addralign = align;
2665 shdr->sh_entsize = 0;
2666
2667 data->d_type = ELF_T_BYTE;
2668 data->d_size = raw_sz;
2669 data->d_buf = (void *)raw_data;
2670 data->d_align = align;
2671 data->d_off = 0;
2672
2673 return 0;
2674}
2675
2676static int finalize_btf(struct bpf_linker *linker)
2677{
2678 LIBBPF_OPTS(btf_dedup_opts, opts);
2679 struct btf *btf = linker->btf;
2680 const void *raw_data;
2681 int i, j, id, err;
2682 __u32 raw_sz;
2683
2684 /* bail out if no BTF data was produced */
2685 if (btf__type_cnt(linker->btf) == 1)
2686 return 0;
2687
2688 for (i = 1; i < linker->sec_cnt; i++) {
2689 struct dst_sec *sec = &linker->secs[i];
2690
2691 if (!sec->has_btf)
2692 continue;
2693
2694 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2695 if (id < 0) {
2696 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2697 sec->sec_name, id);
2698 return id;
2699 }
2700
2701 for (j = 0; j < sec->sec_var_cnt; j++) {
2702 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2703
2704 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2705 return -EINVAL;
2706 }
2707 }
2708
2709 err = finalize_btf_ext(linker);
2710 if (err) {
2711 pr_warn(".BTF.ext generation failed: %d\n", err);
2712 return err;
2713 }
2714
2715 opts.btf_ext = linker->btf_ext;
2716 err = btf__dedup(linker->btf, &opts);
2717 if (err) {
2718 pr_warn("BTF dedup failed: %d\n", err);
2719 return err;
2720 }
2721
2722 /* Emit .BTF section */
2723 raw_data = btf__raw_data(linker->btf, &raw_sz);
2724 if (!raw_data)
2725 return -ENOMEM;
2726
2727 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2728 if (err) {
2729 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2730 return err;
2731 }
2732
2733 /* Emit .BTF.ext section */
2734 if (linker->btf_ext) {
2735 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2736 if (!raw_data)
2737 return -ENOMEM;
2738
2739 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2740 if (err) {
2741 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2742 return err;
2743 }
2744 }
2745
2746 return 0;
2747}
2748
2749static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2750 const char *sec_name, struct btf_ext_sec_data *sec_data)
2751{
2752 struct btf_ext_info_sec *sec_info;
2753 void *cur = output;
2754 int str_off;
2755 size_t sz;
2756
2757 if (!sec_data->rec_cnt)
2758 return 0;
2759
2760 str_off = btf__add_str(linker->btf, sec_name);
2761 if (str_off < 0)
2762 return -ENOMEM;
2763
2764 sec_info = cur;
2765 sec_info->sec_name_off = str_off;
2766 sec_info->num_info = sec_data->rec_cnt;
2767 cur += sizeof(struct btf_ext_info_sec);
2768
2769 sz = sec_data->rec_cnt * sec_data->rec_sz;
2770 memcpy(cur, sec_data->recs, sz);
2771 cur += sz;
2772
2773 return cur - output;
2774}
2775
2776static int finalize_btf_ext(struct bpf_linker *linker)
2777{
2778 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2779 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2780 struct btf_ext_header *hdr;
2781 void *data, *cur;
2782 int i, err, sz;
2783
2784 /* validate that all sections have the same .BTF.ext record sizes
2785 * and calculate total data size for each type of data (func info,
2786 * line info, core relos)
2787 */
2788 for (i = 1; i < linker->sec_cnt; i++) {
2789 struct dst_sec *sec = &linker->secs[i];
2790
2791 if (sec->func_info.rec_cnt) {
2792 if (func_rec_sz == 0)
2793 func_rec_sz = sec->func_info.rec_sz;
2794 if (func_rec_sz != sec->func_info.rec_sz) {
2795 pr_warn("mismatch in func_info record size %zu != %u\n",
2796 func_rec_sz, sec->func_info.rec_sz);
2797 return -EINVAL;
2798 }
2799
2800 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2801 }
2802 if (sec->line_info.rec_cnt) {
2803 if (line_rec_sz == 0)
2804 line_rec_sz = sec->line_info.rec_sz;
2805 if (line_rec_sz != sec->line_info.rec_sz) {
2806 pr_warn("mismatch in line_info record size %zu != %u\n",
2807 line_rec_sz, sec->line_info.rec_sz);
2808 return -EINVAL;
2809 }
2810
2811 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2812 }
2813 if (sec->core_relo_info.rec_cnt) {
2814 if (core_relo_rec_sz == 0)
2815 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2816 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2817 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2818 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2819 return -EINVAL;
2820 }
2821
2822 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2823 }
2824 }
2825
2826 if (!funcs_sz && !lines_sz && !core_relos_sz)
2827 return 0;
2828
2829 total_sz += sizeof(struct btf_ext_header);
2830 if (funcs_sz) {
2831 funcs_sz += sizeof(__u32); /* record size prefix */
2832 total_sz += funcs_sz;
2833 }
2834 if (lines_sz) {
2835 lines_sz += sizeof(__u32); /* record size prefix */
2836 total_sz += lines_sz;
2837 }
2838 if (core_relos_sz) {
2839 core_relos_sz += sizeof(__u32); /* record size prefix */
2840 total_sz += core_relos_sz;
2841 }
2842
2843 cur = data = calloc(1, total_sz);
2844 if (!data)
2845 return -ENOMEM;
2846
2847 hdr = cur;
2848 hdr->magic = BTF_MAGIC;
2849 hdr->version = BTF_VERSION;
2850 hdr->flags = 0;
2851 hdr->hdr_len = sizeof(struct btf_ext_header);
2852 cur += sizeof(struct btf_ext_header);
2853
2854 /* All offsets are in bytes relative to the end of this header */
2855 hdr->func_info_off = 0;
2856 hdr->func_info_len = funcs_sz;
2857 hdr->line_info_off = funcs_sz;
2858 hdr->line_info_len = lines_sz;
2859 hdr->core_relo_off = funcs_sz + lines_sz;
2860 hdr->core_relo_len = core_relos_sz;
2861
2862 if (funcs_sz) {
2863 *(__u32 *)cur = func_rec_sz;
2864 cur += sizeof(__u32);
2865
2866 for (i = 1; i < linker->sec_cnt; i++) {
2867 struct dst_sec *sec = &linker->secs[i];
2868
2869 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2870 if (sz < 0) {
2871 err = sz;
2872 goto out;
2873 }
2874
2875 cur += sz;
2876 }
2877 }
2878
2879 if (lines_sz) {
2880 *(__u32 *)cur = line_rec_sz;
2881 cur += sizeof(__u32);
2882
2883 for (i = 1; i < linker->sec_cnt; i++) {
2884 struct dst_sec *sec = &linker->secs[i];
2885
2886 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2887 if (sz < 0) {
2888 err = sz;
2889 goto out;
2890 }
2891
2892 cur += sz;
2893 }
2894 }
2895
2896 if (core_relos_sz) {
2897 *(__u32 *)cur = core_relo_rec_sz;
2898 cur += sizeof(__u32);
2899
2900 for (i = 1; i < linker->sec_cnt; i++) {
2901 struct dst_sec *sec = &linker->secs[i];
2902
2903 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2904 if (sz < 0) {
2905 err = sz;
2906 goto out;
2907 }
2908
2909 cur += sz;
2910 }
2911 }
2912
2913 linker->btf_ext = btf_ext__new(data, total_sz);
2914 err = libbpf_get_error(linker->btf_ext);
2915 if (err) {
2916 linker->btf_ext = NULL;
2917 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2918 goto out;
2919 }
2920
2921out:
2922 free(data);
2923 return err;
2924}