1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2/*
   3 * BPF static linker
   4 *
   5 * Copyright (c) 2021 Facebook
   6 */
   7#include <stdbool.h>
   8#include <stddef.h>
   9#include <stdio.h>
  10#include <stdlib.h>
  11#include <string.h>
  12#include <unistd.h>
  13#include <errno.h>
  14#include <linux/err.h>
  15#include <linux/btf.h>
  16#include <elf.h>
  17#include <libelf.h>
  18#include <fcntl.h>
  19#include "libbpf.h"
  20#include "btf.h"
  21#include "libbpf_internal.h"
  22#include "strset.h"
  23#include "str_error.h"
  24
  25#define BTF_EXTERN_SEC ".extern"
  26
  27struct src_sec {
  28	const char *sec_name;
  29	/* positional (not necessarily ELF) index in an array of sections */
  30	int id;
  31	/* positional (not necessarily ELF) index of a matching section in a final object file */
  32	int dst_id;
  33	/* section data offset in a matching output section */
  34	int dst_off;
  35	/* whether section is omitted from the final ELF file */
  36	bool skipped;
  37	/* whether section is an ephemeral section, not mapped to an ELF section */
  38	bool ephemeral;
  39
  40	/* ELF info */
  41	size_t sec_idx;
  42	Elf_Scn *scn;
  43	Elf64_Shdr *shdr;
  44	Elf_Data *data;
  45
  46	/* corresponding BTF DATASEC type ID */
  47	int sec_type_id;
  48};
  49
  50struct src_obj {
  51	const char *filename;
  52	int fd;
  53	Elf *elf;
  54	/* Section header strings section index */
  55	size_t shstrs_sec_idx;
  56	/* SYMTAB section index */
  57	size_t symtab_sec_idx;
  58
  59	struct btf *btf;
  60	struct btf_ext *btf_ext;
  61
  62	/* List of sections (including ephemeral). Slot zero is unused. */
  63	struct src_sec *secs;
  64	int sec_cnt;
  65
  66	/* mapping of symbol indices from src to dst ELF */
  67	int *sym_map;
  68	/* mapping from the src BTF type IDs to dst ones */
  69	int *btf_type_map;
  70};
  71
  72/* single .BTF.ext data section */
  73struct btf_ext_sec_data {
  74	size_t rec_cnt;
  75	__u32 rec_sz;
  76	void *recs;
  77};
  78
  79struct glob_sym {
  80	/* ELF symbol index */
  81	int sym_idx;
  82	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
  83	int sec_id;
  84	/* extern name offset in STRTAB */
  85	int name_off;
  86	/* optional associated BTF type ID */
  87	int btf_id;
  88	/* BTF type ID to which VAR/FUNC type is pointing to; used for
  89	 * rewriting types when extern VAR/FUNC is resolved to a concrete
  90	 * definition
  91	 */
  92	int underlying_btf_id;
  93	/* sec_var index in the corresponding dst_sec, if exists */
  94	int var_idx;
  95
  96	/* extern or resolved/global symbol */
  97	bool is_extern;
  98	/* weak or strong symbol, never goes back from strong to weak */
  99	bool is_weak;
 100};
 101
 102struct dst_sec {
 103	char *sec_name;
 104	/* positional (not necessarily ELF) index in an array of sections */
 105	int id;
 106
 107	bool ephemeral;
 108
 109	/* ELF info */
 110	size_t sec_idx;
 111	Elf_Scn *scn;
 112	Elf64_Shdr *shdr;
 113	Elf_Data *data;
 114
 115	/* final output section size */
 116	int sec_sz;
 117	/* final output contents of the section */
 118	void *raw_data;
 119
 120	/* corresponding STT_SECTION symbol index in SYMTAB */
 121	int sec_sym_idx;
 122
 123	/* section's DATASEC variable info, emitted on BTF finalization */
 124	bool has_btf;
 125	int sec_var_cnt;
 126	struct btf_var_secinfo *sec_vars;
 127
 128	/* section's .BTF.ext data */
 129	struct btf_ext_sec_data func_info;
 130	struct btf_ext_sec_data line_info;
 131	struct btf_ext_sec_data core_relo_info;
 132};
 133
 134struct bpf_linker {
 135	char *filename;
 136	int fd;
 137	Elf *elf;
 138	Elf64_Ehdr *elf_hdr;
 139	bool swapped_endian;
 140
 141	/* Output sections metadata */
 142	struct dst_sec *secs;
 143	int sec_cnt;
 144
 145	struct strset *strtab_strs; /* STRTAB unique strings */
 146	size_t strtab_sec_idx; /* STRTAB section index */
 147	size_t symtab_sec_idx; /* SYMTAB section index */
 148
 149	struct btf *btf;
 150	struct btf_ext *btf_ext;
 151
 152	/* global (including extern) ELF symbols */
 153	int glob_sym_cnt;
 154	struct glob_sym *glob_syms;
 155};
 156
 157#define pr_warn_elf(fmt, ...)									\
 158	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 159
 160static int init_output_elf(struct bpf_linker *linker, const char *file);
 161
 162static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 163				const struct bpf_linker_file_opts *opts,
 164				struct src_obj *obj);
 165static int linker_sanity_check_elf(struct src_obj *obj);
 166static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 167static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 168static int linker_sanity_check_btf(struct src_obj *obj);
 169static int linker_sanity_check_btf_ext(struct src_obj *obj);
 170static int linker_fixup_btf(struct src_obj *obj);
 171static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 172static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 173static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 174				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 175static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 176static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 177static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 178
 179static int finalize_btf(struct bpf_linker *linker);
 180static int finalize_btf_ext(struct bpf_linker *linker);
 181
 182void bpf_linker__free(struct bpf_linker *linker)
 183{
 184	int i;
 185
 186	if (!linker)
 187		return;
 188
 189	free(linker->filename);
 190
 191	if (linker->elf)
 192		elf_end(linker->elf);
 193
 194	if (linker->fd >= 0)
 195		close(linker->fd);
 196
 197	strset__free(linker->strtab_strs);
 198
 199	btf__free(linker->btf);
 200	btf_ext__free(linker->btf_ext);
 201
 202	for (i = 1; i < linker->sec_cnt; i++) {
 203		struct dst_sec *sec = &linker->secs[i];
 204
 205		free(sec->sec_name);
 206		free(sec->raw_data);
 207		free(sec->sec_vars);
 208
 209		free(sec->func_info.recs);
 210		free(sec->line_info.recs);
 211		free(sec->core_relo_info.recs);
 212	}
 213	free(linker->secs);
 214
 215	free(linker->glob_syms);
 216	free(linker);
 217}
 218
 219struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 220{
 221	struct bpf_linker *linker;
 222	int err;
 223
 224	if (!OPTS_VALID(opts, bpf_linker_opts))
 225		return errno = EINVAL, NULL;
 226
 227	if (elf_version(EV_CURRENT) == EV_NONE) {
 228		pr_warn_elf("libelf initialization failed");
 229		return errno = EINVAL, NULL;
 230	}
 231
 232	linker = calloc(1, sizeof(*linker));
 233	if (!linker)
 234		return errno = ENOMEM, NULL;
 235
 236	linker->fd = -1;
 237
 238	err = init_output_elf(linker, filename);
 239	if (err)
 240		goto err_out;
 241
 242	return linker;
 243
 244err_out:
 245	bpf_linker__free(linker);
 246	return errno = -err, NULL;
 247}
 248
 249static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 250{
 251	struct dst_sec *secs = linker->secs, *sec;
 252	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 253
 254	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 255	if (!secs)
 256		return NULL;
 257
 258	/* zero out newly allocated memory */
 259	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 260
 261	linker->secs = secs;
 262	linker->sec_cnt = new_cnt;
 263
 264	sec = &linker->secs[new_cnt - 1];
 265	sec->id = new_cnt - 1;
 266	sec->sec_name = strdup(sec_name);
 267	if (!sec->sec_name)
 268		return NULL;
 269
 270	return sec;
 271}
 272
 273static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 274{
 275	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 276	Elf64_Sym *syms, *sym;
 277	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 278
 279	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 280	if (!syms)
 281		return NULL;
 282
 283	sym = &syms[sym_cnt];
 284	memset(sym, 0, sizeof(*sym));
 285
 286	symtab->raw_data = syms;
 287	symtab->sec_sz += sizeof(*sym);
 288	symtab->shdr->sh_size += sizeof(*sym);
 289	symtab->data->d_size += sizeof(*sym);
 290
 291	if (sym_idx)
 292		*sym_idx = sym_cnt;
 293
 294	return sym;
 295}
 296
 297static int init_output_elf(struct bpf_linker *linker, const char *file)
 298{
 299	int err, str_off;
 300	Elf64_Sym *init_sym;
 301	struct dst_sec *sec;
 302
 303	linker->filename = strdup(file);
 304	if (!linker->filename)
 305		return -ENOMEM;
 306
 307	linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
 308	if (linker->fd < 0) {
 309		err = -errno;
 310		pr_warn("failed to create '%s': %s\n", file, errstr(err));
 311		return err;
 312	}
 313
 314	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 315	if (!linker->elf) {
 316		pr_warn_elf("failed to create ELF object");
 317		return -EINVAL;
 318	}
 319
 320	/* ELF header */
 321	linker->elf_hdr = elf64_newehdr(linker->elf);
 322	if (!linker->elf_hdr) {
 323		pr_warn_elf("failed to create ELF header");
 324		return -EINVAL;
 325	}
 326
 327	linker->elf_hdr->e_machine = EM_BPF;
 328	linker->elf_hdr->e_type = ET_REL;
 329	/* Set unknown ELF endianness, assign later from input files */
 330	linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
 331
 332	/* STRTAB */
 333	/* initialize strset with an empty string to conform to ELF */
 334	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 335	if (libbpf_get_error(linker->strtab_strs))
 336		return libbpf_get_error(linker->strtab_strs);
 337
 338	sec = add_dst_sec(linker, ".strtab");
 339	if (!sec)
 340		return -ENOMEM;
 341
 342	sec->scn = elf_newscn(linker->elf);
 343	if (!sec->scn) {
 344		pr_warn_elf("failed to create STRTAB section");
 345		return -EINVAL;
 346	}
 347
 348	sec->shdr = elf64_getshdr(sec->scn);
 349	if (!sec->shdr)
 350		return -EINVAL;
 351
 352	sec->data = elf_newdata(sec->scn);
 353	if (!sec->data) {
 354		pr_warn_elf("failed to create STRTAB data");
 355		return -EINVAL;
 356	}
 357
 358	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 359	if (str_off < 0)
 360		return str_off;
 361
 362	sec->sec_idx = elf_ndxscn(sec->scn);
 363	linker->elf_hdr->e_shstrndx = sec->sec_idx;
 364	linker->strtab_sec_idx = sec->sec_idx;
 365
 366	sec->shdr->sh_name = str_off;
 367	sec->shdr->sh_type = SHT_STRTAB;
 368	sec->shdr->sh_flags = SHF_STRINGS;
 369	sec->shdr->sh_offset = 0;
 370	sec->shdr->sh_link = 0;
 371	sec->shdr->sh_info = 0;
 372	sec->shdr->sh_addralign = 1;
 373	sec->shdr->sh_size = sec->sec_sz = 0;
 374	sec->shdr->sh_entsize = 0;
 375
 376	/* SYMTAB */
 377	sec = add_dst_sec(linker, ".symtab");
 378	if (!sec)
 379		return -ENOMEM;
 380
 381	sec->scn = elf_newscn(linker->elf);
 382	if (!sec->scn) {
 383		pr_warn_elf("failed to create SYMTAB section");
 384		return -EINVAL;
 385	}
 386
 387	sec->shdr = elf64_getshdr(sec->scn);
 388	if (!sec->shdr)
 389		return -EINVAL;
 390
 391	sec->data = elf_newdata(sec->scn);
 392	if (!sec->data) {
 393		pr_warn_elf("failed to create SYMTAB data");
 394		return -EINVAL;
 395	}
 396	/* Ensure libelf translates byte-order of symbol records */
 397	sec->data->d_type = ELF_T_SYM;
 398
 399	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 400	if (str_off < 0)
 401		return str_off;
 402
 403	sec->sec_idx = elf_ndxscn(sec->scn);
 404	linker->symtab_sec_idx = sec->sec_idx;
 405
 406	sec->shdr->sh_name = str_off;
 407	sec->shdr->sh_type = SHT_SYMTAB;
 408	sec->shdr->sh_flags = 0;
 409	sec->shdr->sh_offset = 0;
 410	sec->shdr->sh_link = linker->strtab_sec_idx;
 411	/* sh_info should be one greater than the index of the last local
 412	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 413	 */
 414	sec->shdr->sh_info = 0;
 415	sec->shdr->sh_addralign = 8;
 416	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 417
 418	/* .BTF */
 419	linker->btf = btf__new_empty();
 420	err = libbpf_get_error(linker->btf);
 421	if (err)
 422		return err;
 423
 424	/* add the special all-zero symbol */
 425	init_sym = add_new_sym(linker, NULL);
 426	if (!init_sym)
 427		return -EINVAL;
 428
 429	init_sym->st_name = 0;
 430	init_sym->st_info = 0;
 431	init_sym->st_other = 0;
 432	init_sym->st_shndx = SHN_UNDEF;
 433	init_sym->st_value = 0;
 434	init_sym->st_size = 0;
 435
 436	return 0;
 437}
 438
 439int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 440			 const struct bpf_linker_file_opts *opts)
 441{
 442	struct src_obj obj = {};
 443	int err = 0;
 444
 445	if (!OPTS_VALID(opts, bpf_linker_file_opts))
 446		return libbpf_err(-EINVAL);
 447
 448	if (!linker->elf)
 449		return libbpf_err(-EINVAL);
 450
 451	err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
 452	err = err ?: linker_append_sec_data(linker, &obj);
 453	err = err ?: linker_append_elf_syms(linker, &obj);
 454	err = err ?: linker_append_elf_relos(linker, &obj);
 455	err = err ?: linker_append_btf(linker, &obj);
 456	err = err ?: linker_append_btf_ext(linker, &obj);
 457
 458	/* free up src_obj resources */
 459	free(obj.btf_type_map);
 460	btf__free(obj.btf);
 461	btf_ext__free(obj.btf_ext);
 462	free(obj.secs);
 463	free(obj.sym_map);
 464	if (obj.elf)
 465		elf_end(obj.elf);
 466	if (obj.fd >= 0)
 467		close(obj.fd);
 468
 469	return libbpf_err(err);
 470}
 471
 472static bool is_dwarf_sec_name(const char *name)
 473{
 474	/* approximation, but the actual list is too long */
 475	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 476}
 477
 478static bool is_ignored_sec(struct src_sec *sec)
 479{
 480	Elf64_Shdr *shdr = sec->shdr;
 481	const char *name = sec->sec_name;
 482
 483	/* no special handling of .strtab */
 484	if (shdr->sh_type == SHT_STRTAB)
 485		return true;
 486
 487	/* ignore .llvm_addrsig section as well */
 488	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 489		return true;
 490
 491	/* no subprograms will lead to an empty .text section, ignore it */
 492	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 493	    strcmp(sec->sec_name, ".text") == 0)
 494		return true;
 495
 496	/* DWARF sections */
 497	if (is_dwarf_sec_name(sec->sec_name))
 498		return true;
 499
 500	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 501		name += sizeof(".rel") - 1;
 502		/* DWARF section relocations */
 503		if (is_dwarf_sec_name(name))
 504			return true;
 505
 506		/* .BTF and .BTF.ext don't need relocations */
 507		if (strcmp(name, BTF_ELF_SEC) == 0 ||
 508		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
 509			return true;
 510	}
 511
 512	return false;
 513}
 514
 515static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 516{
 517	struct src_sec *secs = obj->secs, *sec;
 518	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 519
 520	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 521	if (!secs)
 522		return NULL;
 523
 524	/* zero out newly allocated memory */
 525	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 526
 527	obj->secs = secs;
 528	obj->sec_cnt = new_cnt;
 529
 530	sec = &obj->secs[new_cnt - 1];
 531	sec->id = new_cnt - 1;
 532	sec->sec_name = sec_name;
 533
 534	return sec;
 535}
 536
 537static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 538				const struct bpf_linker_file_opts *opts,
 539				struct src_obj *obj)
 540{
 541	int err = 0;
 542	Elf_Scn *scn;
 543	Elf_Data *data;
 544	Elf64_Ehdr *ehdr;
 545	Elf64_Shdr *shdr;
 546	struct src_sec *sec;
 547	unsigned char obj_byteorder;
 548	unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
 549#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 550	const unsigned char host_byteorder = ELFDATA2LSB;
 551#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 552	const unsigned char host_byteorder = ELFDATA2MSB;
 553#else
 554#error "Unknown __BYTE_ORDER__"
 555#endif
 556
 557	pr_debug("linker: adding object file '%s'...\n", filename);
 558
 559	obj->filename = filename;
 560
 561	obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
 562	if (obj->fd < 0) {
 563		err = -errno;
 564		pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
 565		return err;
 566	}
 567	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 568	if (!obj->elf) {
 569		pr_warn_elf("failed to parse ELF file '%s'", filename);
 570		return -EINVAL;
 571	}
 572
 573	/* Sanity check ELF file high-level properties */
 574	ehdr = elf64_getehdr(obj->elf);
 575	if (!ehdr) {
 576		pr_warn_elf("failed to get ELF header for %s", filename);
 577		return -EINVAL;
 578	}
 579
 580	/* Linker output endianness set by first input object */
 581	obj_byteorder = ehdr->e_ident[EI_DATA];
 582	if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
 583		err = -EOPNOTSUPP;
 584		pr_warn("unknown byte order of ELF file %s\n", filename);
 585		return err;
 586	}
 587	if (link_byteorder == ELFDATANONE) {
 588		linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
 589		linker->swapped_endian = obj_byteorder != host_byteorder;
 590		pr_debug("linker: set %s-endian output byte order\n",
 591			 obj_byteorder == ELFDATA2MSB ? "big" : "little");
 592	} else if (link_byteorder != obj_byteorder) {
 593		err = -EOPNOTSUPP;
 594		pr_warn("byte order mismatch with ELF file %s\n", filename);
 595		return err;
 596	}
 597
 598	if (ehdr->e_type != ET_REL
 599	    || ehdr->e_machine != EM_BPF
 600	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 601		err = -EOPNOTSUPP;
 602		pr_warn_elf("unsupported kind of ELF file %s", filename);
 603		return err;
 604	}
 605
 606	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 607		pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 608		return -EINVAL;
 609	}
 610
 611	scn = NULL;
 612	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 613		size_t sec_idx = elf_ndxscn(scn);
 614		const char *sec_name;
 615
 616		shdr = elf64_getshdr(scn);
 617		if (!shdr) {
 618			pr_warn_elf("failed to get section #%zu header for %s",
 619				    sec_idx, filename);
 620			return -EINVAL;
 621		}
 622
 623		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 624		if (!sec_name) {
 625			pr_warn_elf("failed to get section #%zu name for %s",
 626				    sec_idx, filename);
 627			return -EINVAL;
 628		}
 629
 630		data = elf_getdata(scn, 0);
 631		if (!data) {
 632			pr_warn_elf("failed to get section #%zu (%s) data from %s",
 633				    sec_idx, sec_name, filename);
 634			return -EINVAL;
 635		}
 636
 637		sec = add_src_sec(obj, sec_name);
 638		if (!sec)
 639			return -ENOMEM;
 640
 641		sec->scn = scn;
 642		sec->shdr = shdr;
 643		sec->data = data;
 644		sec->sec_idx = elf_ndxscn(scn);
 645
 646		if (is_ignored_sec(sec)) {
 647			sec->skipped = true;
 648			continue;
 649		}
 650
 651		switch (shdr->sh_type) {
 652		case SHT_SYMTAB:
 653			if (obj->symtab_sec_idx) {
 654				err = -EOPNOTSUPP;
 655				pr_warn("multiple SYMTAB sections found, not supported\n");
 656				return err;
 657			}
 658			obj->symtab_sec_idx = sec_idx;
 659			break;
 660		case SHT_STRTAB:
 661			/* we'll construct our own string table */
 662			break;
 663		case SHT_PROGBITS:
 664			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 665				obj->btf = btf__new(data->d_buf, shdr->sh_size);
 666				err = libbpf_get_error(obj->btf);
 667				if (err) {
 668					pr_warn("failed to parse .BTF from %s: %s\n",
 669						filename, errstr(err));
 670					return err;
 671				}
 672				sec->skipped = true;
 673				continue;
 674			}
 675			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 676				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 677				err = libbpf_get_error(obj->btf_ext);
 678				if (err) {
 679					pr_warn("failed to parse .BTF.ext from '%s': %s\n",
 680						filename, errstr(err));
 681					return err;
 682				}
 683				sec->skipped = true;
 684				continue;
 685			}
 686
 687			/* data & code */
 688			break;
 689		case SHT_NOBITS:
 690			/* BSS */
 691			break;
 692		case SHT_REL:
 693			/* relocations */
 694			break;
 695		default:
 696			pr_warn("unrecognized section #%zu (%s) in %s\n",
 697				sec_idx, sec_name, filename);
 698			err = -EINVAL;
 699			return err;
 700		}
 701	}
 702
 703	err = err ?: linker_sanity_check_elf(obj);
 704	err = err ?: linker_sanity_check_btf(obj);
 705	err = err ?: linker_sanity_check_btf_ext(obj);
 706	err = err ?: linker_fixup_btf(obj);
 707
 708	return err;
 709}
 710
 711static int linker_sanity_check_elf(struct src_obj *obj)
 712{
 713	struct src_sec *sec;
 714	int i, err;
 715
 716	if (!obj->symtab_sec_idx) {
 717		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 718		return -EINVAL;
 719	}
 720	if (!obj->shstrs_sec_idx) {
 721		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 722		return -EINVAL;
 723	}
 724
 725	for (i = 1; i < obj->sec_cnt; i++) {
 726		sec = &obj->secs[i];
 727
 728		if (sec->sec_name[0] == '\0') {
 729			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 730			return -EINVAL;
 731		}
 732
 733		if (is_dwarf_sec_name(sec->sec_name))
 734			continue;
 735
 736		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
 737			pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
 738				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 739				obj->filename);
 740			return -EINVAL;
 741		}
 742		if (sec->shdr->sh_addralign != sec->data->d_align) {
 743			pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
 744				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 745				(long long unsigned)sec->data->d_align, obj->filename);
 746			return -EINVAL;
 747		}
 748
 749		if (sec->shdr->sh_size != sec->data->d_size) {
 750			pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
 751				sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 752				(long long unsigned)sec->data->d_size, obj->filename);
 753			return -EINVAL;
 754		}
 755
 756		switch (sec->shdr->sh_type) {
 757		case SHT_SYMTAB:
 758			err = linker_sanity_check_elf_symtab(obj, sec);
 759			if (err)
 760				return err;
 761			break;
 762		case SHT_STRTAB:
 763			break;
 764		case SHT_PROGBITS:
 765			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 766				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
 767					pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
 768						sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 769						obj->filename);
 770					return -EINVAL;
 771				}
 772			}
 773			break;
 774		case SHT_NOBITS:
 775			break;
 776		case SHT_REL:
 777			err = linker_sanity_check_elf_relos(obj, sec);
 778			if (err)
 779				return err;
 780			break;
 781		case SHT_LLVM_ADDRSIG:
 782			break;
 783		default:
 784			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 785				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 786			return -EINVAL;
 787		}
 788	}
 789
 790	return 0;
 791}
 792
 793static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 794{
 795	struct src_sec *link_sec;
 796	Elf64_Sym *sym;
 797	int i, n;
 798
 799	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 800		return -EINVAL;
 801	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 802		return -EINVAL;
 803
 804	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 805		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 806			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 807		return -EINVAL;
 808	}
 809	link_sec = &obj->secs[sec->shdr->sh_link];
 810	if (link_sec->shdr->sh_type != SHT_STRTAB) {
 811		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 812			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 813		return -EINVAL;
 814	}
 815
 816	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 817	sym = sec->data->d_buf;
 818	for (i = 0; i < n; i++, sym++) {
 819		int sym_type = ELF64_ST_TYPE(sym->st_info);
 820		int sym_bind = ELF64_ST_BIND(sym->st_info);
 821		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 822
 823		if (i == 0) {
 824			if (sym->st_name != 0 || sym->st_info != 0
 825			    || sym->st_other != 0 || sym->st_shndx != 0
 826			    || sym->st_value != 0 || sym->st_size != 0) {
 827				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 828				return -EINVAL;
 829			}
 830			continue;
 831		}
 832		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 833			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 834				i, sec->sec_idx, sym_bind);
 835			return -EINVAL;
 836		}
 837		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 838			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 839				i, sec->sec_idx, sym_vis);
 840			return -EINVAL;
 841		}
 842		if (sym->st_shndx == 0) {
 843			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 844			    || sym->st_value != 0 || sym->st_size != 0) {
 845				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 846					i, obj->filename);
 847
 848				return -EINVAL;
 849			}
 850			continue;
 851		}
 852		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 853			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 854				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 855			return -EINVAL;
 856		}
 857		if (sym_type == STT_SECTION) {
 858			if (sym->st_value != 0)
 859				return -EINVAL;
 860			continue;
 861		}
 862	}
 863
 864	return 0;
 865}
 866
 867static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 868{
 869	struct src_sec *link_sec, *sym_sec;
 870	Elf64_Rel *relo;
 871	int i, n;
 872
 873	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 874		return -EINVAL;
 875	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 876		return -EINVAL;
 877
 878	/* SHT_REL's sh_link should point to SYMTAB */
 879	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 880		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 881			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 882		return -EINVAL;
 883	}
 884
 885	/* SHT_REL's sh_info points to relocated section */
 886	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 887		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 888			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 889		return -EINVAL;
 890	}
 891	link_sec = &obj->secs[sec->shdr->sh_info];
 892
 893	/* .rel<secname> -> <secname> pattern is followed */
 894	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 895	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 896		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 897			sec->sec_idx, obj->filename);
 898		return -EINVAL;
 899	}
 900
 901	/* don't further validate relocations for ignored sections */
 902	if (link_sec->skipped)
 903		return 0;
 904
 905	/* relocatable section is data or instructions */
 906	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 907		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 908			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 909		return -EINVAL;
 910	}
 911
 912	/* check sanity of each relocation */
 913	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 914	relo = sec->data->d_buf;
 915	sym_sec = &obj->secs[obj->symtab_sec_idx];
 916	for (i = 0; i < n; i++, relo++) {
 917		size_t sym_idx = ELF64_R_SYM(relo->r_info);
 918		size_t sym_type = ELF64_R_TYPE(relo->r_info);
 919
 920		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
 921		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
 922			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 923				i, sec->sec_idx, sym_type, obj->filename);
 924			return -EINVAL;
 925		}
 926
 927		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 928			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 929				i, sec->sec_idx, sym_idx, obj->filename);
 930			return -EINVAL;
 931		}
 932
 933		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 934			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 935				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 936					i, sec->sec_idx, sym_idx, obj->filename);
 937				return -EINVAL;
 938			}
 939		}
 940	}
 941
 942	return 0;
 943}
 944
 945static int check_btf_type_id(__u32 *type_id, void *ctx)
 946{
 947	struct btf *btf = ctx;
 948
 949	if (*type_id >= btf__type_cnt(btf))
 950		return -EINVAL;
 951
 952	return 0;
 953}
 954
 955static int check_btf_str_off(__u32 *str_off, void *ctx)
 956{
 957	struct btf *btf = ctx;
 958	const char *s;
 959
 960	s = btf__str_by_offset(btf, *str_off);
 961
 962	if (!s)
 963		return -EINVAL;
 964
 965	return 0;
 966}
 967
 968static int linker_sanity_check_btf(struct src_obj *obj)
 969{
 970	struct btf_type *t;
 971	int i, n, err;
 972
 973	if (!obj->btf)
 974		return 0;
 975
 976	n = btf__type_cnt(obj->btf);
 977	for (i = 1; i < n; i++) {
 978		struct btf_field_iter it;
 979		__u32 *type_id, *str_off;
 980
 981		t = btf_type_by_id(obj->btf, i);
 982
 983		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
 984		if (err)
 985			return err;
 986		while ((type_id = btf_field_iter_next(&it))) {
 987			if (*type_id >= n)
 988				return -EINVAL;
 989		}
 990
 991		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
 992		if (err)
 993			return err;
 994		while ((str_off = btf_field_iter_next(&it))) {
 995			if (!btf__str_by_offset(obj->btf, *str_off))
 996				return -EINVAL;
 997		}
 998	}
 999
1000	return 0;
1001}
1002
1003static int linker_sanity_check_btf_ext(struct src_obj *obj)
1004{
1005	int err = 0;
1006
1007	if (!obj->btf_ext)
1008		return 0;
1009
1010	/* can't use .BTF.ext without .BTF */
1011	if (!obj->btf)
1012		return -EINVAL;
1013
1014	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1015	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1016	if (err)
1017		return err;
1018
1019	return 0;
1020}
1021
1022static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1023{
1024	Elf_Scn *scn;
1025	Elf_Data *data;
1026	Elf64_Shdr *shdr;
1027	int name_off;
1028
1029	dst_sec->sec_sz = 0;
1030	dst_sec->sec_idx = 0;
1031	dst_sec->ephemeral = src_sec->ephemeral;
1032
1033	/* ephemeral sections are just thin section shells lacking most parts */
1034	if (src_sec->ephemeral)
1035		return 0;
1036
1037	scn = elf_newscn(linker->elf);
1038	if (!scn)
1039		return -ENOMEM;
1040	data = elf_newdata(scn);
1041	if (!data)
1042		return -ENOMEM;
1043	shdr = elf64_getshdr(scn);
1044	if (!shdr)
1045		return -ENOMEM;
1046
1047	dst_sec->scn = scn;
1048	dst_sec->shdr = shdr;
1049	dst_sec->data = data;
1050	dst_sec->sec_idx = elf_ndxscn(scn);
1051
1052	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1053	if (name_off < 0)
1054		return name_off;
1055
1056	shdr->sh_name = name_off;
1057	shdr->sh_type = src_sec->shdr->sh_type;
1058	shdr->sh_flags = src_sec->shdr->sh_flags;
1059	shdr->sh_size = 0;
1060	/* sh_link and sh_info have different meaning for different types of
1061	 * sections, so we leave it up to the caller code to fill them in, if
1062	 * necessary
1063	 */
1064	shdr->sh_link = 0;
1065	shdr->sh_info = 0;
1066	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1067	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1068
1069	data->d_type = src_sec->data->d_type;
1070	data->d_size = 0;
1071	data->d_buf = NULL;
1072	data->d_align = src_sec->data->d_align;
1073	data->d_off = 0;
1074
1075	return 0;
1076}
1077
1078static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1079{
1080	struct dst_sec *sec;
1081	int i;
1082
1083	for (i = 1; i < linker->sec_cnt; i++) {
1084		sec = &linker->secs[i];
1085
1086		if (strcmp(sec->sec_name, sec_name) == 0)
1087			return sec;
1088	}
1089
1090	return NULL;
1091}
1092
1093static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1094{
1095	if (dst->ephemeral || src->ephemeral)
1096		return true;
1097
1098	if (dst->shdr->sh_type != src->shdr->sh_type) {
1099		pr_warn("sec %s types mismatch\n", dst->sec_name);
1100		return false;
1101	}
1102	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1103		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1104		return false;
1105	}
1106	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1107		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1108		return false;
1109	}
1110
1111	return true;
1112}
1113
1114static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1115{
1116	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1117		return false;
1118	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1119		return false;
1120	return true;
1121}
1122
1123static bool is_exec_sec(struct dst_sec *sec)
1124{
1125	if (!sec || sec->ephemeral)
1126		return false;
1127	return (sec->shdr->sh_type == SHT_PROGBITS) &&
1128	       (sec->shdr->sh_flags & SHF_EXECINSTR);
1129}
1130
1131static void exec_sec_bswap(void *raw_data, int size)
1132{
1133	const int insn_cnt = size / sizeof(struct bpf_insn);
1134	struct bpf_insn *insn = raw_data;
1135	int i;
1136
1137	for (i = 0; i < insn_cnt; i++, insn++)
1138		bpf_insn_bswap(insn);
1139}
1140
1141static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1142{
1143	void *tmp;
1144	size_t dst_align, src_align;
1145	size_t dst_align_sz, dst_final_sz;
1146	int err;
1147
1148	/* Ephemeral source section doesn't contribute anything to ELF
1149	 * section data.
1150	 */
1151	if (src->ephemeral)
1152		return 0;
1153
1154	/* Some sections (like .maps) can contain both externs (and thus be
1155	 * ephemeral) and non-externs (map definitions). So it's possible that
1156	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1157	 * first non-ephemeral entity appears. In such case, we add ELF
1158	 * section, data, etc.
1159	 */
1160	if (dst->ephemeral) {
1161		err = init_sec(linker, dst, src);
1162		if (err)
1163			return err;
1164	}
1165
1166	dst_align = dst->shdr->sh_addralign;
1167	src_align = src->shdr->sh_addralign;
1168	if (dst_align == 0)
1169		dst_align = 1;
1170	if (dst_align < src_align)
1171		dst_align = src_align;
1172
1173	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1174
1175	/* no need to re-align final size */
1176	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1177
1178	if (src->shdr->sh_type != SHT_NOBITS) {
1179		tmp = realloc(dst->raw_data, dst_final_sz);
1180		/* If dst_align_sz == 0, realloc() behaves in a special way:
1181		 * 1. When dst->raw_data is NULL it returns:
1182		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1183		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1184		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1185		 *    at step (1).
1186		 *
1187		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1188		 * dst->raw_data would be freed again in bpf_linker__free().
1189		 *
1190		 * [1] man 3 realloc
1191		 */
1192		if (!tmp && dst_align_sz > 0)
1193			return -ENOMEM;
1194		dst->raw_data = tmp;
1195
1196		/* pad dst section, if it's alignment forced size increase */
1197		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1198		/* now copy src data at a properly aligned offset */
1199		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1200
1201		/* convert added bpf insns to native byte-order */
1202		if (linker->swapped_endian && is_exec_sec(dst))
1203			exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1204	}
1205
1206	dst->sec_sz = dst_final_sz;
1207	dst->shdr->sh_size = dst_final_sz;
1208	dst->data->d_size = dst_final_sz;
1209
1210	dst->shdr->sh_addralign = dst_align;
1211	dst->data->d_align = dst_align;
1212
1213	src->dst_off = dst_align_sz;
1214
1215	return 0;
1216}
1217
1218static bool is_data_sec(struct src_sec *sec)
1219{
1220	if (!sec || sec->skipped)
1221		return false;
1222	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1223	if (sec->ephemeral)
1224		return true;
1225	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1226}
1227
1228static bool is_relo_sec(struct src_sec *sec)
1229{
1230	if (!sec || sec->skipped || sec->ephemeral)
1231		return false;
1232	return sec->shdr->sh_type == SHT_REL;
1233}
1234
1235static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1236{
1237	int i, err;
1238
1239	for (i = 1; i < obj->sec_cnt; i++) {
1240		struct src_sec *src_sec;
1241		struct dst_sec *dst_sec;
1242
1243		src_sec = &obj->secs[i];
1244		if (!is_data_sec(src_sec))
1245			continue;
1246
1247		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1248		if (!dst_sec) {
1249			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1250			if (!dst_sec)
1251				return -ENOMEM;
1252			err = init_sec(linker, dst_sec, src_sec);
1253			if (err) {
1254				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1255				return err;
1256			}
1257		} else {
1258			if (!secs_match(dst_sec, src_sec)) {
1259				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1260				return -1;
1261			}
1262
1263			/* "license" and "version" sections are deduped */
1264			if (strcmp(src_sec->sec_name, "license") == 0
1265			    || strcmp(src_sec->sec_name, "version") == 0) {
1266				if (!sec_content_is_same(dst_sec, src_sec)) {
1267					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1268					return -EINVAL;
1269				}
1270				src_sec->skipped = true;
1271				src_sec->dst_id = dst_sec->id;
1272				continue;
1273			}
1274		}
1275
1276		/* record mapped section index */
1277		src_sec->dst_id = dst_sec->id;
1278
1279		err = extend_sec(linker, dst_sec, src_sec);
1280		if (err)
1281			return err;
1282	}
1283
1284	return 0;
1285}
1286
1287static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1288{
1289	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1290	Elf64_Sym *sym = symtab->data->d_buf;
1291	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1292	int str_sec_idx = symtab->shdr->sh_link;
1293	const char *sym_name;
1294
1295	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1296	if (!obj->sym_map)
1297		return -ENOMEM;
1298
1299	for (i = 0; i < n; i++, sym++) {
1300		/* We already validated all-zero symbol #0 and we already
1301		 * appended it preventively to the final SYMTAB, so skip it.
1302		 */
1303		if (i == 0)
1304			continue;
1305
1306		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1307		if (!sym_name) {
1308			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1309			return -EINVAL;
1310		}
1311
1312		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1313		if (err)
1314			return err;
1315	}
1316
1317	return 0;
1318}
1319
1320static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1321{
1322	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1323	Elf64_Sym *syms = symtab->raw_data;
1324
1325	return &syms[sym_idx];
1326}
1327
1328static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1329{
1330	struct glob_sym *glob_sym;
1331	const char *name;
1332	int i;
1333
1334	for (i = 0; i < linker->glob_sym_cnt; i++) {
1335		glob_sym = &linker->glob_syms[i];
1336		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1337
1338		if (strcmp(name, sym_name) == 0)
1339			return glob_sym;
1340	}
1341
1342	return NULL;
1343}
1344
1345static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1346{
1347	struct glob_sym *syms, *sym;
1348
1349	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1350				   sizeof(*linker->glob_syms));
1351	if (!syms)
1352		return NULL;
1353
1354	sym = &syms[linker->glob_sym_cnt];
1355	memset(sym, 0, sizeof(*sym));
1356	sym->var_idx = -1;
1357
1358	linker->glob_syms = syms;
1359	linker->glob_sym_cnt++;
1360
1361	return sym;
1362}
1363
1364static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1365				 const struct btf *btf1, __u32 id1,
1366				 const struct btf *btf2, __u32 id2)
1367{
1368	const struct btf_type *t1, *t2;
1369	bool is_static1, is_static2;
1370	const char *n1, *n2;
1371	int i, n;
1372
1373recur:
1374	n1 = n2 = NULL;
1375	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1376	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1377
1378	/* check if only one side is FWD, otherwise handle with common logic */
1379	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1380		n1 = btf__str_by_offset(btf1, t1->name_off);
1381		n2 = btf__str_by_offset(btf2, t2->name_off);
1382		if (strcmp(n1, n2) != 0) {
1383			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1384				sym_name, n1, n2);
1385			return false;
1386		}
1387		/* validate if FWD kind matches concrete kind */
1388		if (btf_is_fwd(t1)) {
1389			if (btf_kflag(t1) && btf_is_union(t2))
1390				return true;
1391			if (!btf_kflag(t1) && btf_is_struct(t2))
1392				return true;
1393			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1394				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1395		} else {
1396			if (btf_kflag(t2) && btf_is_union(t1))
1397				return true;
1398			if (!btf_kflag(t2) && btf_is_struct(t1))
1399				return true;
1400			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1401				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1402		}
1403		return false;
1404	}
1405
1406	if (btf_kind(t1) != btf_kind(t2)) {
1407		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1408			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1409		return false;
1410	}
1411
1412	switch (btf_kind(t1)) {
1413	case BTF_KIND_STRUCT:
1414	case BTF_KIND_UNION:
1415	case BTF_KIND_ENUM:
1416	case BTF_KIND_ENUM64:
1417	case BTF_KIND_FWD:
1418	case BTF_KIND_FUNC:
1419	case BTF_KIND_VAR:
1420		n1 = btf__str_by_offset(btf1, t1->name_off);
1421		n2 = btf__str_by_offset(btf2, t2->name_off);
1422		if (strcmp(n1, n2) != 0) {
1423			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1424				sym_name, btf_kind_str(t1), n1, n2);
1425			return false;
1426		}
1427		break;
1428	default:
1429		break;
1430	}
1431
1432	switch (btf_kind(t1)) {
1433	case BTF_KIND_UNKN: /* void */
1434	case BTF_KIND_FWD:
1435		return true;
1436	case BTF_KIND_INT:
1437	case BTF_KIND_FLOAT:
1438	case BTF_KIND_ENUM:
1439	case BTF_KIND_ENUM64:
1440		/* ignore encoding for int and enum values for enum */
1441		if (t1->size != t2->size) {
1442			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1443				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1444			return false;
1445		}
1446		return true;
1447	case BTF_KIND_PTR:
1448		/* just validate overall shape of the referenced type, so no
1449		 * contents comparison for struct/union, and allowed fwd vs
1450		 * struct/union
1451		 */
1452		exact = false;
1453		id1 = t1->type;
1454		id2 = t2->type;
1455		goto recur;
1456	case BTF_KIND_ARRAY:
1457		/* ignore index type and array size */
1458		id1 = btf_array(t1)->type;
1459		id2 = btf_array(t2)->type;
1460		goto recur;
1461	case BTF_KIND_FUNC:
1462		/* extern and global linkages are compatible */
1463		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1464		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1465		if (is_static1 != is_static2) {
1466			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1467			return false;
1468		}
1469
1470		id1 = t1->type;
1471		id2 = t2->type;
1472		goto recur;
1473	case BTF_KIND_VAR:
1474		/* extern and global linkages are compatible */
1475		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1476		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1477		if (is_static1 != is_static2) {
1478			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1479			return false;
1480		}
1481
1482		id1 = t1->type;
1483		id2 = t2->type;
1484		goto recur;
1485	case BTF_KIND_STRUCT:
1486	case BTF_KIND_UNION: {
1487		const struct btf_member *m1, *m2;
1488
1489		if (!exact)
1490			return true;
1491
1492		if (btf_vlen(t1) != btf_vlen(t2)) {
1493			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1494				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1495			return false;
1496		}
1497
1498		n = btf_vlen(t1);
1499		m1 = btf_members(t1);
1500		m2 = btf_members(t2);
1501		for (i = 0; i < n; i++, m1++, m2++) {
1502			n1 = btf__str_by_offset(btf1, m1->name_off);
1503			n2 = btf__str_by_offset(btf2, m2->name_off);
1504			if (strcmp(n1, n2) != 0) {
1505				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1506					sym_name, i, n1, n2);
1507				return false;
1508			}
1509			if (m1->offset != m2->offset) {
1510				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1511					sym_name, i, n1);
1512				return false;
1513			}
1514			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1515				return false;
1516		}
1517
1518		return true;
1519	}
1520	case BTF_KIND_FUNC_PROTO: {
1521		const struct btf_param *m1, *m2;
1522
1523		if (btf_vlen(t1) != btf_vlen(t2)) {
1524			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1525				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1526			return false;
1527		}
1528
1529		n = btf_vlen(t1);
1530		m1 = btf_params(t1);
1531		m2 = btf_params(t2);
1532		for (i = 0; i < n; i++, m1++, m2++) {
1533			/* ignore func arg names */
1534			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1535				return false;
1536		}
1537
1538		/* now check return type as well */
1539		id1 = t1->type;
1540		id2 = t2->type;
1541		goto recur;
1542	}
1543
1544	/* skip_mods_and_typedefs() make this impossible */
1545	case BTF_KIND_TYPEDEF:
1546	case BTF_KIND_VOLATILE:
1547	case BTF_KIND_CONST:
1548	case BTF_KIND_RESTRICT:
1549	/* DATASECs are never compared with each other */
1550	case BTF_KIND_DATASEC:
1551	default:
1552		pr_warn("global '%s': unsupported BTF kind %s\n",
1553			sym_name, btf_kind_str(t1));
1554		return false;
1555	}
1556}
1557
1558static bool map_defs_match(const char *sym_name,
1559			   const struct btf *main_btf,
1560			   const struct btf_map_def *main_def,
1561			   const struct btf_map_def *main_inner_def,
1562			   const struct btf *extra_btf,
1563			   const struct btf_map_def *extra_def,
1564			   const struct btf_map_def *extra_inner_def)
1565{
1566	const char *reason;
1567
1568	if (main_def->map_type != extra_def->map_type) {
1569		reason = "type";
1570		goto mismatch;
1571	}
1572
1573	/* check key type/size match */
1574	if (main_def->key_size != extra_def->key_size) {
1575		reason = "key_size";
1576		goto mismatch;
1577	}
1578	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1579		reason = "key type";
1580		goto mismatch;
1581	}
1582	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1583	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1584				      main_btf, main_def->key_type_id,
1585				      extra_btf, extra_def->key_type_id)) {
1586		reason = "key type";
1587		goto mismatch;
1588	}
1589
1590	/* validate value type/size match */
1591	if (main_def->value_size != extra_def->value_size) {
1592		reason = "value_size";
1593		goto mismatch;
1594	}
1595	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1596		reason = "value type";
1597		goto mismatch;
1598	}
1599	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1600	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1601				      main_btf, main_def->value_type_id,
1602				      extra_btf, extra_def->value_type_id)) {
1603		reason = "key type";
1604		goto mismatch;
1605	}
1606
1607	if (main_def->max_entries != extra_def->max_entries) {
1608		reason = "max_entries";
1609		goto mismatch;
1610	}
1611	if (main_def->map_flags != extra_def->map_flags) {
1612		reason = "map_flags";
1613		goto mismatch;
1614	}
1615	if (main_def->numa_node != extra_def->numa_node) {
1616		reason = "numa_node";
1617		goto mismatch;
1618	}
1619	if (main_def->pinning != extra_def->pinning) {
1620		reason = "pinning";
1621		goto mismatch;
1622	}
1623
1624	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1625		reason = "inner map";
1626		goto mismatch;
1627	}
1628
1629	if (main_def->parts & MAP_DEF_INNER_MAP) {
1630		char inner_map_name[128];
1631
1632		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1633
1634		return map_defs_match(inner_map_name,
1635				      main_btf, main_inner_def, NULL,
1636				      extra_btf, extra_inner_def, NULL);
1637	}
1638
1639	return true;
1640
1641mismatch:
1642	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1643	return false;
1644}
1645
1646static bool glob_map_defs_match(const char *sym_name,
1647				struct bpf_linker *linker, struct glob_sym *glob_sym,
1648				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1649{
1650	struct btf_map_def dst_def = {}, dst_inner_def = {};
1651	struct btf_map_def src_def = {}, src_inner_def = {};
1652	const struct btf_type *t;
1653	int err;
1654
1655	t = btf__type_by_id(obj->btf, btf_id);
1656	if (!btf_is_var(t)) {
1657		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1658		return false;
1659	}
1660	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1661
1662	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1663	if (err) {
1664		pr_warn("global '%s': invalid map definition\n", sym_name);
1665		return false;
1666	}
1667
1668	/* re-parse existing map definition */
1669	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1670	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1671	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1672	if (err) {
1673		/* this should not happen, because we already validated it */
1674		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1675		return false;
1676	}
1677
1678	/* Currently extern map definition has to be complete and match
1679	 * concrete map definition exactly. This restriction might be lifted
1680	 * in the future.
1681	 */
1682	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1683			      obj->btf, &src_def, &src_inner_def);
1684}
1685
1686static bool glob_syms_match(const char *sym_name,
1687			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1688			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1689{
1690	const struct btf_type *src_t;
1691
1692	/* if we are dealing with externs, BTF types describing both global
1693	 * and extern VARs/FUNCs should be completely present in all files
1694	 */
1695	if (!glob_sym->btf_id || !btf_id) {
1696		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1697		return false;
1698	}
1699
1700	src_t = btf__type_by_id(obj->btf, btf_id);
1701	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1702		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1703			btf_kind_str(src_t), sym_name);
1704		return false;
1705	}
1706
1707	/* deal with .maps definitions specially */
1708	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1709		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1710
1711	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1712				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1713		return false;
1714
1715	return true;
1716}
1717
1718static bool btf_is_non_static(const struct btf_type *t)
1719{
1720	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1721	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1722}
1723
1724static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1725			     int *out_btf_sec_id, int *out_btf_id)
1726{
1727	int i, j, n, m, btf_id = 0;
1728	const struct btf_type *t;
1729	const struct btf_var_secinfo *vi;
1730	const char *name;
1731
1732	if (!obj->btf) {
1733		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1734		return -EINVAL;
1735	}
1736
1737	n = btf__type_cnt(obj->btf);
1738	for (i = 1; i < n; i++) {
1739		t = btf__type_by_id(obj->btf, i);
1740
1741		/* some global and extern FUNCs and VARs might not be associated with any
1742		 * DATASEC, so try to detect them in the same pass
1743		 */
1744		if (btf_is_non_static(t)) {
1745			name = btf__str_by_offset(obj->btf, t->name_off);
1746			if (strcmp(name, sym_name) != 0)
1747				continue;
1748
1749			/* remember and still try to find DATASEC */
1750			btf_id = i;
1751			continue;
1752		}
1753
1754		if (!btf_is_datasec(t))
1755			continue;
1756
1757		vi = btf_var_secinfos(t);
1758		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1759			t = btf__type_by_id(obj->btf, vi->type);
1760			name = btf__str_by_offset(obj->btf, t->name_off);
1761
1762			if (strcmp(name, sym_name) != 0)
1763				continue;
1764			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1765				continue;
1766			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1767				continue;
1768
1769			if (btf_id && btf_id != vi->type) {
1770				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1771					sym_name, btf_id, vi->type);
1772				return -EINVAL;
1773			}
1774
1775			*out_btf_sec_id = i;
1776			*out_btf_id = vi->type;
1777
1778			return 0;
1779		}
1780	}
1781
1782	/* free-floating extern or global FUNC */
1783	if (btf_id) {
1784		*out_btf_sec_id = 0;
1785		*out_btf_id = btf_id;
1786		return 0;
1787	}
1788
1789	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1790	return -ENOENT;
1791}
1792
1793static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1794{
1795	struct src_sec *sec;
1796	int i;
1797
1798	for (i = 1; i < obj->sec_cnt; i++) {
1799		sec = &obj->secs[i];
1800
1801		if (strcmp(sec->sec_name, sec_name) == 0)
1802			return sec;
1803	}
1804
1805	return NULL;
1806}
1807
1808static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1809				    struct btf *src_btf, int src_id)
1810{
1811	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1812	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1813	struct btf_param *src_p, *dst_p;
1814	const char *s;
1815	int i, n, off;
1816
1817	/* We already made sure that source and destination types (FUNC or
1818	 * VAR) match in terms of types and argument names.
1819	 */
1820	if (btf_is_var(dst_t)) {
1821		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1822		return 0;
1823	}
1824
1825	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1826
1827	/* now onto FUNC_PROTO types */
1828	src_t = btf_type_by_id(src_btf, src_t->type);
1829	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1830
1831	/* Fill in all the argument names, which for extern FUNCs are missing.
1832	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1833	 * will be taken care of by BTF dedup at the very end.
1834	 * It might be that BTF types for extern in one file has less/more BTF
1835	 * information (e.g., FWD instead of full STRUCT/UNION information),
1836	 * but that should be (in most cases, subject to BTF dedup rules)
1837	 * handled and resolved by BTF dedup algorithm as well, so we won't
1838	 * worry about it. Our only job is to make sure that argument names
1839	 * are populated on both sides, otherwise BTF dedup will pedantically
1840	 * consider them different.
1841	 */
1842	src_p = btf_params(src_t);
1843	dst_p = btf_params(dst_t);
1844	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1845		if (!src_p->name_off)
1846			continue;
1847
1848		/* src_btf has more complete info, so add name to dst_btf */
1849		s = btf__str_by_offset(src_btf, src_p->name_off);
1850		off = btf__add_str(dst_btf, s);
1851		if (off < 0)
1852			return off;
1853		dst_p->name_off = off;
1854	}
1855	return 0;
1856}
1857
1858static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1859{
1860	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1861}
1862
1863static void sym_update_type(Elf64_Sym *sym, int sym_type)
1864{
1865	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1866}
1867
1868static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1869{
1870	/* libelf doesn't provide setters for ST_VISIBILITY,
1871	 * but it is stored in the lower 2 bits of st_other
1872	 */
1873	sym->st_other &= ~0x03;
1874	sym->st_other |= sym_vis;
1875}
1876
1877static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1878				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1879{
1880	struct src_sec *src_sec = NULL;
1881	struct dst_sec *dst_sec = NULL;
1882	struct glob_sym *glob_sym = NULL;
1883	int name_off, sym_type, sym_bind, sym_vis, err;
1884	int btf_sec_id = 0, btf_id = 0;
1885	size_t dst_sym_idx;
1886	Elf64_Sym *dst_sym;
1887	bool sym_is_extern;
1888
1889	sym_type = ELF64_ST_TYPE(sym->st_info);
1890	sym_bind = ELF64_ST_BIND(sym->st_info);
1891	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1892	sym_is_extern = sym->st_shndx == SHN_UNDEF;
1893
1894	if (sym_is_extern) {
1895		if (!obj->btf) {
1896			pr_warn("externs without BTF info are not supported\n");
1897			return -ENOTSUP;
1898		}
1899	} else if (sym->st_shndx < SHN_LORESERVE) {
1900		src_sec = &obj->secs[sym->st_shndx];
1901		if (src_sec->skipped)
1902			return 0;
1903		dst_sec = &linker->secs[src_sec->dst_id];
1904
1905		/* allow only one STT_SECTION symbol per section */
1906		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1907			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1908			return 0;
1909		}
1910	}
1911
1912	if (sym_bind == STB_LOCAL)
1913		goto add_sym;
1914
1915	/* find matching BTF info */
1916	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1917	if (err)
1918		return err;
1919
1920	if (sym_is_extern && btf_sec_id) {
1921		const char *sec_name = NULL;
1922		const struct btf_type *t;
1923
1924		t = btf__type_by_id(obj->btf, btf_sec_id);
1925		sec_name = btf__str_by_offset(obj->btf, t->name_off);
1926
1927		/* Clang puts unannotated extern vars into
1928		 * '.extern' BTF DATASEC. Treat them the same
1929		 * as unannotated extern funcs (which are
1930		 * currently not put into any DATASECs).
1931		 * Those don't have associated src_sec/dst_sec.
1932		 */
1933		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1934			src_sec = find_src_sec_by_name(obj, sec_name);
1935			if (!src_sec) {
1936				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1937				return -ENOENT;
1938			}
1939			dst_sec = &linker->secs[src_sec->dst_id];
1940		}
1941	}
1942
1943	glob_sym = find_glob_sym(linker, sym_name);
1944	if (glob_sym) {
1945		/* Preventively resolve to existing symbol. This is
1946		 * needed for further relocation symbol remapping in
1947		 * the next step of linking.
1948		 */
1949		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1950
1951		/* If both symbols are non-externs, at least one of
1952		 * them has to be STB_WEAK, otherwise they are in
1953		 * a conflict with each other.
1954		 */
1955		if (!sym_is_extern && !glob_sym->is_extern
1956		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1957			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1958				src_sym_idx, sym_name, obj->filename);
1959			return -EINVAL;
1960		}
1961
1962		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1963			return -EINVAL;
1964
1965		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1966
1967		/* If new symbol is strong, then force dst_sym to be strong as
1968		 * well; this way a mix of weak and non-weak extern
1969		 * definitions will end up being strong.
1970		 */
1971		if (sym_bind == STB_GLOBAL) {
1972			/* We still need to preserve type (NOTYPE or
1973			 * OBJECT/FUNC, depending on whether the symbol is
1974			 * extern or not)
1975			 */
1976			sym_update_bind(dst_sym, STB_GLOBAL);
1977			glob_sym->is_weak = false;
1978		}
1979
1980		/* Non-default visibility is "contaminating", with stricter
1981		 * visibility overwriting more permissive ones, even if more
1982		 * permissive visibility comes from just an extern definition.
1983		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1984		 * ensured by ELF symbol sanity checks above.
1985		 */
1986		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1987			sym_update_visibility(dst_sym, sym_vis);
1988
1989		/* If the new symbol is extern, then regardless if
1990		 * existing symbol is extern or resolved global, just
1991		 * keep the existing one untouched.
1992		 */
1993		if (sym_is_extern)
1994			return 0;
1995
1996		/* If existing symbol is a strong resolved symbol, bail out,
1997		 * because we lost resolution battle have nothing to
1998		 * contribute. We already checked above that there is no
1999		 * strong-strong conflict. We also already tightened binding
2000		 * and visibility, so nothing else to contribute at that point.
2001		 */
2002		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2003			return 0;
2004
2005		/* At this point, new symbol is strong non-extern,
2006		 * so overwrite glob_sym with new symbol information.
2007		 * Preserve binding and visibility.
2008		 */
2009		sym_update_type(dst_sym, sym_type);
2010		dst_sym->st_shndx = dst_sec->sec_idx;
2011		dst_sym->st_value = src_sec->dst_off + sym->st_value;
2012		dst_sym->st_size = sym->st_size;
2013
2014		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
2015		glob_sym->sec_id = dst_sec->id;
2016		glob_sym->is_extern = false;
2017
2018		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2019					     obj->btf, btf_id))
2020			return -EINVAL;
2021
2022		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2023		glob_sym->underlying_btf_id = 0;
2024
2025		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2026		return 0;
2027	}
2028
2029add_sym:
2030	name_off = strset__add_str(linker->strtab_strs, sym_name);
2031	if (name_off < 0)
2032		return name_off;
2033
2034	dst_sym = add_new_sym(linker, &dst_sym_idx);
2035	if (!dst_sym)
2036		return -ENOMEM;
2037
2038	dst_sym->st_name = name_off;
2039	dst_sym->st_info = sym->st_info;
2040	dst_sym->st_other = sym->st_other;
2041	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2042	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2043	dst_sym->st_size = sym->st_size;
2044
2045	obj->sym_map[src_sym_idx] = dst_sym_idx;
2046
2047	if (sym_type == STT_SECTION && dst_sym) {
2048		dst_sec->sec_sym_idx = dst_sym_idx;
2049		dst_sym->st_value = 0;
2050	}
2051
2052	if (sym_bind != STB_LOCAL) {
2053		glob_sym = add_glob_sym(linker);
2054		if (!glob_sym)
2055			return -ENOMEM;
2056
2057		glob_sym->sym_idx = dst_sym_idx;
2058		/* we use dst_sec->id (and not dst_sec->sec_idx), because
2059		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2060		 * sec_idx (as they don't have corresponding ELF section), but
2061		 * still have id. .extern doesn't have even ephemeral section
2062		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2063		 */
2064		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2065		glob_sym->name_off = name_off;
2066		/* we will fill btf_id in during BTF merging step */
2067		glob_sym->btf_id = 0;
2068		glob_sym->is_extern = sym_is_extern;
2069		glob_sym->is_weak = sym_bind == STB_WEAK;
2070	}
2071
2072	return 0;
2073}
2074
2075static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2076{
2077	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2078	int i, err;
2079
2080	for (i = 1; i < obj->sec_cnt; i++) {
2081		struct src_sec *src_sec, *src_linked_sec;
2082		struct dst_sec *dst_sec, *dst_linked_sec;
2083		Elf64_Rel *src_rel, *dst_rel;
2084		int j, n;
2085
2086		src_sec = &obj->secs[i];
2087		if (!is_relo_sec(src_sec))
2088			continue;
2089
2090		/* shdr->sh_info points to relocatable section */
2091		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2092		if (src_linked_sec->skipped)
2093			continue;
2094
2095		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2096		if (!dst_sec) {
2097			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2098			if (!dst_sec)
2099				return -ENOMEM;
2100			err = init_sec(linker, dst_sec, src_sec);
2101			if (err) {
2102				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2103				return err;
2104			}
2105		} else if (!secs_match(dst_sec, src_sec)) {
2106			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2107			return -1;
2108		}
2109
2110		/* shdr->sh_link points to SYMTAB */
2111		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2112
2113		/* shdr->sh_info points to relocated section */
2114		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2115		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2116
2117		src_sec->dst_id = dst_sec->id;
2118		err = extend_sec(linker, dst_sec, src_sec);
2119		if (err)
2120			return err;
2121
2122		src_rel = src_sec->data->d_buf;
2123		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2124		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2125		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2126			size_t src_sym_idx, dst_sym_idx, sym_type;
2127			Elf64_Sym *src_sym;
2128
2129			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2130			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2131
2132			dst_sym_idx = obj->sym_map[src_sym_idx];
2133			dst_rel->r_offset += src_linked_sec->dst_off;
2134			sym_type = ELF64_R_TYPE(src_rel->r_info);
2135			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2136
2137			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2138				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2139				struct bpf_insn *insn;
2140
2141				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2142					/* calls to the very first static function inside
2143					 * .text section at offset 0 will
2144					 * reference section symbol, not the
2145					 * function symbol. Fix that up,
2146					 * otherwise it won't be possible to
2147					 * relocate calls to two different
2148					 * static functions with the same name
2149					 * (rom two different object files)
2150					 */
2151					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2152					if (insn->code == (BPF_JMP | BPF_CALL))
2153						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2154					else
2155						insn->imm += sec->dst_off;
2156				} else {
2157					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2158					return -EINVAL;
2159				}
2160			}
2161
2162		}
2163	}
2164
2165	return 0;
2166}
2167
2168static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2169				   int sym_type, const char *sym_name)
2170{
2171	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2172	Elf64_Sym *sym = symtab->data->d_buf;
2173	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2174	int str_sec_idx = symtab->shdr->sh_link;
2175	const char *name;
2176
2177	for (i = 0; i < n; i++, sym++) {
2178		if (sym->st_shndx != sec_idx)
2179			continue;
2180		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2181			continue;
2182
2183		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2184		if (!name)
2185			return NULL;
2186
2187		if (strcmp(sym_name, name) != 0)
2188			continue;
2189
2190		return sym;
2191	}
2192
2193	return NULL;
2194}
2195
2196static int linker_fixup_btf(struct src_obj *obj)
2197{
2198	const char *sec_name;
2199	struct src_sec *sec;
2200	int i, j, n, m;
2201
2202	if (!obj->btf)
2203		return 0;
2204
2205	n = btf__type_cnt(obj->btf);
2206	for (i = 1; i < n; i++) {
2207		struct btf_var_secinfo *vi;
2208		struct btf_type *t;
2209
2210		t = btf_type_by_id(obj->btf, i);
2211		if (btf_kind(t) != BTF_KIND_DATASEC)
2212			continue;
2213
2214		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2215		sec = find_src_sec_by_name(obj, sec_name);
2216		if (sec) {
2217			/* record actual section size, unless ephemeral */
2218			if (sec->shdr)
2219				t->size = sec->shdr->sh_size;
2220		} else {
2221			/* BTF can have some sections that are not represented
2222			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2223			 * for special extern variables.
2224			 *
2225			 * For all but one such special (ephemeral)
2226			 * sections, we pre-create "section shells" to be able
2227			 * to keep track of extra per-section metadata later
2228			 * (e.g., those BTF extern variables).
2229			 *
2230			 * .extern is even more special, though, because it
2231			 * contains extern variables that need to be resolved
2232			 * by static linker, not libbpf and kernel. When such
2233			 * externs are resolved, we are going to remove them
2234			 * from .extern BTF section and might end up not
2235			 * needing it at all. Each resolved extern should have
2236			 * matching non-extern VAR/FUNC in other sections.
2237			 *
2238			 * We do support leaving some of the externs
2239			 * unresolved, though, to support cases of building
2240			 * libraries, which will later be linked against final
2241			 * BPF applications. So if at finalization we still
2242			 * see unresolved externs, we'll create .extern
2243			 * section on our own.
2244			 */
2245			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2246				continue;
2247
2248			sec = add_src_sec(obj, sec_name);
2249			if (!sec)
2250				return -ENOMEM;
2251
2252			sec->ephemeral = true;
2253			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2254		}
2255
2256		/* remember ELF section and its BTF type ID match */
2257		sec->sec_type_id = i;
2258
2259		/* fix up variable offsets */
2260		vi = btf_var_secinfos(t);
2261		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2262			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2263			const char *var_name;
2264			int var_linkage;
2265			Elf64_Sym *sym;
2266
2267			/* could be a variable or function */
2268			if (!btf_is_var(vt))
2269				continue;
2270
2271			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2272			var_linkage = btf_var(vt)->linkage;
2273
2274			/* no need to patch up static or extern vars */
2275			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2276				continue;
2277
2278			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2279			if (!sym) {
2280				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2281				return -ENOENT;
2282			}
2283
2284			vi->offset = sym->st_value;
2285		}
2286	}
2287
2288	return 0;
2289}
2290
2291static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2292{
2293	const struct btf_type *t;
2294	int i, j, n, start_id, id, err;
2295	const char *name;
2296
2297	if (!obj->btf)
2298		return 0;
2299
2300	start_id = btf__type_cnt(linker->btf);
2301	n = btf__type_cnt(obj->btf);
2302
2303	obj->btf_type_map = calloc(n + 1, sizeof(int));
2304	if (!obj->btf_type_map)
2305		return -ENOMEM;
2306
2307	for (i = 1; i < n; i++) {
2308		struct glob_sym *glob_sym = NULL;
2309
2310		t = btf__type_by_id(obj->btf, i);
2311
2312		/* DATASECs are handled specially below */
2313		if (btf_kind(t) == BTF_KIND_DATASEC)
2314			continue;
2315
2316		if (btf_is_non_static(t)) {
2317			/* there should be glob_sym already */
2318			name = btf__str_by_offset(obj->btf, t->name_off);
2319			glob_sym = find_glob_sym(linker, name);
2320
2321			/* VARs without corresponding glob_sym are those that
2322			 * belong to skipped/deduplicated sections (i.e.,
2323			 * license and version), so just skip them
2324			 */
2325			if (!glob_sym)
2326				continue;
2327
2328			/* linker_append_elf_sym() might have requested
2329			 * updating underlying type ID, if extern was resolved
2330			 * to strong symbol or weak got upgraded to non-weak
2331			 */
2332			if (glob_sym->underlying_btf_id == 0)
2333				glob_sym->underlying_btf_id = -t->type;
2334
2335			/* globals from previous object files that match our
2336			 * VAR/FUNC already have a corresponding associated
2337			 * BTF type, so just make sure to use it
2338			 */
2339			if (glob_sym->btf_id) {
2340				/* reuse existing BTF type for global var/func */
2341				obj->btf_type_map[i] = glob_sym->btf_id;
2342				continue;
2343			}
2344		}
2345
2346		id = btf__add_type(linker->btf, obj->btf, t);
2347		if (id < 0) {
2348			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2349			return id;
2350		}
2351
2352		obj->btf_type_map[i] = id;
2353
2354		/* record just appended BTF type for var/func */
2355		if (glob_sym) {
2356			glob_sym->btf_id = id;
2357			glob_sym->underlying_btf_id = -t->type;
2358		}
2359	}
2360
2361	/* remap all the types except DATASECs */
2362	n = btf__type_cnt(linker->btf);
2363	for (i = start_id; i < n; i++) {
2364		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2365		struct btf_field_iter it;
2366		__u32 *type_id;
2367
2368		err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2369		if (err)
2370			return err;
2371
2372		while ((type_id = btf_field_iter_next(&it))) {
2373			int new_id = obj->btf_type_map[*type_id];
2374
2375			/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2376			if (new_id == 0 && *type_id != 0) {
2377				pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2378					*type_id);
2379				return -EINVAL;
2380			}
2381
2382			*type_id = obj->btf_type_map[*type_id];
2383		}
2384	}
2385
2386	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2387	 * actual type), if necessary
2388	 */
2389	for (i = 0; i < linker->glob_sym_cnt; i++) {
2390		struct glob_sym *glob_sym = &linker->glob_syms[i];
2391		struct btf_type *glob_t;
2392
2393		if (glob_sym->underlying_btf_id >= 0)
2394			continue;
2395
2396		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2397
2398		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2399		glob_t->type = glob_sym->underlying_btf_id;
2400	}
2401
2402	/* append DATASEC info */
2403	for (i = 1; i < obj->sec_cnt; i++) {
2404		struct src_sec *src_sec;
2405		struct dst_sec *dst_sec;
2406		const struct btf_var_secinfo *src_var;
2407		struct btf_var_secinfo *dst_var;
2408
2409		src_sec = &obj->secs[i];
2410		if (!src_sec->sec_type_id || src_sec->skipped)
2411			continue;
2412		dst_sec = &linker->secs[src_sec->dst_id];
2413
2414		/* Mark section as having BTF regardless of the presence of
2415		 * variables. In some cases compiler might generate empty BTF
2416		 * with no variables information. E.g., when promoting local
2417		 * array/structure variable initial values and BPF object
2418		 * file otherwise has no read-only static variables in
2419		 * .rodata. We need to preserve such empty BTF and just set
2420		 * correct section size.
2421		 */
2422		dst_sec->has_btf = true;
2423
2424		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2425		src_var = btf_var_secinfos(t);
2426		n = btf_vlen(t);
2427		for (j = 0; j < n; j++, src_var++) {
2428			void *sec_vars = dst_sec->sec_vars;
2429			int new_id = obj->btf_type_map[src_var->type];
2430			struct glob_sym *glob_sym = NULL;
2431
2432			t = btf_type_by_id(linker->btf, new_id);
2433			if (btf_is_non_static(t)) {
2434				name = btf__str_by_offset(linker->btf, t->name_off);
2435				glob_sym = find_glob_sym(linker, name);
2436				if (glob_sym->sec_id != dst_sec->id) {
2437					pr_warn("global '%s': section mismatch %d vs %d\n",
2438						name, glob_sym->sec_id, dst_sec->id);
2439					return -EINVAL;
2440				}
2441			}
2442
2443			/* If there is already a member (VAR or FUNC) mapped
2444			 * to the same type, don't add a duplicate entry.
2445			 * This will happen when multiple object files define
2446			 * the same extern VARs/FUNCs.
2447			 */
2448			if (glob_sym && glob_sym->var_idx >= 0) {
2449				__s64 sz;
2450
2451				/* FUNCs don't have size, nothing to update */
2452				if (btf_is_func(t))
2453					continue;
2454
2455				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2456				/* Because underlying BTF type might have
2457				 * changed, so might its size have changed, so
2458				 * re-calculate and update it in sec_var.
2459				 */
2460				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2461				if (sz < 0) {
2462					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2463						name, (int)sz);
2464					return -EINVAL;
2465				}
2466				dst_var->size = sz;
2467				continue;
2468			}
2469
2470			sec_vars = libbpf_reallocarray(sec_vars,
2471						       dst_sec->sec_var_cnt + 1,
2472						       sizeof(*dst_sec->sec_vars));
2473			if (!sec_vars)
2474				return -ENOMEM;
2475
2476			dst_sec->sec_vars = sec_vars;
2477			dst_sec->sec_var_cnt++;
2478
2479			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2480			dst_var->type = obj->btf_type_map[src_var->type];
2481			dst_var->size = src_var->size;
2482			dst_var->offset = src_sec->dst_off + src_var->offset;
2483
2484			if (glob_sym)
2485				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2486		}
2487	}
2488
2489	return 0;
2490}
2491
2492static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2493{
2494	void *tmp;
2495
2496	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2497	if (!tmp)
2498		return NULL;
2499	ext_data->recs = tmp;
2500
2501	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2502	memcpy(tmp, src_rec, ext_data->rec_sz);
2503
2504	ext_data->rec_cnt++;
2505
2506	return tmp;
2507}
2508
2509static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2510{
2511	const struct btf_ext_info_sec *ext_sec;
2512	const char *sec_name, *s;
2513	struct src_sec *src_sec;
2514	struct dst_sec *dst_sec;
2515	int rec_sz, str_off, i;
2516
2517	if (!obj->btf_ext)
2518		return 0;
2519
2520	rec_sz = obj->btf_ext->func_info.rec_size;
2521	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2522		struct bpf_func_info_min *src_rec, *dst_rec;
2523
2524		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2525		src_sec = find_src_sec_by_name(obj, sec_name);
2526		if (!src_sec) {
2527			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2528			return -EINVAL;
2529		}
2530		dst_sec = &linker->secs[src_sec->dst_id];
2531
2532		if (dst_sec->func_info.rec_sz == 0)
2533			dst_sec->func_info.rec_sz = rec_sz;
2534		if (dst_sec->func_info.rec_sz != rec_sz) {
2535			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2536			return -EINVAL;
2537		}
2538
2539		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2540			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2541			if (!dst_rec)
2542				return -ENOMEM;
2543
2544			dst_rec->insn_off += src_sec->dst_off;
2545			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2546		}
2547	}
2548
2549	rec_sz = obj->btf_ext->line_info.rec_size;
2550	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2551		struct bpf_line_info_min *src_rec, *dst_rec;
2552
2553		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2554		src_sec = find_src_sec_by_name(obj, sec_name);
2555		if (!src_sec) {
2556			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2557			return -EINVAL;
2558		}
2559		dst_sec = &linker->secs[src_sec->dst_id];
2560
2561		if (dst_sec->line_info.rec_sz == 0)
2562			dst_sec->line_info.rec_sz = rec_sz;
2563		if (dst_sec->line_info.rec_sz != rec_sz) {
2564			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2565			return -EINVAL;
2566		}
2567
2568		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2569			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2570			if (!dst_rec)
2571				return -ENOMEM;
2572
2573			dst_rec->insn_off += src_sec->dst_off;
2574
2575			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2576			str_off = btf__add_str(linker->btf, s);
2577			if (str_off < 0)
2578				return -ENOMEM;
2579			dst_rec->file_name_off = str_off;
2580
2581			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2582			str_off = btf__add_str(linker->btf, s);
2583			if (str_off < 0)
2584				return -ENOMEM;
2585			dst_rec->line_off = str_off;
2586
2587			/* dst_rec->line_col is fine */
2588		}
2589	}
2590
2591	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2592	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2593		struct bpf_core_relo *src_rec, *dst_rec;
2594
2595		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2596		src_sec = find_src_sec_by_name(obj, sec_name);
2597		if (!src_sec) {
2598			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2599			return -EINVAL;
2600		}
2601		dst_sec = &linker->secs[src_sec->dst_id];
2602
2603		if (dst_sec->core_relo_info.rec_sz == 0)
2604			dst_sec->core_relo_info.rec_sz = rec_sz;
2605		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2606			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2607			return -EINVAL;
2608		}
2609
2610		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2611			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2612			if (!dst_rec)
2613				return -ENOMEM;
2614
2615			dst_rec->insn_off += src_sec->dst_off;
2616			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2617
2618			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2619			str_off = btf__add_str(linker->btf, s);
2620			if (str_off < 0)
2621				return -ENOMEM;
2622			dst_rec->access_str_off = str_off;
2623
2624			/* dst_rec->kind is fine */
2625		}
2626	}
2627
2628	return 0;
2629}
2630
2631int bpf_linker__finalize(struct bpf_linker *linker)
2632{
2633	struct dst_sec *sec;
2634	size_t strs_sz;
2635	const void *strs;
2636	int err, i;
2637
2638	if (!linker->elf)
2639		return libbpf_err(-EINVAL);
2640
2641	err = finalize_btf(linker);
2642	if (err)
2643		return libbpf_err(err);
2644
2645	/* Finalize strings */
2646	strs_sz = strset__data_size(linker->strtab_strs);
2647	strs = strset__data(linker->strtab_strs);
2648
2649	sec = &linker->secs[linker->strtab_sec_idx];
2650	sec->data->d_align = 1;
2651	sec->data->d_off = 0LL;
2652	sec->data->d_buf = (void *)strs;
2653	sec->data->d_type = ELF_T_BYTE;
2654	sec->data->d_size = strs_sz;
2655	sec->shdr->sh_size = strs_sz;
2656
2657	for (i = 1; i < linker->sec_cnt; i++) {
2658		sec = &linker->secs[i];
2659
2660		/* STRTAB is handled specially above */
2661		if (sec->sec_idx == linker->strtab_sec_idx)
2662			continue;
2663
2664		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2665		if (!sec->scn)
2666			continue;
2667
2668		/* restore sections with bpf insns to target byte-order */
2669		if (linker->swapped_endian && is_exec_sec(sec))
2670			exec_sec_bswap(sec->raw_data, sec->sec_sz);
2671
2672		sec->data->d_buf = sec->raw_data;
2673	}
2674
2675	/* Finalize ELF layout */
2676	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2677		err = -EINVAL;
2678		pr_warn_elf("failed to finalize ELF layout");
2679		return libbpf_err(err);
2680	}
2681
2682	/* Write out final ELF contents */
2683	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2684		err = -EINVAL;
2685		pr_warn_elf("failed to write ELF contents");
2686		return libbpf_err(err);
2687	}
2688
2689	elf_end(linker->elf);
2690	close(linker->fd);
2691
2692	linker->elf = NULL;
2693	linker->fd = -1;
2694
2695	return 0;
2696}
2697
2698static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2699			     size_t align, const void *raw_data, size_t raw_sz)
2700{
2701	Elf_Scn *scn;
2702	Elf_Data *data;
2703	Elf64_Shdr *shdr;
2704	int name_off;
2705
2706	name_off = strset__add_str(linker->strtab_strs, sec_name);
2707	if (name_off < 0)
2708		return name_off;
2709
2710	scn = elf_newscn(linker->elf);
2711	if (!scn)
2712		return -ENOMEM;
2713	data = elf_newdata(scn);
2714	if (!data)
2715		return -ENOMEM;
2716	shdr = elf64_getshdr(scn);
2717	if (!shdr)
2718		return -EINVAL;
2719
2720	shdr->sh_name = name_off;
2721	shdr->sh_type = SHT_PROGBITS;
2722	shdr->sh_flags = 0;
2723	shdr->sh_size = raw_sz;
2724	shdr->sh_link = 0;
2725	shdr->sh_info = 0;
2726	shdr->sh_addralign = align;
2727	shdr->sh_entsize = 0;
2728
2729	data->d_type = ELF_T_BYTE;
2730	data->d_size = raw_sz;
2731	data->d_buf = (void *)raw_data;
2732	data->d_align = align;
2733	data->d_off = 0;
2734
2735	return 0;
2736}
2737
2738static int finalize_btf(struct bpf_linker *linker)
2739{
2740	enum btf_endianness link_endianness;
2741	LIBBPF_OPTS(btf_dedup_opts, opts);
2742	struct btf *btf = linker->btf;
2743	const void *raw_data;
2744	int i, j, id, err;
2745	__u32 raw_sz;
2746
2747	/* bail out if no BTF data was produced */
2748	if (btf__type_cnt(linker->btf) == 1)
2749		return 0;
2750
2751	for (i = 1; i < linker->sec_cnt; i++) {
2752		struct dst_sec *sec = &linker->secs[i];
2753
2754		if (!sec->has_btf)
2755			continue;
2756
2757		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2758		if (id < 0) {
2759			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2760				sec->sec_name, id);
2761			return id;
2762		}
2763
2764		for (j = 0; j < sec->sec_var_cnt; j++) {
2765			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2766
2767			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2768				return -EINVAL;
2769		}
2770	}
2771
2772	err = finalize_btf_ext(linker);
2773	if (err) {
2774		pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2775		return err;
2776	}
2777
2778	opts.btf_ext = linker->btf_ext;
2779	err = btf__dedup(linker->btf, &opts);
2780	if (err) {
2781		pr_warn("BTF dedup failed: %s\n", errstr(err));
2782		return err;
2783	}
2784
2785	/* Set .BTF and .BTF.ext output byte order */
2786	link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2787			  BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2788	btf__set_endianness(linker->btf, link_endianness);
2789	if (linker->btf_ext)
2790		btf_ext__set_endianness(linker->btf_ext, link_endianness);
2791
2792	/* Emit .BTF section */
2793	raw_data = btf__raw_data(linker->btf, &raw_sz);
2794	if (!raw_data)
2795		return -ENOMEM;
2796
2797	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2798	if (err) {
2799		pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2800		return err;
2801	}
2802
2803	/* Emit .BTF.ext section */
2804	if (linker->btf_ext) {
2805		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2806		if (!raw_data)
2807			return -ENOMEM;
2808
2809		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2810		if (err) {
2811			pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2812			return err;
2813		}
2814	}
2815
2816	return 0;
2817}
2818
2819static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2820			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2821{
2822	struct btf_ext_info_sec *sec_info;
2823	void *cur = output;
2824	int str_off;
2825	size_t sz;
2826
2827	if (!sec_data->rec_cnt)
2828		return 0;
2829
2830	str_off = btf__add_str(linker->btf, sec_name);
2831	if (str_off < 0)
2832		return -ENOMEM;
2833
2834	sec_info = cur;
2835	sec_info->sec_name_off = str_off;
2836	sec_info->num_info = sec_data->rec_cnt;
2837	cur += sizeof(struct btf_ext_info_sec);
2838
2839	sz = sec_data->rec_cnt * sec_data->rec_sz;
2840	memcpy(cur, sec_data->recs, sz);
2841	cur += sz;
2842
2843	return cur - output;
2844}
2845
2846static int finalize_btf_ext(struct bpf_linker *linker)
2847{
2848	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2849	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2850	struct btf_ext_header *hdr;
2851	void *data, *cur;
2852	int i, err, sz;
2853
2854	/* validate that all sections have the same .BTF.ext record sizes
2855	 * and calculate total data size for each type of data (func info,
2856	 * line info, core relos)
2857	 */
2858	for (i = 1; i < linker->sec_cnt; i++) {
2859		struct dst_sec *sec = &linker->secs[i];
2860
2861		if (sec->func_info.rec_cnt) {
2862			if (func_rec_sz == 0)
2863				func_rec_sz = sec->func_info.rec_sz;
2864			if (func_rec_sz != sec->func_info.rec_sz) {
2865				pr_warn("mismatch in func_info record size %zu != %u\n",
2866					func_rec_sz, sec->func_info.rec_sz);
2867				return -EINVAL;
2868			}
2869
2870			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2871		}
2872		if (sec->line_info.rec_cnt) {
2873			if (line_rec_sz == 0)
2874				line_rec_sz = sec->line_info.rec_sz;
2875			if (line_rec_sz != sec->line_info.rec_sz) {
2876				pr_warn("mismatch in line_info record size %zu != %u\n",
2877					line_rec_sz, sec->line_info.rec_sz);
2878				return -EINVAL;
2879			}
2880
2881			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2882		}
2883		if (sec->core_relo_info.rec_cnt) {
2884			if (core_relo_rec_sz == 0)
2885				core_relo_rec_sz = sec->core_relo_info.rec_sz;
2886			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2887				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2888					core_relo_rec_sz, sec->core_relo_info.rec_sz);
2889				return -EINVAL;
2890			}
2891
2892			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2893		}
2894	}
2895
2896	if (!funcs_sz && !lines_sz && !core_relos_sz)
2897		return 0;
2898
2899	total_sz += sizeof(struct btf_ext_header);
2900	if (funcs_sz) {
2901		funcs_sz += sizeof(__u32); /* record size prefix */
2902		total_sz += funcs_sz;
2903	}
2904	if (lines_sz) {
2905		lines_sz += sizeof(__u32); /* record size prefix */
2906		total_sz += lines_sz;
2907	}
2908	if (core_relos_sz) {
2909		core_relos_sz += sizeof(__u32); /* record size prefix */
2910		total_sz += core_relos_sz;
2911	}
2912
2913	cur = data = calloc(1, total_sz);
2914	if (!data)
2915		return -ENOMEM;
2916
2917	hdr = cur;
2918	hdr->magic = BTF_MAGIC;
2919	hdr->version = BTF_VERSION;
2920	hdr->flags = 0;
2921	hdr->hdr_len = sizeof(struct btf_ext_header);
2922	cur += sizeof(struct btf_ext_header);
2923
2924	/* All offsets are in bytes relative to the end of this header */
2925	hdr->func_info_off = 0;
2926	hdr->func_info_len = funcs_sz;
2927	hdr->line_info_off = funcs_sz;
2928	hdr->line_info_len = lines_sz;
2929	hdr->core_relo_off = funcs_sz + lines_sz;
2930	hdr->core_relo_len = core_relos_sz;
2931
2932	if (funcs_sz) {
2933		*(__u32 *)cur = func_rec_sz;
2934		cur += sizeof(__u32);
2935
2936		for (i = 1; i < linker->sec_cnt; i++) {
2937			struct dst_sec *sec = &linker->secs[i];
2938
2939			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2940			if (sz < 0) {
2941				err = sz;
2942				goto out;
2943			}
2944
2945			cur += sz;
2946		}
2947	}
2948
2949	if (lines_sz) {
2950		*(__u32 *)cur = line_rec_sz;
2951		cur += sizeof(__u32);
2952
2953		for (i = 1; i < linker->sec_cnt; i++) {
2954			struct dst_sec *sec = &linker->secs[i];
2955
2956			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2957			if (sz < 0) {
2958				err = sz;
2959				goto out;
2960			}
2961
2962			cur += sz;
2963		}
2964	}
2965
2966	if (core_relos_sz) {
2967		*(__u32 *)cur = core_relo_rec_sz;
2968		cur += sizeof(__u32);
2969
2970		for (i = 1; i < linker->sec_cnt; i++) {
2971			struct dst_sec *sec = &linker->secs[i];
2972
2973			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2974			if (sz < 0) {
2975				err = sz;
2976				goto out;
2977			}
2978
2979			cur += sz;
2980		}
2981	}
2982
2983	linker->btf_ext = btf_ext__new(data, total_sz);
2984	err = libbpf_get_error(linker->btf_ext);
2985	if (err) {
2986		linker->btf_ext = NULL;
2987		pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
2988		goto out;
2989	}
2990
2991out:
2992	free(data);
2993	return err;
2994}