Loading...
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2018 Facebook */
3
4#include <endian.h>
5#include <stdio.h>
6#include <stdlib.h>
7#include <string.h>
8#include <fcntl.h>
9#include <unistd.h>
10#include <errno.h>
11#include <sys/utsname.h>
12#include <sys/param.h>
13#include <sys/stat.h>
14#include <linux/kernel.h>
15#include <linux/err.h>
16#include <linux/btf.h>
17#include <gelf.h>
18#include "btf.h"
19#include "bpf.h"
20#include "libbpf.h"
21#include "libbpf_internal.h"
22#include "hashmap.h"
23
24/* make sure libbpf doesn't use kernel-only integer typedefs */
25#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
26
27#define BTF_MAX_NR_TYPES 0x7fffffffU
28#define BTF_MAX_STR_OFFSET 0x7fffffffU
29
30static struct btf_type btf_void;
31
32struct btf {
33 union {
34 struct btf_header *hdr;
35 void *data;
36 };
37 struct btf_type **types;
38 const char *strings;
39 void *nohdr_data;
40 __u32 nr_types;
41 __u32 types_size;
42 __u32 data_size;
43 int fd;
44 int ptr_sz;
45};
46
47static inline __u64 ptr_to_u64(const void *ptr)
48{
49 return (__u64) (unsigned long) ptr;
50}
51
52static int btf_add_type(struct btf *btf, struct btf_type *t)
53{
54 if (btf->types_size - btf->nr_types < 2) {
55 struct btf_type **new_types;
56 __u32 expand_by, new_size;
57
58 if (btf->types_size == BTF_MAX_NR_TYPES)
59 return -E2BIG;
60
61 expand_by = max(btf->types_size >> 2, 16U);
62 new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by);
63
64 new_types = realloc(btf->types, sizeof(*new_types) * new_size);
65 if (!new_types)
66 return -ENOMEM;
67
68 if (btf->nr_types == 0)
69 new_types[0] = &btf_void;
70
71 btf->types = new_types;
72 btf->types_size = new_size;
73 }
74
75 btf->types[++(btf->nr_types)] = t;
76
77 return 0;
78}
79
80static int btf_parse_hdr(struct btf *btf)
81{
82 const struct btf_header *hdr = btf->hdr;
83 __u32 meta_left;
84
85 if (btf->data_size < sizeof(struct btf_header)) {
86 pr_debug("BTF header not found\n");
87 return -EINVAL;
88 }
89
90 if (hdr->magic != BTF_MAGIC) {
91 pr_debug("Invalid BTF magic:%x\n", hdr->magic);
92 return -EINVAL;
93 }
94
95 if (hdr->version != BTF_VERSION) {
96 pr_debug("Unsupported BTF version:%u\n", hdr->version);
97 return -ENOTSUP;
98 }
99
100 if (hdr->flags) {
101 pr_debug("Unsupported BTF flags:%x\n", hdr->flags);
102 return -ENOTSUP;
103 }
104
105 meta_left = btf->data_size - sizeof(*hdr);
106 if (!meta_left) {
107 pr_debug("BTF has no data\n");
108 return -EINVAL;
109 }
110
111 if (meta_left < hdr->type_off) {
112 pr_debug("Invalid BTF type section offset:%u\n", hdr->type_off);
113 return -EINVAL;
114 }
115
116 if (meta_left < hdr->str_off) {
117 pr_debug("Invalid BTF string section offset:%u\n", hdr->str_off);
118 return -EINVAL;
119 }
120
121 if (hdr->type_off >= hdr->str_off) {
122 pr_debug("BTF type section offset >= string section offset. No type?\n");
123 return -EINVAL;
124 }
125
126 if (hdr->type_off & 0x02) {
127 pr_debug("BTF type section is not aligned to 4 bytes\n");
128 return -EINVAL;
129 }
130
131 btf->nohdr_data = btf->hdr + 1;
132
133 return 0;
134}
135
136static int btf_parse_str_sec(struct btf *btf)
137{
138 const struct btf_header *hdr = btf->hdr;
139 const char *start = btf->nohdr_data + hdr->str_off;
140 const char *end = start + btf->hdr->str_len;
141
142 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET ||
143 start[0] || end[-1]) {
144 pr_debug("Invalid BTF string section\n");
145 return -EINVAL;
146 }
147
148 btf->strings = start;
149
150 return 0;
151}
152
153static int btf_type_size(struct btf_type *t)
154{
155 int base_size = sizeof(struct btf_type);
156 __u16 vlen = btf_vlen(t);
157
158 switch (btf_kind(t)) {
159 case BTF_KIND_FWD:
160 case BTF_KIND_CONST:
161 case BTF_KIND_VOLATILE:
162 case BTF_KIND_RESTRICT:
163 case BTF_KIND_PTR:
164 case BTF_KIND_TYPEDEF:
165 case BTF_KIND_FUNC:
166 return base_size;
167 case BTF_KIND_INT:
168 return base_size + sizeof(__u32);
169 case BTF_KIND_ENUM:
170 return base_size + vlen * sizeof(struct btf_enum);
171 case BTF_KIND_ARRAY:
172 return base_size + sizeof(struct btf_array);
173 case BTF_KIND_STRUCT:
174 case BTF_KIND_UNION:
175 return base_size + vlen * sizeof(struct btf_member);
176 case BTF_KIND_FUNC_PROTO:
177 return base_size + vlen * sizeof(struct btf_param);
178 case BTF_KIND_VAR:
179 return base_size + sizeof(struct btf_var);
180 case BTF_KIND_DATASEC:
181 return base_size + vlen * sizeof(struct btf_var_secinfo);
182 default:
183 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
184 return -EINVAL;
185 }
186}
187
188static int btf_parse_type_sec(struct btf *btf)
189{
190 struct btf_header *hdr = btf->hdr;
191 void *nohdr_data = btf->nohdr_data;
192 void *next_type = nohdr_data + hdr->type_off;
193 void *end_type = nohdr_data + hdr->str_off;
194
195 while (next_type < end_type) {
196 struct btf_type *t = next_type;
197 int type_size;
198 int err;
199
200 type_size = btf_type_size(t);
201 if (type_size < 0)
202 return type_size;
203 next_type += type_size;
204 err = btf_add_type(btf, t);
205 if (err)
206 return err;
207 }
208
209 return 0;
210}
211
212__u32 btf__get_nr_types(const struct btf *btf)
213{
214 return btf->nr_types;
215}
216
217const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
218{
219 if (type_id > btf->nr_types)
220 return NULL;
221
222 return btf->types[type_id];
223}
224
225static int determine_ptr_size(const struct btf *btf)
226{
227 const struct btf_type *t;
228 const char *name;
229 int i;
230
231 for (i = 1; i <= btf->nr_types; i++) {
232 t = btf__type_by_id(btf, i);
233 if (!btf_is_int(t))
234 continue;
235
236 name = btf__name_by_offset(btf, t->name_off);
237 if (!name)
238 continue;
239
240 if (strcmp(name, "long int") == 0 ||
241 strcmp(name, "long unsigned int") == 0) {
242 if (t->size != 4 && t->size != 8)
243 continue;
244 return t->size;
245 }
246 }
247
248 return -1;
249}
250
251static size_t btf_ptr_sz(const struct btf *btf)
252{
253 if (!btf->ptr_sz)
254 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
255 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;
256}
257
258/* Return pointer size this BTF instance assumes. The size is heuristically
259 * determined by looking for 'long' or 'unsigned long' integer type and
260 * recording its size in bytes. If BTF type information doesn't have any such
261 * type, this function returns 0. In the latter case, native architecture's
262 * pointer size is assumed, so will be either 4 or 8, depending on
263 * architecture that libbpf was compiled for. It's possible to override
264 * guessed value by using btf__set_pointer_size() API.
265 */
266size_t btf__pointer_size(const struct btf *btf)
267{
268 if (!btf->ptr_sz)
269 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
270
271 if (btf->ptr_sz < 0)
272 /* not enough BTF type info to guess */
273 return 0;
274
275 return btf->ptr_sz;
276}
277
278/* Override or set pointer size in bytes. Only values of 4 and 8 are
279 * supported.
280 */
281int btf__set_pointer_size(struct btf *btf, size_t ptr_sz)
282{
283 if (ptr_sz != 4 && ptr_sz != 8)
284 return -EINVAL;
285 btf->ptr_sz = ptr_sz;
286 return 0;
287}
288
289static bool btf_type_is_void(const struct btf_type *t)
290{
291 return t == &btf_void || btf_is_fwd(t);
292}
293
294static bool btf_type_is_void_or_null(const struct btf_type *t)
295{
296 return !t || btf_type_is_void(t);
297}
298
299#define MAX_RESOLVE_DEPTH 32
300
301__s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
302{
303 const struct btf_array *array;
304 const struct btf_type *t;
305 __u32 nelems = 1;
306 __s64 size = -1;
307 int i;
308
309 t = btf__type_by_id(btf, type_id);
310 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t);
311 i++) {
312 switch (btf_kind(t)) {
313 case BTF_KIND_INT:
314 case BTF_KIND_STRUCT:
315 case BTF_KIND_UNION:
316 case BTF_KIND_ENUM:
317 case BTF_KIND_DATASEC:
318 size = t->size;
319 goto done;
320 case BTF_KIND_PTR:
321 size = btf_ptr_sz(btf);
322 goto done;
323 case BTF_KIND_TYPEDEF:
324 case BTF_KIND_VOLATILE:
325 case BTF_KIND_CONST:
326 case BTF_KIND_RESTRICT:
327 case BTF_KIND_VAR:
328 type_id = t->type;
329 break;
330 case BTF_KIND_ARRAY:
331 array = btf_array(t);
332 if (nelems && array->nelems > UINT32_MAX / nelems)
333 return -E2BIG;
334 nelems *= array->nelems;
335 type_id = array->type;
336 break;
337 default:
338 return -EINVAL;
339 }
340
341 t = btf__type_by_id(btf, type_id);
342 }
343
344done:
345 if (size < 0)
346 return -EINVAL;
347 if (nelems && size > UINT32_MAX / nelems)
348 return -E2BIG;
349
350 return nelems * size;
351}
352
353int btf__align_of(const struct btf *btf, __u32 id)
354{
355 const struct btf_type *t = btf__type_by_id(btf, id);
356 __u16 kind = btf_kind(t);
357
358 switch (kind) {
359 case BTF_KIND_INT:
360 case BTF_KIND_ENUM:
361 return min(btf_ptr_sz(btf), (size_t)t->size);
362 case BTF_KIND_PTR:
363 return btf_ptr_sz(btf);
364 case BTF_KIND_TYPEDEF:
365 case BTF_KIND_VOLATILE:
366 case BTF_KIND_CONST:
367 case BTF_KIND_RESTRICT:
368 return btf__align_of(btf, t->type);
369 case BTF_KIND_ARRAY:
370 return btf__align_of(btf, btf_array(t)->type);
371 case BTF_KIND_STRUCT:
372 case BTF_KIND_UNION: {
373 const struct btf_member *m = btf_members(t);
374 __u16 vlen = btf_vlen(t);
375 int i, max_align = 1, align;
376
377 for (i = 0; i < vlen; i++, m++) {
378 align = btf__align_of(btf, m->type);
379 if (align <= 0)
380 return align;
381 max_align = max(max_align, align);
382 }
383
384 return max_align;
385 }
386 default:
387 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
388 return 0;
389 }
390}
391
392int btf__resolve_type(const struct btf *btf, __u32 type_id)
393{
394 const struct btf_type *t;
395 int depth = 0;
396
397 t = btf__type_by_id(btf, type_id);
398 while (depth < MAX_RESOLVE_DEPTH &&
399 !btf_type_is_void_or_null(t) &&
400 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) {
401 type_id = t->type;
402 t = btf__type_by_id(btf, type_id);
403 depth++;
404 }
405
406 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t))
407 return -EINVAL;
408
409 return type_id;
410}
411
412__s32 btf__find_by_name(const struct btf *btf, const char *type_name)
413{
414 __u32 i;
415
416 if (!strcmp(type_name, "void"))
417 return 0;
418
419 for (i = 1; i <= btf->nr_types; i++) {
420 const struct btf_type *t = btf->types[i];
421 const char *name = btf__name_by_offset(btf, t->name_off);
422
423 if (name && !strcmp(type_name, name))
424 return i;
425 }
426
427 return -ENOENT;
428}
429
430__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
431 __u32 kind)
432{
433 __u32 i;
434
435 if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
436 return 0;
437
438 for (i = 1; i <= btf->nr_types; i++) {
439 const struct btf_type *t = btf->types[i];
440 const char *name;
441
442 if (btf_kind(t) != kind)
443 continue;
444 name = btf__name_by_offset(btf, t->name_off);
445 if (name && !strcmp(type_name, name))
446 return i;
447 }
448
449 return -ENOENT;
450}
451
452void btf__free(struct btf *btf)
453{
454 if (IS_ERR_OR_NULL(btf))
455 return;
456
457 if (btf->fd >= 0)
458 close(btf->fd);
459
460 free(btf->data);
461 free(btf->types);
462 free(btf);
463}
464
465struct btf *btf__new(const void *data, __u32 size)
466{
467 struct btf *btf;
468 int err;
469
470 btf = calloc(1, sizeof(struct btf));
471 if (!btf)
472 return ERR_PTR(-ENOMEM);
473
474 btf->fd = -1;
475
476 btf->data = malloc(size);
477 if (!btf->data) {
478 err = -ENOMEM;
479 goto done;
480 }
481
482 memcpy(btf->data, data, size);
483 btf->data_size = size;
484
485 err = btf_parse_hdr(btf);
486 if (err)
487 goto done;
488
489 err = btf_parse_str_sec(btf);
490 if (err)
491 goto done;
492
493 err = btf_parse_type_sec(btf);
494
495done:
496 if (err) {
497 btf__free(btf);
498 return ERR_PTR(err);
499 }
500
501 return btf;
502}
503
504static bool btf_check_endianness(const GElf_Ehdr *ehdr)
505{
506#if __BYTE_ORDER == __LITTLE_ENDIAN
507 return ehdr->e_ident[EI_DATA] == ELFDATA2LSB;
508#elif __BYTE_ORDER == __BIG_ENDIAN
509 return ehdr->e_ident[EI_DATA] == ELFDATA2MSB;
510#else
511# error "Unrecognized __BYTE_ORDER__"
512#endif
513}
514
515struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
516{
517 Elf_Data *btf_data = NULL, *btf_ext_data = NULL;
518 int err = 0, fd = -1, idx = 0;
519 struct btf *btf = NULL;
520 Elf_Scn *scn = NULL;
521 Elf *elf = NULL;
522 GElf_Ehdr ehdr;
523
524 if (elf_version(EV_CURRENT) == EV_NONE) {
525 pr_warn("failed to init libelf for %s\n", path);
526 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
527 }
528
529 fd = open(path, O_RDONLY);
530 if (fd < 0) {
531 err = -errno;
532 pr_warn("failed to open %s: %s\n", path, strerror(errno));
533 return ERR_PTR(err);
534 }
535
536 err = -LIBBPF_ERRNO__FORMAT;
537
538 elf = elf_begin(fd, ELF_C_READ, NULL);
539 if (!elf) {
540 pr_warn("failed to open %s as ELF file\n", path);
541 goto done;
542 }
543 if (!gelf_getehdr(elf, &ehdr)) {
544 pr_warn("failed to get EHDR from %s\n", path);
545 goto done;
546 }
547 if (!btf_check_endianness(&ehdr)) {
548 pr_warn("non-native ELF endianness is not supported\n");
549 goto done;
550 }
551 if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {
552 pr_warn("failed to get e_shstrndx from %s\n", path);
553 goto done;
554 }
555
556 while ((scn = elf_nextscn(elf, scn)) != NULL) {
557 GElf_Shdr sh;
558 char *name;
559
560 idx++;
561 if (gelf_getshdr(scn, &sh) != &sh) {
562 pr_warn("failed to get section(%d) header from %s\n",
563 idx, path);
564 goto done;
565 }
566 name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);
567 if (!name) {
568 pr_warn("failed to get section(%d) name from %s\n",
569 idx, path);
570 goto done;
571 }
572 if (strcmp(name, BTF_ELF_SEC) == 0) {
573 btf_data = elf_getdata(scn, 0);
574 if (!btf_data) {
575 pr_warn("failed to get section(%d, %s) data from %s\n",
576 idx, name, path);
577 goto done;
578 }
579 continue;
580 } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {
581 btf_ext_data = elf_getdata(scn, 0);
582 if (!btf_ext_data) {
583 pr_warn("failed to get section(%d, %s) data from %s\n",
584 idx, name, path);
585 goto done;
586 }
587 continue;
588 }
589 }
590
591 err = 0;
592
593 if (!btf_data) {
594 err = -ENOENT;
595 goto done;
596 }
597 btf = btf__new(btf_data->d_buf, btf_data->d_size);
598 if (IS_ERR(btf))
599 goto done;
600
601 switch (gelf_getclass(elf)) {
602 case ELFCLASS32:
603 btf__set_pointer_size(btf, 4);
604 break;
605 case ELFCLASS64:
606 btf__set_pointer_size(btf, 8);
607 break;
608 default:
609 pr_warn("failed to get ELF class (bitness) for %s\n", path);
610 break;
611 }
612
613 if (btf_ext && btf_ext_data) {
614 *btf_ext = btf_ext__new(btf_ext_data->d_buf,
615 btf_ext_data->d_size);
616 if (IS_ERR(*btf_ext))
617 goto done;
618 } else if (btf_ext) {
619 *btf_ext = NULL;
620 }
621done:
622 if (elf)
623 elf_end(elf);
624 close(fd);
625
626 if (err)
627 return ERR_PTR(err);
628 /*
629 * btf is always parsed before btf_ext, so no need to clean up
630 * btf_ext, if btf loading failed
631 */
632 if (IS_ERR(btf))
633 return btf;
634 if (btf_ext && IS_ERR(*btf_ext)) {
635 btf__free(btf);
636 err = PTR_ERR(*btf_ext);
637 return ERR_PTR(err);
638 }
639 return btf;
640}
641
642struct btf *btf__parse_raw(const char *path)
643{
644 struct btf *btf = NULL;
645 void *data = NULL;
646 FILE *f = NULL;
647 __u16 magic;
648 int err = 0;
649 long sz;
650
651 f = fopen(path, "rb");
652 if (!f) {
653 err = -errno;
654 goto err_out;
655 }
656
657 /* check BTF magic */
658 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
659 err = -EIO;
660 goto err_out;
661 }
662 if (magic == __bswap_16(BTF_MAGIC)) {
663 /* non-native endian raw BTF */
664 pr_warn("non-native BTF endianness is not supported\n");
665 err = -LIBBPF_ERRNO__ENDIAN;
666 goto err_out;
667 }
668 if (magic != BTF_MAGIC) {
669 /* definitely not a raw BTF */
670 err = -EPROTO;
671 goto err_out;
672 }
673
674 /* get file size */
675 if (fseek(f, 0, SEEK_END)) {
676 err = -errno;
677 goto err_out;
678 }
679 sz = ftell(f);
680 if (sz < 0) {
681 err = -errno;
682 goto err_out;
683 }
684 /* rewind to the start */
685 if (fseek(f, 0, SEEK_SET)) {
686 err = -errno;
687 goto err_out;
688 }
689
690 /* pre-alloc memory and read all of BTF data */
691 data = malloc(sz);
692 if (!data) {
693 err = -ENOMEM;
694 goto err_out;
695 }
696 if (fread(data, 1, sz, f) < sz) {
697 err = -EIO;
698 goto err_out;
699 }
700
701 /* finally parse BTF data */
702 btf = btf__new(data, sz);
703
704err_out:
705 free(data);
706 if (f)
707 fclose(f);
708 return err ? ERR_PTR(err) : btf;
709}
710
711struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
712{
713 struct btf *btf;
714
715 if (btf_ext)
716 *btf_ext = NULL;
717
718 btf = btf__parse_raw(path);
719 if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO)
720 return btf;
721
722 return btf__parse_elf(path, btf_ext);
723}
724
725static int compare_vsi_off(const void *_a, const void *_b)
726{
727 const struct btf_var_secinfo *a = _a;
728 const struct btf_var_secinfo *b = _b;
729
730 return a->offset - b->offset;
731}
732
733static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
734 struct btf_type *t)
735{
736 __u32 size = 0, off = 0, i, vars = btf_vlen(t);
737 const char *name = btf__name_by_offset(btf, t->name_off);
738 const struct btf_type *t_var;
739 struct btf_var_secinfo *vsi;
740 const struct btf_var *var;
741 int ret;
742
743 if (!name) {
744 pr_debug("No name found in string section for DATASEC kind.\n");
745 return -ENOENT;
746 }
747
748 /* .extern datasec size and var offsets were set correctly during
749 * extern collection step, so just skip straight to sorting variables
750 */
751 if (t->size)
752 goto sort_vars;
753
754 ret = bpf_object__section_size(obj, name, &size);
755 if (ret || !size || (t->size && t->size != size)) {
756 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
757 return -ENOENT;
758 }
759
760 t->size = size;
761
762 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
763 t_var = btf__type_by_id(btf, vsi->type);
764 var = btf_var(t_var);
765
766 if (!btf_is_var(t_var)) {
767 pr_debug("Non-VAR type seen in section %s\n", name);
768 return -EINVAL;
769 }
770
771 if (var->linkage == BTF_VAR_STATIC)
772 continue;
773
774 name = btf__name_by_offset(btf, t_var->name_off);
775 if (!name) {
776 pr_debug("No name found in string section for VAR kind\n");
777 return -ENOENT;
778 }
779
780 ret = bpf_object__variable_offset(obj, name, &off);
781 if (ret) {
782 pr_debug("No offset found in symbol table for VAR %s\n",
783 name);
784 return -ENOENT;
785 }
786
787 vsi->offset = off;
788 }
789
790sort_vars:
791 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
792 return 0;
793}
794
795int btf__finalize_data(struct bpf_object *obj, struct btf *btf)
796{
797 int err = 0;
798 __u32 i;
799
800 for (i = 1; i <= btf->nr_types; i++) {
801 struct btf_type *t = btf->types[i];
802
803 /* Loader needs to fix up some of the things compiler
804 * couldn't get its hands on while emitting BTF. This
805 * is section size and global variable offset. We use
806 * the info from the ELF itself for this purpose.
807 */
808 if (btf_is_datasec(t)) {
809 err = btf_fixup_datasec(obj, btf, t);
810 if (err)
811 break;
812 }
813 }
814
815 return err;
816}
817
818int btf__load(struct btf *btf)
819{
820 __u32 log_buf_size = 0;
821 char *log_buf = NULL;
822 int err = 0;
823
824 if (btf->fd >= 0)
825 return -EEXIST;
826
827retry_load:
828 if (log_buf_size) {
829 log_buf = malloc(log_buf_size);
830 if (!log_buf)
831 return -ENOMEM;
832
833 *log_buf = 0;
834 }
835
836 btf->fd = bpf_load_btf(btf->data, btf->data_size,
837 log_buf, log_buf_size, false);
838 if (btf->fd < 0) {
839 if (!log_buf || errno == ENOSPC) {
840 log_buf_size = max((__u32)BPF_LOG_BUF_SIZE,
841 log_buf_size << 1);
842 free(log_buf);
843 goto retry_load;
844 }
845
846 err = -errno;
847 pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
848 if (*log_buf)
849 pr_warn("%s\n", log_buf);
850 goto done;
851 }
852
853done:
854 free(log_buf);
855 return err;
856}
857
858int btf__fd(const struct btf *btf)
859{
860 return btf->fd;
861}
862
863void btf__set_fd(struct btf *btf, int fd)
864{
865 btf->fd = fd;
866}
867
868const void *btf__get_raw_data(const struct btf *btf, __u32 *size)
869{
870 *size = btf->data_size;
871 return btf->data;
872}
873
874const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
875{
876 if (offset < btf->hdr->str_len)
877 return &btf->strings[offset];
878 else
879 return NULL;
880}
881
882int btf__get_from_id(__u32 id, struct btf **btf)
883{
884 struct bpf_btf_info btf_info = { 0 };
885 __u32 len = sizeof(btf_info);
886 __u32 last_size;
887 int btf_fd;
888 void *ptr;
889 int err;
890
891 err = 0;
892 *btf = NULL;
893 btf_fd = bpf_btf_get_fd_by_id(id);
894 if (btf_fd < 0)
895 return 0;
896
897 /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so
898 * let's start with a sane default - 4KiB here - and resize it only if
899 * bpf_obj_get_info_by_fd() needs a bigger buffer.
900 */
901 btf_info.btf_size = 4096;
902 last_size = btf_info.btf_size;
903 ptr = malloc(last_size);
904 if (!ptr) {
905 err = -ENOMEM;
906 goto exit_free;
907 }
908
909 memset(ptr, 0, last_size);
910 btf_info.btf = ptr_to_u64(ptr);
911 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
912
913 if (!err && btf_info.btf_size > last_size) {
914 void *temp_ptr;
915
916 last_size = btf_info.btf_size;
917 temp_ptr = realloc(ptr, last_size);
918 if (!temp_ptr) {
919 err = -ENOMEM;
920 goto exit_free;
921 }
922 ptr = temp_ptr;
923 memset(ptr, 0, last_size);
924 btf_info.btf = ptr_to_u64(ptr);
925 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
926 }
927
928 if (err || btf_info.btf_size > last_size) {
929 err = errno;
930 goto exit_free;
931 }
932
933 *btf = btf__new((__u8 *)(long)btf_info.btf, btf_info.btf_size);
934 if (IS_ERR(*btf)) {
935 err = PTR_ERR(*btf);
936 *btf = NULL;
937 }
938
939exit_free:
940 close(btf_fd);
941 free(ptr);
942
943 return err;
944}
945
946int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
947 __u32 expected_key_size, __u32 expected_value_size,
948 __u32 *key_type_id, __u32 *value_type_id)
949{
950 const struct btf_type *container_type;
951 const struct btf_member *key, *value;
952 const size_t max_name = 256;
953 char container_name[max_name];
954 __s64 key_size, value_size;
955 __s32 container_id;
956
957 if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==
958 max_name) {
959 pr_warn("map:%s length of '____btf_map_%s' is too long\n",
960 map_name, map_name);
961 return -EINVAL;
962 }
963
964 container_id = btf__find_by_name(btf, container_name);
965 if (container_id < 0) {
966 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
967 map_name, container_name);
968 return container_id;
969 }
970
971 container_type = btf__type_by_id(btf, container_id);
972 if (!container_type) {
973 pr_warn("map:%s cannot find BTF type for container_id:%u\n",
974 map_name, container_id);
975 return -EINVAL;
976 }
977
978 if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) {
979 pr_warn("map:%s container_name:%s is an invalid container struct\n",
980 map_name, container_name);
981 return -EINVAL;
982 }
983
984 key = btf_members(container_type);
985 value = key + 1;
986
987 key_size = btf__resolve_size(btf, key->type);
988 if (key_size < 0) {
989 pr_warn("map:%s invalid BTF key_type_size\n", map_name);
990 return key_size;
991 }
992
993 if (expected_key_size != key_size) {
994 pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
995 map_name, (__u32)key_size, expected_key_size);
996 return -EINVAL;
997 }
998
999 value_size = btf__resolve_size(btf, value->type);
1000 if (value_size < 0) {
1001 pr_warn("map:%s invalid BTF value_type_size\n", map_name);
1002 return value_size;
1003 }
1004
1005 if (expected_value_size != value_size) {
1006 pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1007 map_name, (__u32)value_size, expected_value_size);
1008 return -EINVAL;
1009 }
1010
1011 *key_type_id = key->type;
1012 *value_type_id = value->type;
1013
1014 return 0;
1015}
1016
1017struct btf_ext_sec_setup_param {
1018 __u32 off;
1019 __u32 len;
1020 __u32 min_rec_size;
1021 struct btf_ext_info *ext_info;
1022 const char *desc;
1023};
1024
1025static int btf_ext_setup_info(struct btf_ext *btf_ext,
1026 struct btf_ext_sec_setup_param *ext_sec)
1027{
1028 const struct btf_ext_info_sec *sinfo;
1029 struct btf_ext_info *ext_info;
1030 __u32 info_left, record_size;
1031 /* The start of the info sec (including the __u32 record_size). */
1032 void *info;
1033
1034 if (ext_sec->len == 0)
1035 return 0;
1036
1037 if (ext_sec->off & 0x03) {
1038 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n",
1039 ext_sec->desc);
1040 return -EINVAL;
1041 }
1042
1043 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
1044 info_left = ext_sec->len;
1045
1046 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {
1047 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
1048 ext_sec->desc, ext_sec->off, ext_sec->len);
1049 return -EINVAL;
1050 }
1051
1052 /* At least a record size */
1053 if (info_left < sizeof(__u32)) {
1054 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);
1055 return -EINVAL;
1056 }
1057
1058 /* The record size needs to meet the minimum standard */
1059 record_size = *(__u32 *)info;
1060 if (record_size < ext_sec->min_rec_size ||
1061 record_size & 0x03) {
1062 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
1063 ext_sec->desc, record_size);
1064 return -EINVAL;
1065 }
1066
1067 sinfo = info + sizeof(__u32);
1068 info_left -= sizeof(__u32);
1069
1070 /* If no records, return failure now so .BTF.ext won't be used. */
1071 if (!info_left) {
1072 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);
1073 return -EINVAL;
1074 }
1075
1076 while (info_left) {
1077 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
1078 __u64 total_record_size;
1079 __u32 num_records;
1080
1081 if (info_left < sec_hdrlen) {
1082 pr_debug("%s section header is not found in .BTF.ext\n",
1083 ext_sec->desc);
1084 return -EINVAL;
1085 }
1086
1087 num_records = sinfo->num_info;
1088 if (num_records == 0) {
1089 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
1090 ext_sec->desc);
1091 return -EINVAL;
1092 }
1093
1094 total_record_size = sec_hdrlen +
1095 (__u64)num_records * record_size;
1096 if (info_left < total_record_size) {
1097 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
1098 ext_sec->desc);
1099 return -EINVAL;
1100 }
1101
1102 info_left -= total_record_size;
1103 sinfo = (void *)sinfo + total_record_size;
1104 }
1105
1106 ext_info = ext_sec->ext_info;
1107 ext_info->len = ext_sec->len - sizeof(__u32);
1108 ext_info->rec_size = record_size;
1109 ext_info->info = info + sizeof(__u32);
1110
1111 return 0;
1112}
1113
1114static int btf_ext_setup_func_info(struct btf_ext *btf_ext)
1115{
1116 struct btf_ext_sec_setup_param param = {
1117 .off = btf_ext->hdr->func_info_off,
1118 .len = btf_ext->hdr->func_info_len,
1119 .min_rec_size = sizeof(struct bpf_func_info_min),
1120 .ext_info = &btf_ext->func_info,
1121 .desc = "func_info"
1122 };
1123
1124 return btf_ext_setup_info(btf_ext, ¶m);
1125}
1126
1127static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
1128{
1129 struct btf_ext_sec_setup_param param = {
1130 .off = btf_ext->hdr->line_info_off,
1131 .len = btf_ext->hdr->line_info_len,
1132 .min_rec_size = sizeof(struct bpf_line_info_min),
1133 .ext_info = &btf_ext->line_info,
1134 .desc = "line_info",
1135 };
1136
1137 return btf_ext_setup_info(btf_ext, ¶m);
1138}
1139
1140static int btf_ext_setup_field_reloc(struct btf_ext *btf_ext)
1141{
1142 struct btf_ext_sec_setup_param param = {
1143 .off = btf_ext->hdr->field_reloc_off,
1144 .len = btf_ext->hdr->field_reloc_len,
1145 .min_rec_size = sizeof(struct bpf_field_reloc),
1146 .ext_info = &btf_ext->field_reloc_info,
1147 .desc = "field_reloc",
1148 };
1149
1150 return btf_ext_setup_info(btf_ext, ¶m);
1151}
1152
1153static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
1154{
1155 const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
1156
1157 if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
1158 data_size < hdr->hdr_len) {
1159 pr_debug("BTF.ext header not found");
1160 return -EINVAL;
1161 }
1162
1163 if (hdr->magic != BTF_MAGIC) {
1164 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
1165 return -EINVAL;
1166 }
1167
1168 if (hdr->version != BTF_VERSION) {
1169 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
1170 return -ENOTSUP;
1171 }
1172
1173 if (hdr->flags) {
1174 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);
1175 return -ENOTSUP;
1176 }
1177
1178 if (data_size == hdr->hdr_len) {
1179 pr_debug("BTF.ext has no data\n");
1180 return -EINVAL;
1181 }
1182
1183 return 0;
1184}
1185
1186void btf_ext__free(struct btf_ext *btf_ext)
1187{
1188 if (IS_ERR_OR_NULL(btf_ext))
1189 return;
1190 free(btf_ext->data);
1191 free(btf_ext);
1192}
1193
1194struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
1195{
1196 struct btf_ext *btf_ext;
1197 int err;
1198
1199 err = btf_ext_parse_hdr(data, size);
1200 if (err)
1201 return ERR_PTR(err);
1202
1203 btf_ext = calloc(1, sizeof(struct btf_ext));
1204 if (!btf_ext)
1205 return ERR_PTR(-ENOMEM);
1206
1207 btf_ext->data_size = size;
1208 btf_ext->data = malloc(size);
1209 if (!btf_ext->data) {
1210 err = -ENOMEM;
1211 goto done;
1212 }
1213 memcpy(btf_ext->data, data, size);
1214
1215 if (btf_ext->hdr->hdr_len <
1216 offsetofend(struct btf_ext_header, line_info_len))
1217 goto done;
1218 err = btf_ext_setup_func_info(btf_ext);
1219 if (err)
1220 goto done;
1221
1222 err = btf_ext_setup_line_info(btf_ext);
1223 if (err)
1224 goto done;
1225
1226 if (btf_ext->hdr->hdr_len <
1227 offsetofend(struct btf_ext_header, field_reloc_len))
1228 goto done;
1229 err = btf_ext_setup_field_reloc(btf_ext);
1230 if (err)
1231 goto done;
1232
1233done:
1234 if (err) {
1235 btf_ext__free(btf_ext);
1236 return ERR_PTR(err);
1237 }
1238
1239 return btf_ext;
1240}
1241
1242const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size)
1243{
1244 *size = btf_ext->data_size;
1245 return btf_ext->data;
1246}
1247
1248static int btf_ext_reloc_info(const struct btf *btf,
1249 const struct btf_ext_info *ext_info,
1250 const char *sec_name, __u32 insns_cnt,
1251 void **info, __u32 *cnt)
1252{
1253 __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
1254 __u32 i, record_size, existing_len, records_len;
1255 struct btf_ext_info_sec *sinfo;
1256 const char *info_sec_name;
1257 __u64 remain_len;
1258 void *data;
1259
1260 record_size = ext_info->rec_size;
1261 sinfo = ext_info->info;
1262 remain_len = ext_info->len;
1263 while (remain_len > 0) {
1264 records_len = sinfo->num_info * record_size;
1265 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
1266 if (strcmp(info_sec_name, sec_name)) {
1267 remain_len -= sec_hdrlen + records_len;
1268 sinfo = (void *)sinfo + sec_hdrlen + records_len;
1269 continue;
1270 }
1271
1272 existing_len = (*cnt) * record_size;
1273 data = realloc(*info, existing_len + records_len);
1274 if (!data)
1275 return -ENOMEM;
1276
1277 memcpy(data + existing_len, sinfo->data, records_len);
1278 /* adjust insn_off only, the rest data will be passed
1279 * to the kernel.
1280 */
1281 for (i = 0; i < sinfo->num_info; i++) {
1282 __u32 *insn_off;
1283
1284 insn_off = data + existing_len + (i * record_size);
1285 *insn_off = *insn_off / sizeof(struct bpf_insn) +
1286 insns_cnt;
1287 }
1288 *info = data;
1289 *cnt += sinfo->num_info;
1290 return 0;
1291 }
1292
1293 return -ENOENT;
1294}
1295
1296int btf_ext__reloc_func_info(const struct btf *btf,
1297 const struct btf_ext *btf_ext,
1298 const char *sec_name, __u32 insns_cnt,
1299 void **func_info, __u32 *cnt)
1300{
1301 return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
1302 insns_cnt, func_info, cnt);
1303}
1304
1305int btf_ext__reloc_line_info(const struct btf *btf,
1306 const struct btf_ext *btf_ext,
1307 const char *sec_name, __u32 insns_cnt,
1308 void **line_info, __u32 *cnt)
1309{
1310 return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
1311 insns_cnt, line_info, cnt);
1312}
1313
1314__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
1315{
1316 return btf_ext->func_info.rec_size;
1317}
1318
1319__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
1320{
1321 return btf_ext->line_info.rec_size;
1322}
1323
1324struct btf_dedup;
1325
1326static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
1327 const struct btf_dedup_opts *opts);
1328static void btf_dedup_free(struct btf_dedup *d);
1329static int btf_dedup_strings(struct btf_dedup *d);
1330static int btf_dedup_prim_types(struct btf_dedup *d);
1331static int btf_dedup_struct_types(struct btf_dedup *d);
1332static int btf_dedup_ref_types(struct btf_dedup *d);
1333static int btf_dedup_compact_types(struct btf_dedup *d);
1334static int btf_dedup_remap_types(struct btf_dedup *d);
1335
1336/*
1337 * Deduplicate BTF types and strings.
1338 *
1339 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF
1340 * section with all BTF type descriptors and string data. It overwrites that
1341 * memory in-place with deduplicated types and strings without any loss of
1342 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section
1343 * is provided, all the strings referenced from .BTF.ext section are honored
1344 * and updated to point to the right offsets after deduplication.
1345 *
1346 * If function returns with error, type/string data might be garbled and should
1347 * be discarded.
1348 *
1349 * More verbose and detailed description of both problem btf_dedup is solving,
1350 * as well as solution could be found at:
1351 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
1352 *
1353 * Problem description and justification
1354 * =====================================
1355 *
1356 * BTF type information is typically emitted either as a result of conversion
1357 * from DWARF to BTF or directly by compiler. In both cases, each compilation
1358 * unit contains information about a subset of all the types that are used
1359 * in an application. These subsets are frequently overlapping and contain a lot
1360 * of duplicated information when later concatenated together into a single
1361 * binary. This algorithm ensures that each unique type is represented by single
1362 * BTF type descriptor, greatly reducing resulting size of BTF data.
1363 *
1364 * Compilation unit isolation and subsequent duplication of data is not the only
1365 * problem. The same type hierarchy (e.g., struct and all the type that struct
1366 * references) in different compilation units can be represented in BTF to
1367 * various degrees of completeness (or, rather, incompleteness) due to
1368 * struct/union forward declarations.
1369 *
1370 * Let's take a look at an example, that we'll use to better understand the
1371 * problem (and solution). Suppose we have two compilation units, each using
1372 * same `struct S`, but each of them having incomplete type information about
1373 * struct's fields:
1374 *
1375 * // CU #1:
1376 * struct S;
1377 * struct A {
1378 * int a;
1379 * struct A* self;
1380 * struct S* parent;
1381 * };
1382 * struct B;
1383 * struct S {
1384 * struct A* a_ptr;
1385 * struct B* b_ptr;
1386 * };
1387 *
1388 * // CU #2:
1389 * struct S;
1390 * struct A;
1391 * struct B {
1392 * int b;
1393 * struct B* self;
1394 * struct S* parent;
1395 * };
1396 * struct S {
1397 * struct A* a_ptr;
1398 * struct B* b_ptr;
1399 * };
1400 *
1401 * In case of CU #1, BTF data will know only that `struct B` exist (but no
1402 * more), but will know the complete type information about `struct A`. While
1403 * for CU #2, it will know full type information about `struct B`, but will
1404 * only know about forward declaration of `struct A` (in BTF terms, it will
1405 * have `BTF_KIND_FWD` type descriptor with name `B`).
1406 *
1407 * This compilation unit isolation means that it's possible that there is no
1408 * single CU with complete type information describing structs `S`, `A`, and
1409 * `B`. Also, we might get tons of duplicated and redundant type information.
1410 *
1411 * Additional complication we need to keep in mind comes from the fact that
1412 * types, in general, can form graphs containing cycles, not just DAGs.
1413 *
1414 * While algorithm does deduplication, it also merges and resolves type
1415 * information (unless disabled throught `struct btf_opts`), whenever possible.
1416 * E.g., in the example above with two compilation units having partial type
1417 * information for structs `A` and `B`, the output of algorithm will emit
1418 * a single copy of each BTF type that describes structs `A`, `B`, and `S`
1419 * (as well as type information for `int` and pointers), as if they were defined
1420 * in a single compilation unit as:
1421 *
1422 * struct A {
1423 * int a;
1424 * struct A* self;
1425 * struct S* parent;
1426 * };
1427 * struct B {
1428 * int b;
1429 * struct B* self;
1430 * struct S* parent;
1431 * };
1432 * struct S {
1433 * struct A* a_ptr;
1434 * struct B* b_ptr;
1435 * };
1436 *
1437 * Algorithm summary
1438 * =================
1439 *
1440 * Algorithm completes its work in 6 separate passes:
1441 *
1442 * 1. Strings deduplication.
1443 * 2. Primitive types deduplication (int, enum, fwd).
1444 * 3. Struct/union types deduplication.
1445 * 4. Reference types deduplication (pointers, typedefs, arrays, funcs, func
1446 * protos, and const/volatile/restrict modifiers).
1447 * 5. Types compaction.
1448 * 6. Types remapping.
1449 *
1450 * Algorithm determines canonical type descriptor, which is a single
1451 * representative type for each truly unique type. This canonical type is the
1452 * one that will go into final deduplicated BTF type information. For
1453 * struct/unions, it is also the type that algorithm will merge additional type
1454 * information into (while resolving FWDs), as it discovers it from data in
1455 * other CUs. Each input BTF type eventually gets either mapped to itself, if
1456 * that type is canonical, or to some other type, if that type is equivalent
1457 * and was chosen as canonical representative. This mapping is stored in
1458 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
1459 * FWD type got resolved to.
1460 *
1461 * To facilitate fast discovery of canonical types, we also maintain canonical
1462 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
1463 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types
1464 * that match that signature. With sufficiently good choice of type signature
1465 * hashing function, we can limit number of canonical types for each unique type
1466 * signature to a very small number, allowing to find canonical type for any
1467 * duplicated type very quickly.
1468 *
1469 * Struct/union deduplication is the most critical part and algorithm for
1470 * deduplicating structs/unions is described in greater details in comments for
1471 * `btf_dedup_is_equiv` function.
1472 */
1473int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
1474 const struct btf_dedup_opts *opts)
1475{
1476 struct btf_dedup *d = btf_dedup_new(btf, btf_ext, opts);
1477 int err;
1478
1479 if (IS_ERR(d)) {
1480 pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d));
1481 return -EINVAL;
1482 }
1483
1484 err = btf_dedup_strings(d);
1485 if (err < 0) {
1486 pr_debug("btf_dedup_strings failed:%d\n", err);
1487 goto done;
1488 }
1489 err = btf_dedup_prim_types(d);
1490 if (err < 0) {
1491 pr_debug("btf_dedup_prim_types failed:%d\n", err);
1492 goto done;
1493 }
1494 err = btf_dedup_struct_types(d);
1495 if (err < 0) {
1496 pr_debug("btf_dedup_struct_types failed:%d\n", err);
1497 goto done;
1498 }
1499 err = btf_dedup_ref_types(d);
1500 if (err < 0) {
1501 pr_debug("btf_dedup_ref_types failed:%d\n", err);
1502 goto done;
1503 }
1504 err = btf_dedup_compact_types(d);
1505 if (err < 0) {
1506 pr_debug("btf_dedup_compact_types failed:%d\n", err);
1507 goto done;
1508 }
1509 err = btf_dedup_remap_types(d);
1510 if (err < 0) {
1511 pr_debug("btf_dedup_remap_types failed:%d\n", err);
1512 goto done;
1513 }
1514
1515done:
1516 btf_dedup_free(d);
1517 return err;
1518}
1519
1520#define BTF_UNPROCESSED_ID ((__u32)-1)
1521#define BTF_IN_PROGRESS_ID ((__u32)-2)
1522
1523struct btf_dedup {
1524 /* .BTF section to be deduped in-place */
1525 struct btf *btf;
1526 /*
1527 * Optional .BTF.ext section. When provided, any strings referenced
1528 * from it will be taken into account when deduping strings
1529 */
1530 struct btf_ext *btf_ext;
1531 /*
1532 * This is a map from any type's signature hash to a list of possible
1533 * canonical representative type candidates. Hash collisions are
1534 * ignored, so even types of various kinds can share same list of
1535 * candidates, which is fine because we rely on subsequent
1536 * btf_xxx_equal() checks to authoritatively verify type equality.
1537 */
1538 struct hashmap *dedup_table;
1539 /* Canonical types map */
1540 __u32 *map;
1541 /* Hypothetical mapping, used during type graph equivalence checks */
1542 __u32 *hypot_map;
1543 __u32 *hypot_list;
1544 size_t hypot_cnt;
1545 size_t hypot_cap;
1546 /* Various option modifying behavior of algorithm */
1547 struct btf_dedup_opts opts;
1548};
1549
1550struct btf_str_ptr {
1551 const char *str;
1552 __u32 new_off;
1553 bool used;
1554};
1555
1556struct btf_str_ptrs {
1557 struct btf_str_ptr *ptrs;
1558 const char *data;
1559 __u32 cnt;
1560 __u32 cap;
1561};
1562
1563static long hash_combine(long h, long value)
1564{
1565 return h * 31 + value;
1566}
1567
1568#define for_each_dedup_cand(d, node, hash) \
1569 hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
1570
1571static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id)
1572{
1573 return hashmap__append(d->dedup_table,
1574 (void *)hash, (void *)(long)type_id);
1575}
1576
1577static int btf_dedup_hypot_map_add(struct btf_dedup *d,
1578 __u32 from_id, __u32 to_id)
1579{
1580 if (d->hypot_cnt == d->hypot_cap) {
1581 __u32 *new_list;
1582
1583 d->hypot_cap += max((size_t)16, d->hypot_cap / 2);
1584 new_list = realloc(d->hypot_list, sizeof(__u32) * d->hypot_cap);
1585 if (!new_list)
1586 return -ENOMEM;
1587 d->hypot_list = new_list;
1588 }
1589 d->hypot_list[d->hypot_cnt++] = from_id;
1590 d->hypot_map[from_id] = to_id;
1591 return 0;
1592}
1593
1594static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
1595{
1596 int i;
1597
1598 for (i = 0; i < d->hypot_cnt; i++)
1599 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
1600 d->hypot_cnt = 0;
1601}
1602
1603static void btf_dedup_free(struct btf_dedup *d)
1604{
1605 hashmap__free(d->dedup_table);
1606 d->dedup_table = NULL;
1607
1608 free(d->map);
1609 d->map = NULL;
1610
1611 free(d->hypot_map);
1612 d->hypot_map = NULL;
1613
1614 free(d->hypot_list);
1615 d->hypot_list = NULL;
1616
1617 free(d);
1618}
1619
1620static size_t btf_dedup_identity_hash_fn(const void *key, void *ctx)
1621{
1622 return (size_t)key;
1623}
1624
1625static size_t btf_dedup_collision_hash_fn(const void *key, void *ctx)
1626{
1627 return 0;
1628}
1629
1630static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx)
1631{
1632 return k1 == k2;
1633}
1634
1635static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
1636 const struct btf_dedup_opts *opts)
1637{
1638 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
1639 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
1640 int i, err = 0;
1641
1642 if (!d)
1643 return ERR_PTR(-ENOMEM);
1644
1645 d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds;
1646 /* dedup_table_size is now used only to force collisions in tests */
1647 if (opts && opts->dedup_table_size == 1)
1648 hash_fn = btf_dedup_collision_hash_fn;
1649
1650 d->btf = btf;
1651 d->btf_ext = btf_ext;
1652
1653 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);
1654 if (IS_ERR(d->dedup_table)) {
1655 err = PTR_ERR(d->dedup_table);
1656 d->dedup_table = NULL;
1657 goto done;
1658 }
1659
1660 d->map = malloc(sizeof(__u32) * (1 + btf->nr_types));
1661 if (!d->map) {
1662 err = -ENOMEM;
1663 goto done;
1664 }
1665 /* special BTF "void" type is made canonical immediately */
1666 d->map[0] = 0;
1667 for (i = 1; i <= btf->nr_types; i++) {
1668 struct btf_type *t = d->btf->types[i];
1669
1670 /* VAR and DATASEC are never deduped and are self-canonical */
1671 if (btf_is_var(t) || btf_is_datasec(t))
1672 d->map[i] = i;
1673 else
1674 d->map[i] = BTF_UNPROCESSED_ID;
1675 }
1676
1677 d->hypot_map = malloc(sizeof(__u32) * (1 + btf->nr_types));
1678 if (!d->hypot_map) {
1679 err = -ENOMEM;
1680 goto done;
1681 }
1682 for (i = 0; i <= btf->nr_types; i++)
1683 d->hypot_map[i] = BTF_UNPROCESSED_ID;
1684
1685done:
1686 if (err) {
1687 btf_dedup_free(d);
1688 return ERR_PTR(err);
1689 }
1690
1691 return d;
1692}
1693
1694typedef int (*str_off_fn_t)(__u32 *str_off_ptr, void *ctx);
1695
1696/*
1697 * Iterate over all possible places in .BTF and .BTF.ext that can reference
1698 * string and pass pointer to it to a provided callback `fn`.
1699 */
1700static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx)
1701{
1702 void *line_data_cur, *line_data_end;
1703 int i, j, r, rec_size;
1704 struct btf_type *t;
1705
1706 for (i = 1; i <= d->btf->nr_types; i++) {
1707 t = d->btf->types[i];
1708 r = fn(&t->name_off, ctx);
1709 if (r)
1710 return r;
1711
1712 switch (btf_kind(t)) {
1713 case BTF_KIND_STRUCT:
1714 case BTF_KIND_UNION: {
1715 struct btf_member *m = btf_members(t);
1716 __u16 vlen = btf_vlen(t);
1717
1718 for (j = 0; j < vlen; j++) {
1719 r = fn(&m->name_off, ctx);
1720 if (r)
1721 return r;
1722 m++;
1723 }
1724 break;
1725 }
1726 case BTF_KIND_ENUM: {
1727 struct btf_enum *m = btf_enum(t);
1728 __u16 vlen = btf_vlen(t);
1729
1730 for (j = 0; j < vlen; j++) {
1731 r = fn(&m->name_off, ctx);
1732 if (r)
1733 return r;
1734 m++;
1735 }
1736 break;
1737 }
1738 case BTF_KIND_FUNC_PROTO: {
1739 struct btf_param *m = btf_params(t);
1740 __u16 vlen = btf_vlen(t);
1741
1742 for (j = 0; j < vlen; j++) {
1743 r = fn(&m->name_off, ctx);
1744 if (r)
1745 return r;
1746 m++;
1747 }
1748 break;
1749 }
1750 default:
1751 break;
1752 }
1753 }
1754
1755 if (!d->btf_ext)
1756 return 0;
1757
1758 line_data_cur = d->btf_ext->line_info.info;
1759 line_data_end = d->btf_ext->line_info.info + d->btf_ext->line_info.len;
1760 rec_size = d->btf_ext->line_info.rec_size;
1761
1762 while (line_data_cur < line_data_end) {
1763 struct btf_ext_info_sec *sec = line_data_cur;
1764 struct bpf_line_info_min *line_info;
1765 __u32 num_info = sec->num_info;
1766
1767 r = fn(&sec->sec_name_off, ctx);
1768 if (r)
1769 return r;
1770
1771 line_data_cur += sizeof(struct btf_ext_info_sec);
1772 for (i = 0; i < num_info; i++) {
1773 line_info = line_data_cur;
1774 r = fn(&line_info->file_name_off, ctx);
1775 if (r)
1776 return r;
1777 r = fn(&line_info->line_off, ctx);
1778 if (r)
1779 return r;
1780 line_data_cur += rec_size;
1781 }
1782 }
1783
1784 return 0;
1785}
1786
1787static int str_sort_by_content(const void *a1, const void *a2)
1788{
1789 const struct btf_str_ptr *p1 = a1;
1790 const struct btf_str_ptr *p2 = a2;
1791
1792 return strcmp(p1->str, p2->str);
1793}
1794
1795static int str_sort_by_offset(const void *a1, const void *a2)
1796{
1797 const struct btf_str_ptr *p1 = a1;
1798 const struct btf_str_ptr *p2 = a2;
1799
1800 if (p1->str != p2->str)
1801 return p1->str < p2->str ? -1 : 1;
1802 return 0;
1803}
1804
1805static int btf_dedup_str_ptr_cmp(const void *str_ptr, const void *pelem)
1806{
1807 const struct btf_str_ptr *p = pelem;
1808
1809 if (str_ptr != p->str)
1810 return (const char *)str_ptr < p->str ? -1 : 1;
1811 return 0;
1812}
1813
1814static int btf_str_mark_as_used(__u32 *str_off_ptr, void *ctx)
1815{
1816 struct btf_str_ptrs *strs;
1817 struct btf_str_ptr *s;
1818
1819 if (*str_off_ptr == 0)
1820 return 0;
1821
1822 strs = ctx;
1823 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
1824 sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
1825 if (!s)
1826 return -EINVAL;
1827 s->used = true;
1828 return 0;
1829}
1830
1831static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx)
1832{
1833 struct btf_str_ptrs *strs;
1834 struct btf_str_ptr *s;
1835
1836 if (*str_off_ptr == 0)
1837 return 0;
1838
1839 strs = ctx;
1840 s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
1841 sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
1842 if (!s)
1843 return -EINVAL;
1844 *str_off_ptr = s->new_off;
1845 return 0;
1846}
1847
1848/*
1849 * Dedup string and filter out those that are not referenced from either .BTF
1850 * or .BTF.ext (if provided) sections.
1851 *
1852 * This is done by building index of all strings in BTF's string section,
1853 * then iterating over all entities that can reference strings (e.g., type
1854 * names, struct field names, .BTF.ext line info, etc) and marking corresponding
1855 * strings as used. After that all used strings are deduped and compacted into
1856 * sequential blob of memory and new offsets are calculated. Then all the string
1857 * references are iterated again and rewritten using new offsets.
1858 */
1859static int btf_dedup_strings(struct btf_dedup *d)
1860{
1861 const struct btf_header *hdr = d->btf->hdr;
1862 char *start = (char *)d->btf->nohdr_data + hdr->str_off;
1863 char *end = start + d->btf->hdr->str_len;
1864 char *p = start, *tmp_strs = NULL;
1865 struct btf_str_ptrs strs = {
1866 .cnt = 0,
1867 .cap = 0,
1868 .ptrs = NULL,
1869 .data = start,
1870 };
1871 int i, j, err = 0, grp_idx;
1872 bool grp_used;
1873
1874 /* build index of all strings */
1875 while (p < end) {
1876 if (strs.cnt + 1 > strs.cap) {
1877 struct btf_str_ptr *new_ptrs;
1878
1879 strs.cap += max(strs.cnt / 2, 16U);
1880 new_ptrs = realloc(strs.ptrs,
1881 sizeof(strs.ptrs[0]) * strs.cap);
1882 if (!new_ptrs) {
1883 err = -ENOMEM;
1884 goto done;
1885 }
1886 strs.ptrs = new_ptrs;
1887 }
1888
1889 strs.ptrs[strs.cnt].str = p;
1890 strs.ptrs[strs.cnt].used = false;
1891
1892 p += strlen(p) + 1;
1893 strs.cnt++;
1894 }
1895
1896 /* temporary storage for deduplicated strings */
1897 tmp_strs = malloc(d->btf->hdr->str_len);
1898 if (!tmp_strs) {
1899 err = -ENOMEM;
1900 goto done;
1901 }
1902
1903 /* mark all used strings */
1904 strs.ptrs[0].used = true;
1905 err = btf_for_each_str_off(d, btf_str_mark_as_used, &strs);
1906 if (err)
1907 goto done;
1908
1909 /* sort strings by context, so that we can identify duplicates */
1910 qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_content);
1911
1912 /*
1913 * iterate groups of equal strings and if any instance in a group was
1914 * referenced, emit single instance and remember new offset
1915 */
1916 p = tmp_strs;
1917 grp_idx = 0;
1918 grp_used = strs.ptrs[0].used;
1919 /* iterate past end to avoid code duplication after loop */
1920 for (i = 1; i <= strs.cnt; i++) {
1921 /*
1922 * when i == strs.cnt, we want to skip string comparison and go
1923 * straight to handling last group of strings (otherwise we'd
1924 * need to handle last group after the loop w/ duplicated code)
1925 */
1926 if (i < strs.cnt &&
1927 !strcmp(strs.ptrs[i].str, strs.ptrs[grp_idx].str)) {
1928 grp_used = grp_used || strs.ptrs[i].used;
1929 continue;
1930 }
1931
1932 /*
1933 * this check would have been required after the loop to handle
1934 * last group of strings, but due to <= condition in a loop
1935 * we avoid that duplication
1936 */
1937 if (grp_used) {
1938 int new_off = p - tmp_strs;
1939 __u32 len = strlen(strs.ptrs[grp_idx].str);
1940
1941 memmove(p, strs.ptrs[grp_idx].str, len + 1);
1942 for (j = grp_idx; j < i; j++)
1943 strs.ptrs[j].new_off = new_off;
1944 p += len + 1;
1945 }
1946
1947 if (i < strs.cnt) {
1948 grp_idx = i;
1949 grp_used = strs.ptrs[i].used;
1950 }
1951 }
1952
1953 /* replace original strings with deduped ones */
1954 d->btf->hdr->str_len = p - tmp_strs;
1955 memmove(start, tmp_strs, d->btf->hdr->str_len);
1956 end = start + d->btf->hdr->str_len;
1957
1958 /* restore original order for further binary search lookups */
1959 qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_offset);
1960
1961 /* remap string offsets */
1962 err = btf_for_each_str_off(d, btf_str_remap_offset, &strs);
1963 if (err)
1964 goto done;
1965
1966 d->btf->hdr->str_len = end - start;
1967
1968done:
1969 free(tmp_strs);
1970 free(strs.ptrs);
1971 return err;
1972}
1973
1974static long btf_hash_common(struct btf_type *t)
1975{
1976 long h;
1977
1978 h = hash_combine(0, t->name_off);
1979 h = hash_combine(h, t->info);
1980 h = hash_combine(h, t->size);
1981 return h;
1982}
1983
1984static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2)
1985{
1986 return t1->name_off == t2->name_off &&
1987 t1->info == t2->info &&
1988 t1->size == t2->size;
1989}
1990
1991/* Calculate type signature hash of INT. */
1992static long btf_hash_int(struct btf_type *t)
1993{
1994 __u32 info = *(__u32 *)(t + 1);
1995 long h;
1996
1997 h = btf_hash_common(t);
1998 h = hash_combine(h, info);
1999 return h;
2000}
2001
2002/* Check structural equality of two INTs. */
2003static bool btf_equal_int(struct btf_type *t1, struct btf_type *t2)
2004{
2005 __u32 info1, info2;
2006
2007 if (!btf_equal_common(t1, t2))
2008 return false;
2009 info1 = *(__u32 *)(t1 + 1);
2010 info2 = *(__u32 *)(t2 + 1);
2011 return info1 == info2;
2012}
2013
2014/* Calculate type signature hash of ENUM. */
2015static long btf_hash_enum(struct btf_type *t)
2016{
2017 long h;
2018
2019 /* don't hash vlen and enum members to support enum fwd resolving */
2020 h = hash_combine(0, t->name_off);
2021 h = hash_combine(h, t->info & ~0xffff);
2022 h = hash_combine(h, t->size);
2023 return h;
2024}
2025
2026/* Check structural equality of two ENUMs. */
2027static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
2028{
2029 const struct btf_enum *m1, *m2;
2030 __u16 vlen;
2031 int i;
2032
2033 if (!btf_equal_common(t1, t2))
2034 return false;
2035
2036 vlen = btf_vlen(t1);
2037 m1 = btf_enum(t1);
2038 m2 = btf_enum(t2);
2039 for (i = 0; i < vlen; i++) {
2040 if (m1->name_off != m2->name_off || m1->val != m2->val)
2041 return false;
2042 m1++;
2043 m2++;
2044 }
2045 return true;
2046}
2047
2048static inline bool btf_is_enum_fwd(struct btf_type *t)
2049{
2050 return btf_is_enum(t) && btf_vlen(t) == 0;
2051}
2052
2053static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
2054{
2055 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
2056 return btf_equal_enum(t1, t2);
2057 /* ignore vlen when comparing */
2058 return t1->name_off == t2->name_off &&
2059 (t1->info & ~0xffff) == (t2->info & ~0xffff) &&
2060 t1->size == t2->size;
2061}
2062
2063/*
2064 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
2065 * as referenced type IDs equivalence is established separately during type
2066 * graph equivalence check algorithm.
2067 */
2068static long btf_hash_struct(struct btf_type *t)
2069{
2070 const struct btf_member *member = btf_members(t);
2071 __u32 vlen = btf_vlen(t);
2072 long h = btf_hash_common(t);
2073 int i;
2074
2075 for (i = 0; i < vlen; i++) {
2076 h = hash_combine(h, member->name_off);
2077 h = hash_combine(h, member->offset);
2078 /* no hashing of referenced type ID, it can be unresolved yet */
2079 member++;
2080 }
2081 return h;
2082}
2083
2084/*
2085 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
2086 * IDs. This check is performed during type graph equivalence check and
2087 * referenced types equivalence is checked separately.
2088 */
2089static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
2090{
2091 const struct btf_member *m1, *m2;
2092 __u16 vlen;
2093 int i;
2094
2095 if (!btf_equal_common(t1, t2))
2096 return false;
2097
2098 vlen = btf_vlen(t1);
2099 m1 = btf_members(t1);
2100 m2 = btf_members(t2);
2101 for (i = 0; i < vlen; i++) {
2102 if (m1->name_off != m2->name_off || m1->offset != m2->offset)
2103 return false;
2104 m1++;
2105 m2++;
2106 }
2107 return true;
2108}
2109
2110/*
2111 * Calculate type signature hash of ARRAY, including referenced type IDs,
2112 * under assumption that they were already resolved to canonical type IDs and
2113 * are not going to change.
2114 */
2115static long btf_hash_array(struct btf_type *t)
2116{
2117 const struct btf_array *info = btf_array(t);
2118 long h = btf_hash_common(t);
2119
2120 h = hash_combine(h, info->type);
2121 h = hash_combine(h, info->index_type);
2122 h = hash_combine(h, info->nelems);
2123 return h;
2124}
2125
2126/*
2127 * Check exact equality of two ARRAYs, taking into account referenced
2128 * type IDs, under assumption that they were already resolved to canonical
2129 * type IDs and are not going to change.
2130 * This function is called during reference types deduplication to compare
2131 * ARRAY to potential canonical representative.
2132 */
2133static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2)
2134{
2135 const struct btf_array *info1, *info2;
2136
2137 if (!btf_equal_common(t1, t2))
2138 return false;
2139
2140 info1 = btf_array(t1);
2141 info2 = btf_array(t2);
2142 return info1->type == info2->type &&
2143 info1->index_type == info2->index_type &&
2144 info1->nelems == info2->nelems;
2145}
2146
2147/*
2148 * Check structural compatibility of two ARRAYs, ignoring referenced type
2149 * IDs. This check is performed during type graph equivalence check and
2150 * referenced types equivalence is checked separately.
2151 */
2152static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2)
2153{
2154 if (!btf_equal_common(t1, t2))
2155 return false;
2156
2157 return btf_array(t1)->nelems == btf_array(t2)->nelems;
2158}
2159
2160/*
2161 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs,
2162 * under assumption that they were already resolved to canonical type IDs and
2163 * are not going to change.
2164 */
2165static long btf_hash_fnproto(struct btf_type *t)
2166{
2167 const struct btf_param *member = btf_params(t);
2168 __u16 vlen = btf_vlen(t);
2169 long h = btf_hash_common(t);
2170 int i;
2171
2172 for (i = 0; i < vlen; i++) {
2173 h = hash_combine(h, member->name_off);
2174 h = hash_combine(h, member->type);
2175 member++;
2176 }
2177 return h;
2178}
2179
2180/*
2181 * Check exact equality of two FUNC_PROTOs, taking into account referenced
2182 * type IDs, under assumption that they were already resolved to canonical
2183 * type IDs and are not going to change.
2184 * This function is called during reference types deduplication to compare
2185 * FUNC_PROTO to potential canonical representative.
2186 */
2187static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2)
2188{
2189 const struct btf_param *m1, *m2;
2190 __u16 vlen;
2191 int i;
2192
2193 if (!btf_equal_common(t1, t2))
2194 return false;
2195
2196 vlen = btf_vlen(t1);
2197 m1 = btf_params(t1);
2198 m2 = btf_params(t2);
2199 for (i = 0; i < vlen; i++) {
2200 if (m1->name_off != m2->name_off || m1->type != m2->type)
2201 return false;
2202 m1++;
2203 m2++;
2204 }
2205 return true;
2206}
2207
2208/*
2209 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
2210 * IDs. This check is performed during type graph equivalence check and
2211 * referenced types equivalence is checked separately.
2212 */
2213static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
2214{
2215 const struct btf_param *m1, *m2;
2216 __u16 vlen;
2217 int i;
2218
2219 /* skip return type ID */
2220 if (t1->name_off != t2->name_off || t1->info != t2->info)
2221 return false;
2222
2223 vlen = btf_vlen(t1);
2224 m1 = btf_params(t1);
2225 m2 = btf_params(t2);
2226 for (i = 0; i < vlen; i++) {
2227 if (m1->name_off != m2->name_off)
2228 return false;
2229 m1++;
2230 m2++;
2231 }
2232 return true;
2233}
2234
2235/*
2236 * Deduplicate primitive types, that can't reference other types, by calculating
2237 * their type signature hash and comparing them with any possible canonical
2238 * candidate. If no canonical candidate matches, type itself is marked as
2239 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
2240 */
2241static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
2242{
2243 struct btf_type *t = d->btf->types[type_id];
2244 struct hashmap_entry *hash_entry;
2245 struct btf_type *cand;
2246 /* if we don't find equivalent type, then we are canonical */
2247 __u32 new_id = type_id;
2248 __u32 cand_id;
2249 long h;
2250
2251 switch (btf_kind(t)) {
2252 case BTF_KIND_CONST:
2253 case BTF_KIND_VOLATILE:
2254 case BTF_KIND_RESTRICT:
2255 case BTF_KIND_PTR:
2256 case BTF_KIND_TYPEDEF:
2257 case BTF_KIND_ARRAY:
2258 case BTF_KIND_STRUCT:
2259 case BTF_KIND_UNION:
2260 case BTF_KIND_FUNC:
2261 case BTF_KIND_FUNC_PROTO:
2262 case BTF_KIND_VAR:
2263 case BTF_KIND_DATASEC:
2264 return 0;
2265
2266 case BTF_KIND_INT:
2267 h = btf_hash_int(t);
2268 for_each_dedup_cand(d, hash_entry, h) {
2269 cand_id = (__u32)(long)hash_entry->value;
2270 cand = d->btf->types[cand_id];
2271 if (btf_equal_int(t, cand)) {
2272 new_id = cand_id;
2273 break;
2274 }
2275 }
2276 break;
2277
2278 case BTF_KIND_ENUM:
2279 h = btf_hash_enum(t);
2280 for_each_dedup_cand(d, hash_entry, h) {
2281 cand_id = (__u32)(long)hash_entry->value;
2282 cand = d->btf->types[cand_id];
2283 if (btf_equal_enum(t, cand)) {
2284 new_id = cand_id;
2285 break;
2286 }
2287 if (d->opts.dont_resolve_fwds)
2288 continue;
2289 if (btf_compat_enum(t, cand)) {
2290 if (btf_is_enum_fwd(t)) {
2291 /* resolve fwd to full enum */
2292 new_id = cand_id;
2293 break;
2294 }
2295 /* resolve canonical enum fwd to full enum */
2296 d->map[cand_id] = type_id;
2297 }
2298 }
2299 break;
2300
2301 case BTF_KIND_FWD:
2302 h = btf_hash_common(t);
2303 for_each_dedup_cand(d, hash_entry, h) {
2304 cand_id = (__u32)(long)hash_entry->value;
2305 cand = d->btf->types[cand_id];
2306 if (btf_equal_common(t, cand)) {
2307 new_id = cand_id;
2308 break;
2309 }
2310 }
2311 break;
2312
2313 default:
2314 return -EINVAL;
2315 }
2316
2317 d->map[type_id] = new_id;
2318 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
2319 return -ENOMEM;
2320
2321 return 0;
2322}
2323
2324static int btf_dedup_prim_types(struct btf_dedup *d)
2325{
2326 int i, err;
2327
2328 for (i = 1; i <= d->btf->nr_types; i++) {
2329 err = btf_dedup_prim_type(d, i);
2330 if (err)
2331 return err;
2332 }
2333 return 0;
2334}
2335
2336/*
2337 * Check whether type is already mapped into canonical one (could be to itself).
2338 */
2339static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id)
2340{
2341 return d->map[type_id] <= BTF_MAX_NR_TYPES;
2342}
2343
2344/*
2345 * Resolve type ID into its canonical type ID, if any; otherwise return original
2346 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow
2347 * STRUCT/UNION link and resolve it into canonical type ID as well.
2348 */
2349static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id)
2350{
2351 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
2352 type_id = d->map[type_id];
2353 return type_id;
2354}
2355
2356/*
2357 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original
2358 * type ID.
2359 */
2360static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id)
2361{
2362 __u32 orig_type_id = type_id;
2363
2364 if (!btf_is_fwd(d->btf->types[type_id]))
2365 return type_id;
2366
2367 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
2368 type_id = d->map[type_id];
2369
2370 if (!btf_is_fwd(d->btf->types[type_id]))
2371 return type_id;
2372
2373 return orig_type_id;
2374}
2375
2376
2377static inline __u16 btf_fwd_kind(struct btf_type *t)
2378{
2379 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
2380}
2381
2382/*
2383 * Check equivalence of BTF type graph formed by candidate struct/union (we'll
2384 * call it "candidate graph" in this description for brevity) to a type graph
2385 * formed by (potential) canonical struct/union ("canonical graph" for brevity
2386 * here, though keep in mind that not all types in canonical graph are
2387 * necessarily canonical representatives themselves, some of them might be
2388 * duplicates or its uniqueness might not have been established yet).
2389 * Returns:
2390 * - >0, if type graphs are equivalent;
2391 * - 0, if not equivalent;
2392 * - <0, on error.
2393 *
2394 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
2395 * equivalence of BTF types at each step. If at any point BTF types in candidate
2396 * and canonical graphs are not compatible structurally, whole graphs are
2397 * incompatible. If types are structurally equivalent (i.e., all information
2398 * except referenced type IDs is exactly the same), a mapping from `canon_id` to
2399 * a `cand_id` is recored in hypothetical mapping (`btf_dedup->hypot_map`).
2400 * If a type references other types, then those referenced types are checked
2401 * for equivalence recursively.
2402 *
2403 * During DFS traversal, if we find that for current `canon_id` type we
2404 * already have some mapping in hypothetical map, we check for two possible
2405 * situations:
2406 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
2407 * happen when type graphs have cycles. In this case we assume those two
2408 * types are equivalent.
2409 * - `canon_id` is mapped to different type. This is contradiction in our
2410 * hypothetical mapping, because same graph in canonical graph corresponds
2411 * to two different types in candidate graph, which for equivalent type
2412 * graphs shouldn't happen. This condition terminates equivalence check
2413 * with negative result.
2414 *
2415 * If type graphs traversal exhausts types to check and find no contradiction,
2416 * then type graphs are equivalent.
2417 *
2418 * When checking types for equivalence, there is one special case: FWD types.
2419 * If FWD type resolution is allowed and one of the types (either from canonical
2420 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind
2421 * flag) and their names match, hypothetical mapping is updated to point from
2422 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully,
2423 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
2424 *
2425 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution,
2426 * if there are two exactly named (or anonymous) structs/unions that are
2427 * compatible structurally, one of which has FWD field, while other is concrete
2428 * STRUCT/UNION, but according to C sources they are different structs/unions
2429 * that are referencing different types with the same name. This is extremely
2430 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if
2431 * this logic is causing problems.
2432 *
2433 * Doing FWD resolution means that both candidate and/or canonical graphs can
2434 * consists of portions of the graph that come from multiple compilation units.
2435 * This is due to the fact that types within single compilation unit are always
2436 * deduplicated and FWDs are already resolved, if referenced struct/union
2437 * definiton is available. So, if we had unresolved FWD and found corresponding
2438 * STRUCT/UNION, they will be from different compilation units. This
2439 * consequently means that when we "link" FWD to corresponding STRUCT/UNION,
2440 * type graph will likely have at least two different BTF types that describe
2441 * same type (e.g., most probably there will be two different BTF types for the
2442 * same 'int' primitive type) and could even have "overlapping" parts of type
2443 * graph that describe same subset of types.
2444 *
2445 * This in turn means that our assumption that each type in canonical graph
2446 * must correspond to exactly one type in candidate graph might not hold
2447 * anymore and will make it harder to detect contradictions using hypothetical
2448 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
2449 * resolution only in canonical graph. FWDs in candidate graphs are never
2450 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs
2451 * that can occur:
2452 * - Both types in canonical and candidate graphs are FWDs. If they are
2453 * structurally equivalent, then they can either be both resolved to the
2454 * same STRUCT/UNION or not resolved at all. In both cases they are
2455 * equivalent and there is no need to resolve FWD on candidate side.
2456 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
2457 * so nothing to resolve as well, algorithm will check equivalence anyway.
2458 * - Type in canonical graph is FWD, while type in candidate is concrete
2459 * STRUCT/UNION. In this case candidate graph comes from single compilation
2460 * unit, so there is exactly one BTF type for each unique C type. After
2461 * resolving FWD into STRUCT/UNION, there might be more than one BTF type
2462 * in canonical graph mapping to single BTF type in candidate graph, but
2463 * because hypothetical mapping maps from canonical to candidate types, it's
2464 * alright, and we still maintain the property of having single `canon_id`
2465 * mapping to single `cand_id` (there could be two different `canon_id`
2466 * mapped to the same `cand_id`, but it's not contradictory).
2467 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
2468 * graph is FWD. In this case we are just going to check compatibility of
2469 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll
2470 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to
2471 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs
2472 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
2473 * canonical graph.
2474 */
2475static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
2476 __u32 canon_id)
2477{
2478 struct btf_type *cand_type;
2479 struct btf_type *canon_type;
2480 __u32 hypot_type_id;
2481 __u16 cand_kind;
2482 __u16 canon_kind;
2483 int i, eq;
2484
2485 /* if both resolve to the same canonical, they must be equivalent */
2486 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id))
2487 return 1;
2488
2489 canon_id = resolve_fwd_id(d, canon_id);
2490
2491 hypot_type_id = d->hypot_map[canon_id];
2492 if (hypot_type_id <= BTF_MAX_NR_TYPES)
2493 return hypot_type_id == cand_id;
2494
2495 if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
2496 return -ENOMEM;
2497
2498 cand_type = d->btf->types[cand_id];
2499 canon_type = d->btf->types[canon_id];
2500 cand_kind = btf_kind(cand_type);
2501 canon_kind = btf_kind(canon_type);
2502
2503 if (cand_type->name_off != canon_type->name_off)
2504 return 0;
2505
2506 /* FWD <--> STRUCT/UNION equivalence check, if enabled */
2507 if (!d->opts.dont_resolve_fwds
2508 && (cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD)
2509 && cand_kind != canon_kind) {
2510 __u16 real_kind;
2511 __u16 fwd_kind;
2512
2513 if (cand_kind == BTF_KIND_FWD) {
2514 real_kind = canon_kind;
2515 fwd_kind = btf_fwd_kind(cand_type);
2516 } else {
2517 real_kind = cand_kind;
2518 fwd_kind = btf_fwd_kind(canon_type);
2519 }
2520 return fwd_kind == real_kind;
2521 }
2522
2523 if (cand_kind != canon_kind)
2524 return 0;
2525
2526 switch (cand_kind) {
2527 case BTF_KIND_INT:
2528 return btf_equal_int(cand_type, canon_type);
2529
2530 case BTF_KIND_ENUM:
2531 if (d->opts.dont_resolve_fwds)
2532 return btf_equal_enum(cand_type, canon_type);
2533 else
2534 return btf_compat_enum(cand_type, canon_type);
2535
2536 case BTF_KIND_FWD:
2537 return btf_equal_common(cand_type, canon_type);
2538
2539 case BTF_KIND_CONST:
2540 case BTF_KIND_VOLATILE:
2541 case BTF_KIND_RESTRICT:
2542 case BTF_KIND_PTR:
2543 case BTF_KIND_TYPEDEF:
2544 case BTF_KIND_FUNC:
2545 if (cand_type->info != canon_type->info)
2546 return 0;
2547 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
2548
2549 case BTF_KIND_ARRAY: {
2550 const struct btf_array *cand_arr, *canon_arr;
2551
2552 if (!btf_compat_array(cand_type, canon_type))
2553 return 0;
2554 cand_arr = btf_array(cand_type);
2555 canon_arr = btf_array(canon_type);
2556 eq = btf_dedup_is_equiv(d,
2557 cand_arr->index_type, canon_arr->index_type);
2558 if (eq <= 0)
2559 return eq;
2560 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
2561 }
2562
2563 case BTF_KIND_STRUCT:
2564 case BTF_KIND_UNION: {
2565 const struct btf_member *cand_m, *canon_m;
2566 __u16 vlen;
2567
2568 if (!btf_shallow_equal_struct(cand_type, canon_type))
2569 return 0;
2570 vlen = btf_vlen(cand_type);
2571 cand_m = btf_members(cand_type);
2572 canon_m = btf_members(canon_type);
2573 for (i = 0; i < vlen; i++) {
2574 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
2575 if (eq <= 0)
2576 return eq;
2577 cand_m++;
2578 canon_m++;
2579 }
2580
2581 return 1;
2582 }
2583
2584 case BTF_KIND_FUNC_PROTO: {
2585 const struct btf_param *cand_p, *canon_p;
2586 __u16 vlen;
2587
2588 if (!btf_compat_fnproto(cand_type, canon_type))
2589 return 0;
2590 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
2591 if (eq <= 0)
2592 return eq;
2593 vlen = btf_vlen(cand_type);
2594 cand_p = btf_params(cand_type);
2595 canon_p = btf_params(canon_type);
2596 for (i = 0; i < vlen; i++) {
2597 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);
2598 if (eq <= 0)
2599 return eq;
2600 cand_p++;
2601 canon_p++;
2602 }
2603 return 1;
2604 }
2605
2606 default:
2607 return -EINVAL;
2608 }
2609 return 0;
2610}
2611
2612/*
2613 * Use hypothetical mapping, produced by successful type graph equivalence
2614 * check, to augment existing struct/union canonical mapping, where possible.
2615 *
2616 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record
2617 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
2618 * it doesn't matter if FWD type was part of canonical graph or candidate one,
2619 * we are recording the mapping anyway. As opposed to carefulness required
2620 * for struct/union correspondence mapping (described below), for FWD resolution
2621 * it's not important, as by the time that FWD type (reference type) will be
2622 * deduplicated all structs/unions will be deduped already anyway.
2623 *
2624 * Recording STRUCT/UNION mapping is purely a performance optimization and is
2625 * not required for correctness. It needs to be done carefully to ensure that
2626 * struct/union from candidate's type graph is not mapped into corresponding
2627 * struct/union from canonical type graph that itself hasn't been resolved into
2628 * canonical representative. The only guarantee we have is that canonical
2629 * struct/union was determined as canonical and that won't change. But any
2630 * types referenced through that struct/union fields could have been not yet
2631 * resolved, so in case like that it's too early to establish any kind of
2632 * correspondence between structs/unions.
2633 *
2634 * No canonical correspondence is derived for primitive types (they are already
2635 * deduplicated completely already anyway) or reference types (they rely on
2636 * stability of struct/union canonical relationship for equivalence checks).
2637 */
2638static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
2639{
2640 __u32 cand_type_id, targ_type_id;
2641 __u16 t_kind, c_kind;
2642 __u32 t_id, c_id;
2643 int i;
2644
2645 for (i = 0; i < d->hypot_cnt; i++) {
2646 cand_type_id = d->hypot_list[i];
2647 targ_type_id = d->hypot_map[cand_type_id];
2648 t_id = resolve_type_id(d, targ_type_id);
2649 c_id = resolve_type_id(d, cand_type_id);
2650 t_kind = btf_kind(d->btf->types[t_id]);
2651 c_kind = btf_kind(d->btf->types[c_id]);
2652 /*
2653 * Resolve FWD into STRUCT/UNION.
2654 * It's ok to resolve FWD into STRUCT/UNION that's not yet
2655 * mapped to canonical representative (as opposed to
2656 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because
2657 * eventually that struct is going to be mapped and all resolved
2658 * FWDs will automatically resolve to correct canonical
2659 * representative. This will happen before ref type deduping,
2660 * which critically depends on stability of these mapping. This
2661 * stability is not a requirement for STRUCT/UNION equivalence
2662 * checks, though.
2663 */
2664 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
2665 d->map[c_id] = t_id;
2666 else if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
2667 d->map[t_id] = c_id;
2668
2669 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
2670 c_kind != BTF_KIND_FWD &&
2671 is_type_mapped(d, c_id) &&
2672 !is_type_mapped(d, t_id)) {
2673 /*
2674 * as a perf optimization, we can map struct/union
2675 * that's part of type graph we just verified for
2676 * equivalence. We can do that for struct/union that has
2677 * canonical representative only, though.
2678 */
2679 d->map[t_id] = c_id;
2680 }
2681 }
2682}
2683
2684/*
2685 * Deduplicate struct/union types.
2686 *
2687 * For each struct/union type its type signature hash is calculated, taking
2688 * into account type's name, size, number, order and names of fields, but
2689 * ignoring type ID's referenced from fields, because they might not be deduped
2690 * completely until after reference types deduplication phase. This type hash
2691 * is used to iterate over all potential canonical types, sharing same hash.
2692 * For each canonical candidate we check whether type graphs that they form
2693 * (through referenced types in fields and so on) are equivalent using algorithm
2694 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and
2695 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping
2696 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
2697 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
2698 * potentially map other structs/unions to their canonical representatives,
2699 * if such relationship hasn't yet been established. This speeds up algorithm
2700 * by eliminating some of the duplicate work.
2701 *
2702 * If no matching canonical representative was found, struct/union is marked
2703 * as canonical for itself and is added into btf_dedup->dedup_table hash map
2704 * for further look ups.
2705 */
2706static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
2707{
2708 struct btf_type *cand_type, *t;
2709 struct hashmap_entry *hash_entry;
2710 /* if we don't find equivalent type, then we are canonical */
2711 __u32 new_id = type_id;
2712 __u16 kind;
2713 long h;
2714
2715 /* already deduped or is in process of deduping (loop detected) */
2716 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
2717 return 0;
2718
2719 t = d->btf->types[type_id];
2720 kind = btf_kind(t);
2721
2722 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
2723 return 0;
2724
2725 h = btf_hash_struct(t);
2726 for_each_dedup_cand(d, hash_entry, h) {
2727 __u32 cand_id = (__u32)(long)hash_entry->value;
2728 int eq;
2729
2730 /*
2731 * Even though btf_dedup_is_equiv() checks for
2732 * btf_shallow_equal_struct() internally when checking two
2733 * structs (unions) for equivalence, we need to guard here
2734 * from picking matching FWD type as a dedup candidate.
2735 * This can happen due to hash collision. In such case just
2736 * relying on btf_dedup_is_equiv() would lead to potentially
2737 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because
2738 * FWD and compatible STRUCT/UNION are considered equivalent.
2739 */
2740 cand_type = d->btf->types[cand_id];
2741 if (!btf_shallow_equal_struct(t, cand_type))
2742 continue;
2743
2744 btf_dedup_clear_hypot_map(d);
2745 eq = btf_dedup_is_equiv(d, type_id, cand_id);
2746 if (eq < 0)
2747 return eq;
2748 if (!eq)
2749 continue;
2750 new_id = cand_id;
2751 btf_dedup_merge_hypot_map(d);
2752 break;
2753 }
2754
2755 d->map[type_id] = new_id;
2756 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
2757 return -ENOMEM;
2758
2759 return 0;
2760}
2761
2762static int btf_dedup_struct_types(struct btf_dedup *d)
2763{
2764 int i, err;
2765
2766 for (i = 1; i <= d->btf->nr_types; i++) {
2767 err = btf_dedup_struct_type(d, i);
2768 if (err)
2769 return err;
2770 }
2771 return 0;
2772}
2773
2774/*
2775 * Deduplicate reference type.
2776 *
2777 * Once all primitive and struct/union types got deduplicated, we can easily
2778 * deduplicate all other (reference) BTF types. This is done in two steps:
2779 *
2780 * 1. Resolve all referenced type IDs into their canonical type IDs. This
2781 * resolution can be done either immediately for primitive or struct/union types
2782 * (because they were deduped in previous two phases) or recursively for
2783 * reference types. Recursion will always terminate at either primitive or
2784 * struct/union type, at which point we can "unwind" chain of reference types
2785 * one by one. There is no danger of encountering cycles because in C type
2786 * system the only way to form type cycle is through struct/union, so any chain
2787 * of reference types, even those taking part in a type cycle, will inevitably
2788 * reach struct/union at some point.
2789 *
2790 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type
2791 * becomes "stable", in the sense that no further deduplication will cause
2792 * any changes to it. With that, it's now possible to calculate type's signature
2793 * hash (this time taking into account referenced type IDs) and loop over all
2794 * potential canonical representatives. If no match was found, current type
2795 * will become canonical representative of itself and will be added into
2796 * btf_dedup->dedup_table as another possible canonical representative.
2797 */
2798static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id)
2799{
2800 struct hashmap_entry *hash_entry;
2801 __u32 new_id = type_id, cand_id;
2802 struct btf_type *t, *cand;
2803 /* if we don't find equivalent type, then we are representative type */
2804 int ref_type_id;
2805 long h;
2806
2807 if (d->map[type_id] == BTF_IN_PROGRESS_ID)
2808 return -ELOOP;
2809 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
2810 return resolve_type_id(d, type_id);
2811
2812 t = d->btf->types[type_id];
2813 d->map[type_id] = BTF_IN_PROGRESS_ID;
2814
2815 switch (btf_kind(t)) {
2816 case BTF_KIND_CONST:
2817 case BTF_KIND_VOLATILE:
2818 case BTF_KIND_RESTRICT:
2819 case BTF_KIND_PTR:
2820 case BTF_KIND_TYPEDEF:
2821 case BTF_KIND_FUNC:
2822 ref_type_id = btf_dedup_ref_type(d, t->type);
2823 if (ref_type_id < 0)
2824 return ref_type_id;
2825 t->type = ref_type_id;
2826
2827 h = btf_hash_common(t);
2828 for_each_dedup_cand(d, hash_entry, h) {
2829 cand_id = (__u32)(long)hash_entry->value;
2830 cand = d->btf->types[cand_id];
2831 if (btf_equal_common(t, cand)) {
2832 new_id = cand_id;
2833 break;
2834 }
2835 }
2836 break;
2837
2838 case BTF_KIND_ARRAY: {
2839 struct btf_array *info = btf_array(t);
2840
2841 ref_type_id = btf_dedup_ref_type(d, info->type);
2842 if (ref_type_id < 0)
2843 return ref_type_id;
2844 info->type = ref_type_id;
2845
2846 ref_type_id = btf_dedup_ref_type(d, info->index_type);
2847 if (ref_type_id < 0)
2848 return ref_type_id;
2849 info->index_type = ref_type_id;
2850
2851 h = btf_hash_array(t);
2852 for_each_dedup_cand(d, hash_entry, h) {
2853 cand_id = (__u32)(long)hash_entry->value;
2854 cand = d->btf->types[cand_id];
2855 if (btf_equal_array(t, cand)) {
2856 new_id = cand_id;
2857 break;
2858 }
2859 }
2860 break;
2861 }
2862
2863 case BTF_KIND_FUNC_PROTO: {
2864 struct btf_param *param;
2865 __u16 vlen;
2866 int i;
2867
2868 ref_type_id = btf_dedup_ref_type(d, t->type);
2869 if (ref_type_id < 0)
2870 return ref_type_id;
2871 t->type = ref_type_id;
2872
2873 vlen = btf_vlen(t);
2874 param = btf_params(t);
2875 for (i = 0; i < vlen; i++) {
2876 ref_type_id = btf_dedup_ref_type(d, param->type);
2877 if (ref_type_id < 0)
2878 return ref_type_id;
2879 param->type = ref_type_id;
2880 param++;
2881 }
2882
2883 h = btf_hash_fnproto(t);
2884 for_each_dedup_cand(d, hash_entry, h) {
2885 cand_id = (__u32)(long)hash_entry->value;
2886 cand = d->btf->types[cand_id];
2887 if (btf_equal_fnproto(t, cand)) {
2888 new_id = cand_id;
2889 break;
2890 }
2891 }
2892 break;
2893 }
2894
2895 default:
2896 return -EINVAL;
2897 }
2898
2899 d->map[type_id] = new_id;
2900 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
2901 return -ENOMEM;
2902
2903 return new_id;
2904}
2905
2906static int btf_dedup_ref_types(struct btf_dedup *d)
2907{
2908 int i, err;
2909
2910 for (i = 1; i <= d->btf->nr_types; i++) {
2911 err = btf_dedup_ref_type(d, i);
2912 if (err < 0)
2913 return err;
2914 }
2915 /* we won't need d->dedup_table anymore */
2916 hashmap__free(d->dedup_table);
2917 d->dedup_table = NULL;
2918 return 0;
2919}
2920
2921/*
2922 * Compact types.
2923 *
2924 * After we established for each type its corresponding canonical representative
2925 * type, we now can eliminate types that are not canonical and leave only
2926 * canonical ones layed out sequentially in memory by copying them over
2927 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
2928 * a map from original type ID to a new compacted type ID, which will be used
2929 * during next phase to "fix up" type IDs, referenced from struct/union and
2930 * reference types.
2931 */
2932static int btf_dedup_compact_types(struct btf_dedup *d)
2933{
2934 struct btf_type **new_types;
2935 __u32 next_type_id = 1;
2936 char *types_start, *p;
2937 int i, len;
2938
2939 /* we are going to reuse hypot_map to store compaction remapping */
2940 d->hypot_map[0] = 0;
2941 for (i = 1; i <= d->btf->nr_types; i++)
2942 d->hypot_map[i] = BTF_UNPROCESSED_ID;
2943
2944 types_start = d->btf->nohdr_data + d->btf->hdr->type_off;
2945 p = types_start;
2946
2947 for (i = 1; i <= d->btf->nr_types; i++) {
2948 if (d->map[i] != i)
2949 continue;
2950
2951 len = btf_type_size(d->btf->types[i]);
2952 if (len < 0)
2953 return len;
2954
2955 memmove(p, d->btf->types[i], len);
2956 d->hypot_map[i] = next_type_id;
2957 d->btf->types[next_type_id] = (struct btf_type *)p;
2958 p += len;
2959 next_type_id++;
2960 }
2961
2962 /* shrink struct btf's internal types index and update btf_header */
2963 d->btf->nr_types = next_type_id - 1;
2964 d->btf->types_size = d->btf->nr_types;
2965 d->btf->hdr->type_len = p - types_start;
2966 new_types = realloc(d->btf->types,
2967 (1 + d->btf->nr_types) * sizeof(struct btf_type *));
2968 if (!new_types)
2969 return -ENOMEM;
2970 d->btf->types = new_types;
2971
2972 /* make sure string section follows type information without gaps */
2973 d->btf->hdr->str_off = p - (char *)d->btf->nohdr_data;
2974 memmove(p, d->btf->strings, d->btf->hdr->str_len);
2975 d->btf->strings = p;
2976 p += d->btf->hdr->str_len;
2977
2978 d->btf->data_size = p - (char *)d->btf->data;
2979 return 0;
2980}
2981
2982/*
2983 * Figure out final (deduplicated and compacted) type ID for provided original
2984 * `type_id` by first resolving it into corresponding canonical type ID and
2985 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
2986 * which is populated during compaction phase.
2987 */
2988static int btf_dedup_remap_type_id(struct btf_dedup *d, __u32 type_id)
2989{
2990 __u32 resolved_type_id, new_type_id;
2991
2992 resolved_type_id = resolve_type_id(d, type_id);
2993 new_type_id = d->hypot_map[resolved_type_id];
2994 if (new_type_id > BTF_MAX_NR_TYPES)
2995 return -EINVAL;
2996 return new_type_id;
2997}
2998
2999/*
3000 * Remap referenced type IDs into deduped type IDs.
3001 *
3002 * After BTF types are deduplicated and compacted, their final type IDs may
3003 * differ from original ones. The map from original to a corresponding
3004 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
3005 * compaction phase. During remapping phase we are rewriting all type IDs
3006 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to
3007 * their final deduped type IDs.
3008 */
3009static int btf_dedup_remap_type(struct btf_dedup *d, __u32 type_id)
3010{
3011 struct btf_type *t = d->btf->types[type_id];
3012 int i, r;
3013
3014 switch (btf_kind(t)) {
3015 case BTF_KIND_INT:
3016 case BTF_KIND_ENUM:
3017 break;
3018
3019 case BTF_KIND_FWD:
3020 case BTF_KIND_CONST:
3021 case BTF_KIND_VOLATILE:
3022 case BTF_KIND_RESTRICT:
3023 case BTF_KIND_PTR:
3024 case BTF_KIND_TYPEDEF:
3025 case BTF_KIND_FUNC:
3026 case BTF_KIND_VAR:
3027 r = btf_dedup_remap_type_id(d, t->type);
3028 if (r < 0)
3029 return r;
3030 t->type = r;
3031 break;
3032
3033 case BTF_KIND_ARRAY: {
3034 struct btf_array *arr_info = btf_array(t);
3035
3036 r = btf_dedup_remap_type_id(d, arr_info->type);
3037 if (r < 0)
3038 return r;
3039 arr_info->type = r;
3040 r = btf_dedup_remap_type_id(d, arr_info->index_type);
3041 if (r < 0)
3042 return r;
3043 arr_info->index_type = r;
3044 break;
3045 }
3046
3047 case BTF_KIND_STRUCT:
3048 case BTF_KIND_UNION: {
3049 struct btf_member *member = btf_members(t);
3050 __u16 vlen = btf_vlen(t);
3051
3052 for (i = 0; i < vlen; i++) {
3053 r = btf_dedup_remap_type_id(d, member->type);
3054 if (r < 0)
3055 return r;
3056 member->type = r;
3057 member++;
3058 }
3059 break;
3060 }
3061
3062 case BTF_KIND_FUNC_PROTO: {
3063 struct btf_param *param = btf_params(t);
3064 __u16 vlen = btf_vlen(t);
3065
3066 r = btf_dedup_remap_type_id(d, t->type);
3067 if (r < 0)
3068 return r;
3069 t->type = r;
3070
3071 for (i = 0; i < vlen; i++) {
3072 r = btf_dedup_remap_type_id(d, param->type);
3073 if (r < 0)
3074 return r;
3075 param->type = r;
3076 param++;
3077 }
3078 break;
3079 }
3080
3081 case BTF_KIND_DATASEC: {
3082 struct btf_var_secinfo *var = btf_var_secinfos(t);
3083 __u16 vlen = btf_vlen(t);
3084
3085 for (i = 0; i < vlen; i++) {
3086 r = btf_dedup_remap_type_id(d, var->type);
3087 if (r < 0)
3088 return r;
3089 var->type = r;
3090 var++;
3091 }
3092 break;
3093 }
3094
3095 default:
3096 return -EINVAL;
3097 }
3098
3099 return 0;
3100}
3101
3102static int btf_dedup_remap_types(struct btf_dedup *d)
3103{
3104 int i, r;
3105
3106 for (i = 1; i <= d->btf->nr_types; i++) {
3107 r = btf_dedup_remap_type(d, i);
3108 if (r < 0)
3109 return r;
3110 }
3111 return 0;
3112}
3113
3114/*
3115 * Probe few well-known locations for vmlinux kernel image and try to load BTF
3116 * data out of it to use for target BTF.
3117 */
3118struct btf *libbpf_find_kernel_btf(void)
3119{
3120 struct {
3121 const char *path_fmt;
3122 bool raw_btf;
3123 } locations[] = {
3124 /* try canonical vmlinux BTF through sysfs first */
3125 { "/sys/kernel/btf/vmlinux", true /* raw BTF */ },
3126 /* fall back to trying to find vmlinux ELF on disk otherwise */
3127 { "/boot/vmlinux-%1$s" },
3128 { "/lib/modules/%1$s/vmlinux-%1$s" },
3129 { "/lib/modules/%1$s/build/vmlinux" },
3130 { "/usr/lib/modules/%1$s/kernel/vmlinux" },
3131 { "/usr/lib/debug/boot/vmlinux-%1$s" },
3132 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
3133 { "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
3134 };
3135 char path[PATH_MAX + 1];
3136 struct utsname buf;
3137 struct btf *btf;
3138 int i;
3139
3140 uname(&buf);
3141
3142 for (i = 0; i < ARRAY_SIZE(locations); i++) {
3143 snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release);
3144
3145 if (access(path, R_OK))
3146 continue;
3147
3148 if (locations[i].raw_btf)
3149 btf = btf__parse_raw(path);
3150 else
3151 btf = btf__parse_elf(path, NULL);
3152
3153 pr_debug("loading kernel BTF '%s': %ld\n",
3154 path, IS_ERR(btf) ? PTR_ERR(btf) : 0);
3155 if (IS_ERR(btf))
3156 continue;
3157
3158 return btf;
3159 }
3160
3161 pr_warn("failed to find valid kernel BTF\n");
3162 return ERR_PTR(-ESRCH);
3163}
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2018 Facebook */
3
4#include <byteswap.h>
5#include <endian.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <string.h>
9#include <fcntl.h>
10#include <unistd.h>
11#include <errno.h>
12#include <sys/utsname.h>
13#include <sys/param.h>
14#include <sys/stat.h>
15#include <linux/kernel.h>
16#include <linux/err.h>
17#include <linux/btf.h>
18#include <gelf.h>
19#include "btf.h"
20#include "bpf.h"
21#include "libbpf.h"
22#include "libbpf_internal.h"
23#include "hashmap.h"
24#include "strset.h"
25#include "str_error.h"
26
27#define BTF_MAX_NR_TYPES 0x7fffffffU
28#define BTF_MAX_STR_OFFSET 0x7fffffffU
29
30static struct btf_type btf_void;
31
32struct btf {
33 /* raw BTF data in native endianness */
34 void *raw_data;
35 /* raw BTF data in non-native endianness */
36 void *raw_data_swapped;
37 __u32 raw_size;
38 /* whether target endianness differs from the native one */
39 bool swapped_endian;
40
41 /*
42 * When BTF is loaded from an ELF or raw memory it is stored
43 * in a contiguous memory block. The hdr, type_data, and, strs_data
44 * point inside that memory region to their respective parts of BTF
45 * representation:
46 *
47 * +--------------------------------+
48 * | Header | Types | Strings |
49 * +--------------------------------+
50 * ^ ^ ^
51 * | | |
52 * hdr | |
53 * types_data-+ |
54 * strs_data------------+
55 *
56 * If BTF data is later modified, e.g., due to types added or
57 * removed, BTF deduplication performed, etc, this contiguous
58 * representation is broken up into three independently allocated
59 * memory regions to be able to modify them independently.
60 * raw_data is nulled out at that point, but can be later allocated
61 * and cached again if user calls btf__raw_data(), at which point
62 * raw_data will contain a contiguous copy of header, types, and
63 * strings:
64 *
65 * +----------+ +---------+ +-----------+
66 * | Header | | Types | | Strings |
67 * +----------+ +---------+ +-----------+
68 * ^ ^ ^
69 * | | |
70 * hdr | |
71 * types_data----+ |
72 * strset__data(strs_set)-----+
73 *
74 * +----------+---------+-----------+
75 * | Header | Types | Strings |
76 * raw_data----->+----------+---------+-----------+
77 */
78 struct btf_header *hdr;
79
80 void *types_data;
81 size_t types_data_cap; /* used size stored in hdr->type_len */
82
83 /* type ID to `struct btf_type *` lookup index
84 * type_offs[0] corresponds to the first non-VOID type:
85 * - for base BTF it's type [1];
86 * - for split BTF it's the first non-base BTF type.
87 */
88 __u32 *type_offs;
89 size_t type_offs_cap;
90 /* number of types in this BTF instance:
91 * - doesn't include special [0] void type;
92 * - for split BTF counts number of types added on top of base BTF.
93 */
94 __u32 nr_types;
95 /* if not NULL, points to the base BTF on top of which the current
96 * split BTF is based
97 */
98 struct btf *base_btf;
99 /* BTF type ID of the first type in this BTF instance:
100 * - for base BTF it's equal to 1;
101 * - for split BTF it's equal to biggest type ID of base BTF plus 1.
102 */
103 int start_id;
104 /* logical string offset of this BTF instance:
105 * - for base BTF it's equal to 0;
106 * - for split BTF it's equal to total size of base BTF's string section size.
107 */
108 int start_str_off;
109
110 /* only one of strs_data or strs_set can be non-NULL, depending on
111 * whether BTF is in a modifiable state (strs_set is used) or not
112 * (strs_data points inside raw_data)
113 */
114 void *strs_data;
115 /* a set of unique strings */
116 struct strset *strs_set;
117 /* whether strings are already deduplicated */
118 bool strs_deduped;
119
120 /* whether base_btf should be freed in btf_free for this instance */
121 bool owns_base;
122
123 /* BTF object FD, if loaded into kernel */
124 int fd;
125
126 /* Pointer size (in bytes) for a target architecture of this BTF */
127 int ptr_sz;
128};
129
130static inline __u64 ptr_to_u64(const void *ptr)
131{
132 return (__u64) (unsigned long) ptr;
133}
134
135/* Ensure given dynamically allocated memory region pointed to by *data* with
136 * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough
137 * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements
138 * are already used. At most *max_cnt* elements can be ever allocated.
139 * If necessary, memory is reallocated and all existing data is copied over,
140 * new pointer to the memory region is stored at *data, new memory region
141 * capacity (in number of elements) is stored in *cap.
142 * On success, memory pointer to the beginning of unused memory is returned.
143 * On error, NULL is returned.
144 */
145void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
146 size_t cur_cnt, size_t max_cnt, size_t add_cnt)
147{
148 size_t new_cnt;
149 void *new_data;
150
151 if (cur_cnt + add_cnt <= *cap_cnt)
152 return *data + cur_cnt * elem_sz;
153
154 /* requested more than the set limit */
155 if (cur_cnt + add_cnt > max_cnt)
156 return NULL;
157
158 new_cnt = *cap_cnt;
159 new_cnt += new_cnt / 4; /* expand by 25% */
160 if (new_cnt < 16) /* but at least 16 elements */
161 new_cnt = 16;
162 if (new_cnt > max_cnt) /* but not exceeding a set limit */
163 new_cnt = max_cnt;
164 if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */
165 new_cnt = cur_cnt + add_cnt;
166
167 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz);
168 if (!new_data)
169 return NULL;
170
171 /* zero out newly allocated portion of memory */
172 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz);
173
174 *data = new_data;
175 *cap_cnt = new_cnt;
176 return new_data + cur_cnt * elem_sz;
177}
178
179/* Ensure given dynamically allocated memory region has enough allocated space
180 * to accommodate *need_cnt* elements of size *elem_sz* bytes each
181 */
182int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt)
183{
184 void *p;
185
186 if (need_cnt <= *cap_cnt)
187 return 0;
188
189 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt);
190 if (!p)
191 return -ENOMEM;
192
193 return 0;
194}
195
196static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt)
197{
198 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
199 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt);
200}
201
202static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
203{
204 __u32 *p;
205
206 p = btf_add_type_offs_mem(btf, 1);
207 if (!p)
208 return -ENOMEM;
209
210 *p = type_off;
211 return 0;
212}
213
214static void btf_bswap_hdr(struct btf_header *h)
215{
216 h->magic = bswap_16(h->magic);
217 h->hdr_len = bswap_32(h->hdr_len);
218 h->type_off = bswap_32(h->type_off);
219 h->type_len = bswap_32(h->type_len);
220 h->str_off = bswap_32(h->str_off);
221 h->str_len = bswap_32(h->str_len);
222}
223
224static int btf_parse_hdr(struct btf *btf)
225{
226 struct btf_header *hdr = btf->hdr;
227 __u32 meta_left;
228
229 if (btf->raw_size < sizeof(struct btf_header)) {
230 pr_debug("BTF header not found\n");
231 return -EINVAL;
232 }
233
234 if (hdr->magic == bswap_16(BTF_MAGIC)) {
235 btf->swapped_endian = true;
236 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
237 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
238 bswap_32(hdr->hdr_len));
239 return -ENOTSUP;
240 }
241 btf_bswap_hdr(hdr);
242 } else if (hdr->magic != BTF_MAGIC) {
243 pr_debug("Invalid BTF magic: %x\n", hdr->magic);
244 return -EINVAL;
245 }
246
247 if (btf->raw_size < hdr->hdr_len) {
248 pr_debug("BTF header len %u larger than data size %u\n",
249 hdr->hdr_len, btf->raw_size);
250 return -EINVAL;
251 }
252
253 meta_left = btf->raw_size - hdr->hdr_len;
254 if (meta_left < (long long)hdr->str_off + hdr->str_len) {
255 pr_debug("Invalid BTF total size: %u\n", btf->raw_size);
256 return -EINVAL;
257 }
258
259 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) {
260 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
261 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
262 return -EINVAL;
263 }
264
265 if (hdr->type_off % 4) {
266 pr_debug("BTF type section is not aligned to 4 bytes\n");
267 return -EINVAL;
268 }
269
270 return 0;
271}
272
273static int btf_parse_str_sec(struct btf *btf)
274{
275 const struct btf_header *hdr = btf->hdr;
276 const char *start = btf->strs_data;
277 const char *end = start + btf->hdr->str_len;
278
279 if (btf->base_btf && hdr->str_len == 0)
280 return 0;
281 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
282 pr_debug("Invalid BTF string section\n");
283 return -EINVAL;
284 }
285 if (!btf->base_btf && start[0]) {
286 pr_debug("Invalid BTF string section\n");
287 return -EINVAL;
288 }
289 return 0;
290}
291
292static int btf_type_size(const struct btf_type *t)
293{
294 const int base_size = sizeof(struct btf_type);
295 __u16 vlen = btf_vlen(t);
296
297 switch (btf_kind(t)) {
298 case BTF_KIND_FWD:
299 case BTF_KIND_CONST:
300 case BTF_KIND_VOLATILE:
301 case BTF_KIND_RESTRICT:
302 case BTF_KIND_PTR:
303 case BTF_KIND_TYPEDEF:
304 case BTF_KIND_FUNC:
305 case BTF_KIND_FLOAT:
306 case BTF_KIND_TYPE_TAG:
307 return base_size;
308 case BTF_KIND_INT:
309 return base_size + sizeof(__u32);
310 case BTF_KIND_ENUM:
311 return base_size + vlen * sizeof(struct btf_enum);
312 case BTF_KIND_ENUM64:
313 return base_size + vlen * sizeof(struct btf_enum64);
314 case BTF_KIND_ARRAY:
315 return base_size + sizeof(struct btf_array);
316 case BTF_KIND_STRUCT:
317 case BTF_KIND_UNION:
318 return base_size + vlen * sizeof(struct btf_member);
319 case BTF_KIND_FUNC_PROTO:
320 return base_size + vlen * sizeof(struct btf_param);
321 case BTF_KIND_VAR:
322 return base_size + sizeof(struct btf_var);
323 case BTF_KIND_DATASEC:
324 return base_size + vlen * sizeof(struct btf_var_secinfo);
325 case BTF_KIND_DECL_TAG:
326 return base_size + sizeof(struct btf_decl_tag);
327 default:
328 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
329 return -EINVAL;
330 }
331}
332
333static void btf_bswap_type_base(struct btf_type *t)
334{
335 t->name_off = bswap_32(t->name_off);
336 t->info = bswap_32(t->info);
337 t->type = bswap_32(t->type);
338}
339
340static int btf_bswap_type_rest(struct btf_type *t)
341{
342 struct btf_var_secinfo *v;
343 struct btf_enum64 *e64;
344 struct btf_member *m;
345 struct btf_array *a;
346 struct btf_param *p;
347 struct btf_enum *e;
348 __u16 vlen = btf_vlen(t);
349 int i;
350
351 switch (btf_kind(t)) {
352 case BTF_KIND_FWD:
353 case BTF_KIND_CONST:
354 case BTF_KIND_VOLATILE:
355 case BTF_KIND_RESTRICT:
356 case BTF_KIND_PTR:
357 case BTF_KIND_TYPEDEF:
358 case BTF_KIND_FUNC:
359 case BTF_KIND_FLOAT:
360 case BTF_KIND_TYPE_TAG:
361 return 0;
362 case BTF_KIND_INT:
363 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1));
364 return 0;
365 case BTF_KIND_ENUM:
366 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) {
367 e->name_off = bswap_32(e->name_off);
368 e->val = bswap_32(e->val);
369 }
370 return 0;
371 case BTF_KIND_ENUM64:
372 for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) {
373 e64->name_off = bswap_32(e64->name_off);
374 e64->val_lo32 = bswap_32(e64->val_lo32);
375 e64->val_hi32 = bswap_32(e64->val_hi32);
376 }
377 return 0;
378 case BTF_KIND_ARRAY:
379 a = btf_array(t);
380 a->type = bswap_32(a->type);
381 a->index_type = bswap_32(a->index_type);
382 a->nelems = bswap_32(a->nelems);
383 return 0;
384 case BTF_KIND_STRUCT:
385 case BTF_KIND_UNION:
386 for (i = 0, m = btf_members(t); i < vlen; i++, m++) {
387 m->name_off = bswap_32(m->name_off);
388 m->type = bswap_32(m->type);
389 m->offset = bswap_32(m->offset);
390 }
391 return 0;
392 case BTF_KIND_FUNC_PROTO:
393 for (i = 0, p = btf_params(t); i < vlen; i++, p++) {
394 p->name_off = bswap_32(p->name_off);
395 p->type = bswap_32(p->type);
396 }
397 return 0;
398 case BTF_KIND_VAR:
399 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);
400 return 0;
401 case BTF_KIND_DATASEC:
402 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) {
403 v->type = bswap_32(v->type);
404 v->offset = bswap_32(v->offset);
405 v->size = bswap_32(v->size);
406 }
407 return 0;
408 case BTF_KIND_DECL_TAG:
409 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx);
410 return 0;
411 default:
412 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
413 return -EINVAL;
414 }
415}
416
417static int btf_parse_type_sec(struct btf *btf)
418{
419 struct btf_header *hdr = btf->hdr;
420 void *next_type = btf->types_data;
421 void *end_type = next_type + hdr->type_len;
422 int err, type_size;
423
424 while (next_type + sizeof(struct btf_type) <= end_type) {
425 if (btf->swapped_endian)
426 btf_bswap_type_base(next_type);
427
428 type_size = btf_type_size(next_type);
429 if (type_size < 0)
430 return type_size;
431 if (next_type + type_size > end_type) {
432 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
433 return -EINVAL;
434 }
435
436 if (btf->swapped_endian && btf_bswap_type_rest(next_type))
437 return -EINVAL;
438
439 err = btf_add_type_idx_entry(btf, next_type - btf->types_data);
440 if (err)
441 return err;
442
443 next_type += type_size;
444 btf->nr_types++;
445 }
446
447 if (next_type != end_type) {
448 pr_warn("BTF types data is malformed\n");
449 return -EINVAL;
450 }
451
452 return 0;
453}
454
455static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id)
456{
457 const char *s;
458
459 s = btf__str_by_offset(btf, str_off);
460 if (!s) {
461 pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off);
462 return -EINVAL;
463 }
464
465 return 0;
466}
467
468static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id)
469{
470 const struct btf_type *t;
471
472 t = btf__type_by_id(btf, id);
473 if (!t) {
474 pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id);
475 return -EINVAL;
476 }
477
478 return 0;
479}
480
481static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id)
482{
483 __u32 kind = btf_kind(t);
484 int err, i, n;
485
486 err = btf_validate_str(btf, t->name_off, "type name", id);
487 if (err)
488 return err;
489
490 switch (kind) {
491 case BTF_KIND_UNKN:
492 case BTF_KIND_INT:
493 case BTF_KIND_FWD:
494 case BTF_KIND_FLOAT:
495 break;
496 case BTF_KIND_PTR:
497 case BTF_KIND_TYPEDEF:
498 case BTF_KIND_VOLATILE:
499 case BTF_KIND_CONST:
500 case BTF_KIND_RESTRICT:
501 case BTF_KIND_VAR:
502 case BTF_KIND_DECL_TAG:
503 case BTF_KIND_TYPE_TAG:
504 err = btf_validate_id(btf, t->type, id);
505 if (err)
506 return err;
507 break;
508 case BTF_KIND_ARRAY: {
509 const struct btf_array *a = btf_array(t);
510
511 err = btf_validate_id(btf, a->type, id);
512 err = err ?: btf_validate_id(btf, a->index_type, id);
513 if (err)
514 return err;
515 break;
516 }
517 case BTF_KIND_STRUCT:
518 case BTF_KIND_UNION: {
519 const struct btf_member *m = btf_members(t);
520
521 n = btf_vlen(t);
522 for (i = 0; i < n; i++, m++) {
523 err = btf_validate_str(btf, m->name_off, "field name", id);
524 err = err ?: btf_validate_id(btf, m->type, id);
525 if (err)
526 return err;
527 }
528 break;
529 }
530 case BTF_KIND_ENUM: {
531 const struct btf_enum *m = btf_enum(t);
532
533 n = btf_vlen(t);
534 for (i = 0; i < n; i++, m++) {
535 err = btf_validate_str(btf, m->name_off, "enum name", id);
536 if (err)
537 return err;
538 }
539 break;
540 }
541 case BTF_KIND_ENUM64: {
542 const struct btf_enum64 *m = btf_enum64(t);
543
544 n = btf_vlen(t);
545 for (i = 0; i < n; i++, m++) {
546 err = btf_validate_str(btf, m->name_off, "enum name", id);
547 if (err)
548 return err;
549 }
550 break;
551 }
552 case BTF_KIND_FUNC: {
553 const struct btf_type *ft;
554
555 err = btf_validate_id(btf, t->type, id);
556 if (err)
557 return err;
558 ft = btf__type_by_id(btf, t->type);
559 if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) {
560 pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type);
561 return -EINVAL;
562 }
563 break;
564 }
565 case BTF_KIND_FUNC_PROTO: {
566 const struct btf_param *m = btf_params(t);
567
568 n = btf_vlen(t);
569 for (i = 0; i < n; i++, m++) {
570 err = btf_validate_str(btf, m->name_off, "param name", id);
571 err = err ?: btf_validate_id(btf, m->type, id);
572 if (err)
573 return err;
574 }
575 break;
576 }
577 case BTF_KIND_DATASEC: {
578 const struct btf_var_secinfo *m = btf_var_secinfos(t);
579
580 n = btf_vlen(t);
581 for (i = 0; i < n; i++, m++) {
582 err = btf_validate_id(btf, m->type, id);
583 if (err)
584 return err;
585 }
586 break;
587 }
588 default:
589 pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind);
590 return -EINVAL;
591 }
592 return 0;
593}
594
595/* Validate basic sanity of BTF. It's intentionally less thorough than
596 * kernel's validation and validates only properties of BTF that libbpf relies
597 * on to be correct (e.g., valid type IDs, valid string offsets, etc)
598 */
599static int btf_sanity_check(const struct btf *btf)
600{
601 const struct btf_type *t;
602 __u32 i, n = btf__type_cnt(btf);
603 int err;
604
605 for (i = btf->start_id; i < n; i++) {
606 t = btf_type_by_id(btf, i);
607 err = btf_validate_type(btf, t, i);
608 if (err)
609 return err;
610 }
611 return 0;
612}
613
614__u32 btf__type_cnt(const struct btf *btf)
615{
616 return btf->start_id + btf->nr_types;
617}
618
619const struct btf *btf__base_btf(const struct btf *btf)
620{
621 return btf->base_btf;
622}
623
624/* internal helper returning non-const pointer to a type */
625struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id)
626{
627 if (type_id == 0)
628 return &btf_void;
629 if (type_id < btf->start_id)
630 return btf_type_by_id(btf->base_btf, type_id);
631 return btf->types_data + btf->type_offs[type_id - btf->start_id];
632}
633
634const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
635{
636 if (type_id >= btf->start_id + btf->nr_types)
637 return errno = EINVAL, NULL;
638 return btf_type_by_id((struct btf *)btf, type_id);
639}
640
641static int determine_ptr_size(const struct btf *btf)
642{
643 static const char * const long_aliases[] = {
644 "long",
645 "long int",
646 "int long",
647 "unsigned long",
648 "long unsigned",
649 "unsigned long int",
650 "unsigned int long",
651 "long unsigned int",
652 "long int unsigned",
653 "int unsigned long",
654 "int long unsigned",
655 };
656 const struct btf_type *t;
657 const char *name;
658 int i, j, n;
659
660 if (btf->base_btf && btf->base_btf->ptr_sz > 0)
661 return btf->base_btf->ptr_sz;
662
663 n = btf__type_cnt(btf);
664 for (i = 1; i < n; i++) {
665 t = btf__type_by_id(btf, i);
666 if (!btf_is_int(t))
667 continue;
668
669 if (t->size != 4 && t->size != 8)
670 continue;
671
672 name = btf__name_by_offset(btf, t->name_off);
673 if (!name)
674 continue;
675
676 for (j = 0; j < ARRAY_SIZE(long_aliases); j++) {
677 if (strcmp(name, long_aliases[j]) == 0)
678 return t->size;
679 }
680 }
681
682 return -1;
683}
684
685static size_t btf_ptr_sz(const struct btf *btf)
686{
687 if (!btf->ptr_sz)
688 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
689 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;
690}
691
692/* Return pointer size this BTF instance assumes. The size is heuristically
693 * determined by looking for 'long' or 'unsigned long' integer type and
694 * recording its size in bytes. If BTF type information doesn't have any such
695 * type, this function returns 0. In the latter case, native architecture's
696 * pointer size is assumed, so will be either 4 or 8, depending on
697 * architecture that libbpf was compiled for. It's possible to override
698 * guessed value by using btf__set_pointer_size() API.
699 */
700size_t btf__pointer_size(const struct btf *btf)
701{
702 if (!btf->ptr_sz)
703 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
704
705 if (btf->ptr_sz < 0)
706 /* not enough BTF type info to guess */
707 return 0;
708
709 return btf->ptr_sz;
710}
711
712/* Override or set pointer size in bytes. Only values of 4 and 8 are
713 * supported.
714 */
715int btf__set_pointer_size(struct btf *btf, size_t ptr_sz)
716{
717 if (ptr_sz != 4 && ptr_sz != 8)
718 return libbpf_err(-EINVAL);
719 btf->ptr_sz = ptr_sz;
720 return 0;
721}
722
723static bool is_host_big_endian(void)
724{
725#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
726 return false;
727#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
728 return true;
729#else
730# error "Unrecognized __BYTE_ORDER__"
731#endif
732}
733
734enum btf_endianness btf__endianness(const struct btf *btf)
735{
736 if (is_host_big_endian())
737 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
738 else
739 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
740}
741
742int btf__set_endianness(struct btf *btf, enum btf_endianness endian)
743{
744 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
745 return libbpf_err(-EINVAL);
746
747 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
748 if (!btf->swapped_endian) {
749 free(btf->raw_data_swapped);
750 btf->raw_data_swapped = NULL;
751 }
752 return 0;
753}
754
755static bool btf_type_is_void(const struct btf_type *t)
756{
757 return t == &btf_void || btf_is_fwd(t);
758}
759
760static bool btf_type_is_void_or_null(const struct btf_type *t)
761{
762 return !t || btf_type_is_void(t);
763}
764
765#define MAX_RESOLVE_DEPTH 32
766
767__s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
768{
769 const struct btf_array *array;
770 const struct btf_type *t;
771 __u32 nelems = 1;
772 __s64 size = -1;
773 int i;
774
775 t = btf__type_by_id(btf, type_id);
776 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) {
777 switch (btf_kind(t)) {
778 case BTF_KIND_INT:
779 case BTF_KIND_STRUCT:
780 case BTF_KIND_UNION:
781 case BTF_KIND_ENUM:
782 case BTF_KIND_ENUM64:
783 case BTF_KIND_DATASEC:
784 case BTF_KIND_FLOAT:
785 size = t->size;
786 goto done;
787 case BTF_KIND_PTR:
788 size = btf_ptr_sz(btf);
789 goto done;
790 case BTF_KIND_TYPEDEF:
791 case BTF_KIND_VOLATILE:
792 case BTF_KIND_CONST:
793 case BTF_KIND_RESTRICT:
794 case BTF_KIND_VAR:
795 case BTF_KIND_DECL_TAG:
796 case BTF_KIND_TYPE_TAG:
797 type_id = t->type;
798 break;
799 case BTF_KIND_ARRAY:
800 array = btf_array(t);
801 if (nelems && array->nelems > UINT32_MAX / nelems)
802 return libbpf_err(-E2BIG);
803 nelems *= array->nelems;
804 type_id = array->type;
805 break;
806 default:
807 return libbpf_err(-EINVAL);
808 }
809
810 t = btf__type_by_id(btf, type_id);
811 }
812
813done:
814 if (size < 0)
815 return libbpf_err(-EINVAL);
816 if (nelems && size > UINT32_MAX / nelems)
817 return libbpf_err(-E2BIG);
818
819 return nelems * size;
820}
821
822int btf__align_of(const struct btf *btf, __u32 id)
823{
824 const struct btf_type *t = btf__type_by_id(btf, id);
825 __u16 kind = btf_kind(t);
826
827 switch (kind) {
828 case BTF_KIND_INT:
829 case BTF_KIND_ENUM:
830 case BTF_KIND_ENUM64:
831 case BTF_KIND_FLOAT:
832 return min(btf_ptr_sz(btf), (size_t)t->size);
833 case BTF_KIND_PTR:
834 return btf_ptr_sz(btf);
835 case BTF_KIND_TYPEDEF:
836 case BTF_KIND_VOLATILE:
837 case BTF_KIND_CONST:
838 case BTF_KIND_RESTRICT:
839 case BTF_KIND_TYPE_TAG:
840 return btf__align_of(btf, t->type);
841 case BTF_KIND_ARRAY:
842 return btf__align_of(btf, btf_array(t)->type);
843 case BTF_KIND_STRUCT:
844 case BTF_KIND_UNION: {
845 const struct btf_member *m = btf_members(t);
846 __u16 vlen = btf_vlen(t);
847 int i, max_align = 1, align;
848
849 for (i = 0; i < vlen; i++, m++) {
850 align = btf__align_of(btf, m->type);
851 if (align <= 0)
852 return libbpf_err(align);
853 max_align = max(max_align, align);
854
855 /* if field offset isn't aligned according to field
856 * type's alignment, then struct must be packed
857 */
858 if (btf_member_bitfield_size(t, i) == 0 &&
859 (m->offset % (8 * align)) != 0)
860 return 1;
861 }
862
863 /* if struct/union size isn't a multiple of its alignment,
864 * then struct must be packed
865 */
866 if ((t->size % max_align) != 0)
867 return 1;
868
869 return max_align;
870 }
871 default:
872 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
873 return errno = EINVAL, 0;
874 }
875}
876
877int btf__resolve_type(const struct btf *btf, __u32 type_id)
878{
879 const struct btf_type *t;
880 int depth = 0;
881
882 t = btf__type_by_id(btf, type_id);
883 while (depth < MAX_RESOLVE_DEPTH &&
884 !btf_type_is_void_or_null(t) &&
885 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) {
886 type_id = t->type;
887 t = btf__type_by_id(btf, type_id);
888 depth++;
889 }
890
891 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t))
892 return libbpf_err(-EINVAL);
893
894 return type_id;
895}
896
897__s32 btf__find_by_name(const struct btf *btf, const char *type_name)
898{
899 __u32 i, nr_types = btf__type_cnt(btf);
900
901 if (!strcmp(type_name, "void"))
902 return 0;
903
904 for (i = 1; i < nr_types; i++) {
905 const struct btf_type *t = btf__type_by_id(btf, i);
906 const char *name = btf__name_by_offset(btf, t->name_off);
907
908 if (name && !strcmp(type_name, name))
909 return i;
910 }
911
912 return libbpf_err(-ENOENT);
913}
914
915static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id,
916 const char *type_name, __u32 kind)
917{
918 __u32 i, nr_types = btf__type_cnt(btf);
919
920 if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
921 return 0;
922
923 for (i = start_id; i < nr_types; i++) {
924 const struct btf_type *t = btf__type_by_id(btf, i);
925 const char *name;
926
927 if (btf_kind(t) != kind)
928 continue;
929 name = btf__name_by_offset(btf, t->name_off);
930 if (name && !strcmp(type_name, name))
931 return i;
932 }
933
934 return libbpf_err(-ENOENT);
935}
936
937__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
938 __u32 kind)
939{
940 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind);
941}
942
943__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
944 __u32 kind)
945{
946 return btf_find_by_name_kind(btf, 1, type_name, kind);
947}
948
949static bool btf_is_modifiable(const struct btf *btf)
950{
951 return (void *)btf->hdr != btf->raw_data;
952}
953
954void btf__free(struct btf *btf)
955{
956 if (IS_ERR_OR_NULL(btf))
957 return;
958
959 if (btf->fd >= 0)
960 close(btf->fd);
961
962 if (btf_is_modifiable(btf)) {
963 /* if BTF was modified after loading, it will have a split
964 * in-memory representation for header, types, and strings
965 * sections, so we need to free all of them individually. It
966 * might still have a cached contiguous raw data present,
967 * which will be unconditionally freed below.
968 */
969 free(btf->hdr);
970 free(btf->types_data);
971 strset__free(btf->strs_set);
972 }
973 free(btf->raw_data);
974 free(btf->raw_data_swapped);
975 free(btf->type_offs);
976 if (btf->owns_base)
977 btf__free(btf->base_btf);
978 free(btf);
979}
980
981static struct btf *btf_new_empty(struct btf *base_btf)
982{
983 struct btf *btf;
984
985 btf = calloc(1, sizeof(*btf));
986 if (!btf)
987 return ERR_PTR(-ENOMEM);
988
989 btf->nr_types = 0;
990 btf->start_id = 1;
991 btf->start_str_off = 0;
992 btf->fd = -1;
993 btf->ptr_sz = sizeof(void *);
994 btf->swapped_endian = false;
995
996 if (base_btf) {
997 btf->base_btf = base_btf;
998 btf->start_id = btf__type_cnt(base_btf);
999 btf->start_str_off = base_btf->hdr->str_len;
1000 btf->swapped_endian = base_btf->swapped_endian;
1001 }
1002
1003 /* +1 for empty string at offset 0 */
1004 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
1005 btf->raw_data = calloc(1, btf->raw_size);
1006 if (!btf->raw_data) {
1007 free(btf);
1008 return ERR_PTR(-ENOMEM);
1009 }
1010
1011 btf->hdr = btf->raw_data;
1012 btf->hdr->hdr_len = sizeof(struct btf_header);
1013 btf->hdr->magic = BTF_MAGIC;
1014 btf->hdr->version = BTF_VERSION;
1015
1016 btf->types_data = btf->raw_data + btf->hdr->hdr_len;
1017 btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
1018 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
1019
1020 return btf;
1021}
1022
1023struct btf *btf__new_empty(void)
1024{
1025 return libbpf_ptr(btf_new_empty(NULL));
1026}
1027
1028struct btf *btf__new_empty_split(struct btf *base_btf)
1029{
1030 return libbpf_ptr(btf_new_empty(base_btf));
1031}
1032
1033static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
1034{
1035 struct btf *btf;
1036 int err;
1037
1038 btf = calloc(1, sizeof(struct btf));
1039 if (!btf)
1040 return ERR_PTR(-ENOMEM);
1041
1042 btf->nr_types = 0;
1043 btf->start_id = 1;
1044 btf->start_str_off = 0;
1045 btf->fd = -1;
1046
1047 if (base_btf) {
1048 btf->base_btf = base_btf;
1049 btf->start_id = btf__type_cnt(base_btf);
1050 btf->start_str_off = base_btf->hdr->str_len;
1051 }
1052
1053 btf->raw_data = malloc(size);
1054 if (!btf->raw_data) {
1055 err = -ENOMEM;
1056 goto done;
1057 }
1058 memcpy(btf->raw_data, data, size);
1059 btf->raw_size = size;
1060
1061 btf->hdr = btf->raw_data;
1062 err = btf_parse_hdr(btf);
1063 if (err)
1064 goto done;
1065
1066 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;
1067 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
1068
1069 err = btf_parse_str_sec(btf);
1070 err = err ?: btf_parse_type_sec(btf);
1071 err = err ?: btf_sanity_check(btf);
1072 if (err)
1073 goto done;
1074
1075done:
1076 if (err) {
1077 btf__free(btf);
1078 return ERR_PTR(err);
1079 }
1080
1081 return btf;
1082}
1083
1084struct btf *btf__new(const void *data, __u32 size)
1085{
1086 return libbpf_ptr(btf_new(data, size, NULL));
1087}
1088
1089struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf)
1090{
1091 return libbpf_ptr(btf_new(data, size, base_btf));
1092}
1093
1094struct btf_elf_secs {
1095 Elf_Data *btf_data;
1096 Elf_Data *btf_ext_data;
1097 Elf_Data *btf_base_data;
1098};
1099
1100static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs *secs)
1101{
1102 Elf_Scn *scn = NULL;
1103 Elf_Data *data;
1104 GElf_Ehdr ehdr;
1105 size_t shstrndx;
1106 int idx = 0;
1107
1108 if (!gelf_getehdr(elf, &ehdr)) {
1109 pr_warn("failed to get EHDR from %s\n", path);
1110 goto err;
1111 }
1112
1113 if (elf_getshdrstrndx(elf, &shstrndx)) {
1114 pr_warn("failed to get section names section index for %s\n",
1115 path);
1116 goto err;
1117 }
1118
1119 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
1120 pr_warn("failed to get e_shstrndx from %s\n", path);
1121 goto err;
1122 }
1123
1124 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1125 Elf_Data **field;
1126 GElf_Shdr sh;
1127 char *name;
1128
1129 idx++;
1130 if (gelf_getshdr(scn, &sh) != &sh) {
1131 pr_warn("failed to get section(%d) header from %s\n",
1132 idx, path);
1133 goto err;
1134 }
1135 name = elf_strptr(elf, shstrndx, sh.sh_name);
1136 if (!name) {
1137 pr_warn("failed to get section(%d) name from %s\n",
1138 idx, path);
1139 goto err;
1140 }
1141
1142 if (strcmp(name, BTF_ELF_SEC) == 0)
1143 field = &secs->btf_data;
1144 else if (strcmp(name, BTF_EXT_ELF_SEC) == 0)
1145 field = &secs->btf_ext_data;
1146 else if (strcmp(name, BTF_BASE_ELF_SEC) == 0)
1147 field = &secs->btf_base_data;
1148 else
1149 continue;
1150
1151 data = elf_getdata(scn, 0);
1152 if (!data) {
1153 pr_warn("failed to get section(%d, %s) data from %s\n",
1154 idx, name, path);
1155 goto err;
1156 }
1157 *field = data;
1158 }
1159
1160 return 0;
1161
1162err:
1163 return -LIBBPF_ERRNO__FORMAT;
1164}
1165
1166static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
1167 struct btf_ext **btf_ext)
1168{
1169 struct btf_elf_secs secs = {};
1170 struct btf *dist_base_btf = NULL;
1171 struct btf *btf = NULL;
1172 int err = 0, fd = -1;
1173 Elf *elf = NULL;
1174
1175 if (elf_version(EV_CURRENT) == EV_NONE) {
1176 pr_warn("failed to init libelf for %s\n", path);
1177 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1178 }
1179
1180 fd = open(path, O_RDONLY | O_CLOEXEC);
1181 if (fd < 0) {
1182 err = -errno;
1183 pr_warn("failed to open %s: %s\n", path, errstr(err));
1184 return ERR_PTR(err);
1185 }
1186
1187 elf = elf_begin(fd, ELF_C_READ, NULL);
1188 if (!elf) {
1189 err = -LIBBPF_ERRNO__FORMAT;
1190 pr_warn("failed to open %s as ELF file\n", path);
1191 goto done;
1192 }
1193
1194 err = btf_find_elf_sections(elf, path, &secs);
1195 if (err)
1196 goto done;
1197
1198 if (!secs.btf_data) {
1199 pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path);
1200 err = -ENODATA;
1201 goto done;
1202 }
1203
1204 if (secs.btf_base_data) {
1205 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size,
1206 NULL);
1207 if (IS_ERR(dist_base_btf)) {
1208 err = PTR_ERR(dist_base_btf);
1209 dist_base_btf = NULL;
1210 goto done;
1211 }
1212 }
1213
1214 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size,
1215 dist_base_btf ?: base_btf);
1216 if (IS_ERR(btf)) {
1217 err = PTR_ERR(btf);
1218 goto done;
1219 }
1220 if (dist_base_btf && base_btf) {
1221 err = btf__relocate(btf, base_btf);
1222 if (err)
1223 goto done;
1224 btf__free(dist_base_btf);
1225 dist_base_btf = NULL;
1226 }
1227
1228 if (dist_base_btf)
1229 btf->owns_base = true;
1230
1231 switch (gelf_getclass(elf)) {
1232 case ELFCLASS32:
1233 btf__set_pointer_size(btf, 4);
1234 break;
1235 case ELFCLASS64:
1236 btf__set_pointer_size(btf, 8);
1237 break;
1238 default:
1239 pr_warn("failed to get ELF class (bitness) for %s\n", path);
1240 break;
1241 }
1242
1243 if (btf_ext && secs.btf_ext_data) {
1244 *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size);
1245 if (IS_ERR(*btf_ext)) {
1246 err = PTR_ERR(*btf_ext);
1247 goto done;
1248 }
1249 } else if (btf_ext) {
1250 *btf_ext = NULL;
1251 }
1252done:
1253 if (elf)
1254 elf_end(elf);
1255 close(fd);
1256
1257 if (!err)
1258 return btf;
1259
1260 if (btf_ext)
1261 btf_ext__free(*btf_ext);
1262 btf__free(dist_base_btf);
1263 btf__free(btf);
1264
1265 return ERR_PTR(err);
1266}
1267
1268struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
1269{
1270 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext));
1271}
1272
1273struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
1274{
1275 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL));
1276}
1277
1278static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
1279{
1280 struct btf *btf = NULL;
1281 void *data = NULL;
1282 FILE *f = NULL;
1283 __u16 magic;
1284 int err = 0;
1285 long sz;
1286
1287 f = fopen(path, "rbe");
1288 if (!f) {
1289 err = -errno;
1290 goto err_out;
1291 }
1292
1293 /* check BTF magic */
1294 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
1295 err = -EIO;
1296 goto err_out;
1297 }
1298 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
1299 /* definitely not a raw BTF */
1300 err = -EPROTO;
1301 goto err_out;
1302 }
1303
1304 /* get file size */
1305 if (fseek(f, 0, SEEK_END)) {
1306 err = -errno;
1307 goto err_out;
1308 }
1309 sz = ftell(f);
1310 if (sz < 0) {
1311 err = -errno;
1312 goto err_out;
1313 }
1314 /* rewind to the start */
1315 if (fseek(f, 0, SEEK_SET)) {
1316 err = -errno;
1317 goto err_out;
1318 }
1319
1320 /* pre-alloc memory and read all of BTF data */
1321 data = malloc(sz);
1322 if (!data) {
1323 err = -ENOMEM;
1324 goto err_out;
1325 }
1326 if (fread(data, 1, sz, f) < sz) {
1327 err = -EIO;
1328 goto err_out;
1329 }
1330
1331 /* finally parse BTF data */
1332 btf = btf_new(data, sz, base_btf);
1333
1334err_out:
1335 free(data);
1336 if (f)
1337 fclose(f);
1338 return err ? ERR_PTR(err) : btf;
1339}
1340
1341struct btf *btf__parse_raw(const char *path)
1342{
1343 return libbpf_ptr(btf_parse_raw(path, NULL));
1344}
1345
1346struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
1347{
1348 return libbpf_ptr(btf_parse_raw(path, base_btf));
1349}
1350
1351static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
1352{
1353 struct btf *btf;
1354 int err;
1355
1356 if (btf_ext)
1357 *btf_ext = NULL;
1358
1359 btf = btf_parse_raw(path, base_btf);
1360 err = libbpf_get_error(btf);
1361 if (!err)
1362 return btf;
1363 if (err != -EPROTO)
1364 return ERR_PTR(err);
1365 return btf_parse_elf(path, base_btf, btf_ext);
1366}
1367
1368struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
1369{
1370 return libbpf_ptr(btf_parse(path, NULL, btf_ext));
1371}
1372
1373struct btf *btf__parse_split(const char *path, struct btf *base_btf)
1374{
1375 return libbpf_ptr(btf_parse(path, base_btf, NULL));
1376}
1377
1378static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian);
1379
1380int btf_load_into_kernel(struct btf *btf,
1381 char *log_buf, size_t log_sz, __u32 log_level,
1382 int token_fd)
1383{
1384 LIBBPF_OPTS(bpf_btf_load_opts, opts);
1385 __u32 buf_sz = 0, raw_size;
1386 char *buf = NULL, *tmp;
1387 void *raw_data;
1388 int err = 0;
1389
1390 if (btf->fd >= 0)
1391 return libbpf_err(-EEXIST);
1392 if (log_sz && !log_buf)
1393 return libbpf_err(-EINVAL);
1394
1395 /* cache native raw data representation */
1396 raw_data = btf_get_raw_data(btf, &raw_size, false);
1397 if (!raw_data) {
1398 err = -ENOMEM;
1399 goto done;
1400 }
1401 btf->raw_size = raw_size;
1402 btf->raw_data = raw_data;
1403
1404retry_load:
1405 /* if log_level is 0, we won't provide log_buf/log_size to the kernel,
1406 * initially. Only if BTF loading fails, we bump log_level to 1 and
1407 * retry, using either auto-allocated or custom log_buf. This way
1408 * non-NULL custom log_buf provides a buffer just in case, but hopes
1409 * for successful load and no need for log_buf.
1410 */
1411 if (log_level) {
1412 /* if caller didn't provide custom log_buf, we'll keep
1413 * allocating our own progressively bigger buffers for BTF
1414 * verification log
1415 */
1416 if (!log_buf) {
1417 buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2);
1418 tmp = realloc(buf, buf_sz);
1419 if (!tmp) {
1420 err = -ENOMEM;
1421 goto done;
1422 }
1423 buf = tmp;
1424 buf[0] = '\0';
1425 }
1426
1427 opts.log_buf = log_buf ? log_buf : buf;
1428 opts.log_size = log_buf ? log_sz : buf_sz;
1429 opts.log_level = log_level;
1430 }
1431
1432 opts.token_fd = token_fd;
1433 if (token_fd)
1434 opts.btf_flags |= BPF_F_TOKEN_FD;
1435
1436 btf->fd = bpf_btf_load(raw_data, raw_size, &opts);
1437 if (btf->fd < 0) {
1438 /* time to turn on verbose mode and try again */
1439 if (log_level == 0) {
1440 log_level = 1;
1441 goto retry_load;
1442 }
1443 /* only retry if caller didn't provide custom log_buf, but
1444 * make sure we can never overflow buf_sz
1445 */
1446 if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2)
1447 goto retry_load;
1448
1449 err = -errno;
1450 pr_warn("BTF loading error: %s\n", errstr(err));
1451 /* don't print out contents of custom log_buf */
1452 if (!log_buf && buf[0])
1453 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf);
1454 }
1455
1456done:
1457 free(buf);
1458 return libbpf_err(err);
1459}
1460
1461int btf__load_into_kernel(struct btf *btf)
1462{
1463 return btf_load_into_kernel(btf, NULL, 0, 0, 0);
1464}
1465
1466int btf__fd(const struct btf *btf)
1467{
1468 return btf->fd;
1469}
1470
1471void btf__set_fd(struct btf *btf, int fd)
1472{
1473 btf->fd = fd;
1474}
1475
1476static const void *btf_strs_data(const struct btf *btf)
1477{
1478 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set);
1479}
1480
1481static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian)
1482{
1483 struct btf_header *hdr = btf->hdr;
1484 struct btf_type *t;
1485 void *data, *p;
1486 __u32 data_sz;
1487 int i;
1488
1489 data = swap_endian ? btf->raw_data_swapped : btf->raw_data;
1490 if (data) {
1491 *size = btf->raw_size;
1492 return data;
1493 }
1494
1495 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
1496 data = calloc(1, data_sz);
1497 if (!data)
1498 return NULL;
1499 p = data;
1500
1501 memcpy(p, hdr, hdr->hdr_len);
1502 if (swap_endian)
1503 btf_bswap_hdr(p);
1504 p += hdr->hdr_len;
1505
1506 memcpy(p, btf->types_data, hdr->type_len);
1507 if (swap_endian) {
1508 for (i = 0; i < btf->nr_types; i++) {
1509 t = p + btf->type_offs[i];
1510 /* btf_bswap_type_rest() relies on native t->info, so
1511 * we swap base type info after we swapped all the
1512 * additional information
1513 */
1514 if (btf_bswap_type_rest(t))
1515 goto err_out;
1516 btf_bswap_type_base(t);
1517 }
1518 }
1519 p += hdr->type_len;
1520
1521 memcpy(p, btf_strs_data(btf), hdr->str_len);
1522 p += hdr->str_len;
1523
1524 *size = data_sz;
1525 return data;
1526err_out:
1527 free(data);
1528 return NULL;
1529}
1530
1531const void *btf__raw_data(const struct btf *btf_ro, __u32 *size)
1532{
1533 struct btf *btf = (struct btf *)btf_ro;
1534 __u32 data_sz;
1535 void *data;
1536
1537 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);
1538 if (!data)
1539 return errno = ENOMEM, NULL;
1540
1541 btf->raw_size = data_sz;
1542 if (btf->swapped_endian)
1543 btf->raw_data_swapped = data;
1544 else
1545 btf->raw_data = data;
1546 *size = data_sz;
1547 return data;
1548}
1549
1550__attribute__((alias("btf__raw_data")))
1551const void *btf__get_raw_data(const struct btf *btf, __u32 *size);
1552
1553const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
1554{
1555 if (offset < btf->start_str_off)
1556 return btf__str_by_offset(btf->base_btf, offset);
1557 else if (offset - btf->start_str_off < btf->hdr->str_len)
1558 return btf_strs_data(btf) + (offset - btf->start_str_off);
1559 else
1560 return errno = EINVAL, NULL;
1561}
1562
1563const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
1564{
1565 return btf__str_by_offset(btf, offset);
1566}
1567
1568struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
1569{
1570 struct bpf_btf_info btf_info;
1571 __u32 len = sizeof(btf_info);
1572 __u32 last_size;
1573 struct btf *btf;
1574 void *ptr;
1575 int err;
1576
1577 /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so
1578 * let's start with a sane default - 4KiB here - and resize it only if
1579 * bpf_btf_get_info_by_fd() needs a bigger buffer.
1580 */
1581 last_size = 4096;
1582 ptr = malloc(last_size);
1583 if (!ptr)
1584 return ERR_PTR(-ENOMEM);
1585
1586 memset(&btf_info, 0, sizeof(btf_info));
1587 btf_info.btf = ptr_to_u64(ptr);
1588 btf_info.btf_size = last_size;
1589 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len);
1590
1591 if (!err && btf_info.btf_size > last_size) {
1592 void *temp_ptr;
1593
1594 last_size = btf_info.btf_size;
1595 temp_ptr = realloc(ptr, last_size);
1596 if (!temp_ptr) {
1597 btf = ERR_PTR(-ENOMEM);
1598 goto exit_free;
1599 }
1600 ptr = temp_ptr;
1601
1602 len = sizeof(btf_info);
1603 memset(&btf_info, 0, sizeof(btf_info));
1604 btf_info.btf = ptr_to_u64(ptr);
1605 btf_info.btf_size = last_size;
1606
1607 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len);
1608 }
1609
1610 if (err || btf_info.btf_size > last_size) {
1611 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
1612 goto exit_free;
1613 }
1614
1615 btf = btf_new(ptr, btf_info.btf_size, base_btf);
1616
1617exit_free:
1618 free(ptr);
1619 return btf;
1620}
1621
1622struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
1623{
1624 struct btf *btf;
1625 int btf_fd;
1626
1627 btf_fd = bpf_btf_get_fd_by_id(id);
1628 if (btf_fd < 0)
1629 return libbpf_err_ptr(-errno);
1630
1631 btf = btf_get_from_fd(btf_fd, base_btf);
1632 close(btf_fd);
1633
1634 return libbpf_ptr(btf);
1635}
1636
1637struct btf *btf__load_from_kernel_by_id(__u32 id)
1638{
1639 return btf__load_from_kernel_by_id_split(id, NULL);
1640}
1641
1642static void btf_invalidate_raw_data(struct btf *btf)
1643{
1644 if (btf->raw_data) {
1645 free(btf->raw_data);
1646 btf->raw_data = NULL;
1647 }
1648 if (btf->raw_data_swapped) {
1649 free(btf->raw_data_swapped);
1650 btf->raw_data_swapped = NULL;
1651 }
1652}
1653
1654/* Ensure BTF is ready to be modified (by splitting into a three memory
1655 * regions for header, types, and strings). Also invalidate cached
1656 * raw_data, if any.
1657 */
1658static int btf_ensure_modifiable(struct btf *btf)
1659{
1660 void *hdr, *types;
1661 struct strset *set = NULL;
1662 int err = -ENOMEM;
1663
1664 if (btf_is_modifiable(btf)) {
1665 /* any BTF modification invalidates raw_data */
1666 btf_invalidate_raw_data(btf);
1667 return 0;
1668 }
1669
1670 /* split raw data into three memory regions */
1671 hdr = malloc(btf->hdr->hdr_len);
1672 types = malloc(btf->hdr->type_len);
1673 if (!hdr || !types)
1674 goto err_out;
1675
1676 memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
1677 memcpy(types, btf->types_data, btf->hdr->type_len);
1678
1679 /* build lookup index for all strings */
1680 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);
1681 if (IS_ERR(set)) {
1682 err = PTR_ERR(set);
1683 goto err_out;
1684 }
1685
1686 /* only when everything was successful, update internal state */
1687 btf->hdr = hdr;
1688 btf->types_data = types;
1689 btf->types_data_cap = btf->hdr->type_len;
1690 btf->strs_data = NULL;
1691 btf->strs_set = set;
1692 /* if BTF was created from scratch, all strings are guaranteed to be
1693 * unique and deduplicated
1694 */
1695 if (btf->hdr->str_len == 0)
1696 btf->strs_deduped = true;
1697 if (!btf->base_btf && btf->hdr->str_len == 1)
1698 btf->strs_deduped = true;
1699
1700 /* invalidate raw_data representation */
1701 btf_invalidate_raw_data(btf);
1702
1703 return 0;
1704
1705err_out:
1706 strset__free(set);
1707 free(hdr);
1708 free(types);
1709 return err;
1710}
1711
1712/* Find an offset in BTF string section that corresponds to a given string *s*.
1713 * Returns:
1714 * - >0 offset into string section, if string is found;
1715 * - -ENOENT, if string is not in the string section;
1716 * - <0, on any other error.
1717 */
1718int btf__find_str(struct btf *btf, const char *s)
1719{
1720 int off;
1721
1722 if (btf->base_btf) {
1723 off = btf__find_str(btf->base_btf, s);
1724 if (off != -ENOENT)
1725 return off;
1726 }
1727
1728 /* BTF needs to be in a modifiable state to build string lookup index */
1729 if (btf_ensure_modifiable(btf))
1730 return libbpf_err(-ENOMEM);
1731
1732 off = strset__find_str(btf->strs_set, s);
1733 if (off < 0)
1734 return libbpf_err(off);
1735
1736 return btf->start_str_off + off;
1737}
1738
1739/* Add a string s to the BTF string section.
1740 * Returns:
1741 * - > 0 offset into string section, on success;
1742 * - < 0, on error.
1743 */
1744int btf__add_str(struct btf *btf, const char *s)
1745{
1746 int off;
1747
1748 if (btf->base_btf) {
1749 off = btf__find_str(btf->base_btf, s);
1750 if (off != -ENOENT)
1751 return off;
1752 }
1753
1754 if (btf_ensure_modifiable(btf))
1755 return libbpf_err(-ENOMEM);
1756
1757 off = strset__add_str(btf->strs_set, s);
1758 if (off < 0)
1759 return libbpf_err(off);
1760
1761 btf->hdr->str_len = strset__data_size(btf->strs_set);
1762
1763 return btf->start_str_off + off;
1764}
1765
1766static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
1767{
1768 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1,
1769 btf->hdr->type_len, UINT_MAX, add_sz);
1770}
1771
1772static void btf_type_inc_vlen(struct btf_type *t)
1773{
1774 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
1775}
1776
1777static int btf_commit_type(struct btf *btf, int data_sz)
1778{
1779 int err;
1780
1781 err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
1782 if (err)
1783 return libbpf_err(err);
1784
1785 btf->hdr->type_len += data_sz;
1786 btf->hdr->str_off += data_sz;
1787 btf->nr_types++;
1788 return btf->start_id + btf->nr_types - 1;
1789}
1790
1791struct btf_pipe {
1792 const struct btf *src;
1793 struct btf *dst;
1794 struct hashmap *str_off_map; /* map string offsets from src to dst */
1795};
1796
1797static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off)
1798{
1799 long mapped_off;
1800 int off, err;
1801
1802 if (!*str_off) /* nothing to do for empty strings */
1803 return 0;
1804
1805 if (p->str_off_map &&
1806 hashmap__find(p->str_off_map, *str_off, &mapped_off)) {
1807 *str_off = mapped_off;
1808 return 0;
1809 }
1810
1811 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off));
1812 if (off < 0)
1813 return off;
1814
1815 /* Remember string mapping from src to dst. It avoids
1816 * performing expensive string comparisons.
1817 */
1818 if (p->str_off_map) {
1819 err = hashmap__append(p->str_off_map, *str_off, off);
1820 if (err)
1821 return err;
1822 }
1823
1824 *str_off = off;
1825 return 0;
1826}
1827
1828static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type)
1829{
1830 struct btf_field_iter it;
1831 struct btf_type *t;
1832 __u32 *str_off;
1833 int sz, err;
1834
1835 sz = btf_type_size(src_type);
1836 if (sz < 0)
1837 return libbpf_err(sz);
1838
1839 /* deconstruct BTF, if necessary, and invalidate raw_data */
1840 if (btf_ensure_modifiable(p->dst))
1841 return libbpf_err(-ENOMEM);
1842
1843 t = btf_add_type_mem(p->dst, sz);
1844 if (!t)
1845 return libbpf_err(-ENOMEM);
1846
1847 memcpy(t, src_type, sz);
1848
1849 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1850 if (err)
1851 return libbpf_err(err);
1852
1853 while ((str_off = btf_field_iter_next(&it))) {
1854 err = btf_rewrite_str(p, str_off);
1855 if (err)
1856 return libbpf_err(err);
1857 }
1858
1859 return btf_commit_type(p->dst, sz);
1860}
1861
1862int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type)
1863{
1864 struct btf_pipe p = { .src = src_btf, .dst = btf };
1865
1866 return btf_add_type(&p, src_type);
1867}
1868
1869static size_t btf_dedup_identity_hash_fn(long key, void *ctx);
1870static bool btf_dedup_equal_fn(long k1, long k2, void *ctx);
1871
1872int btf__add_btf(struct btf *btf, const struct btf *src_btf)
1873{
1874 struct btf_pipe p = { .src = src_btf, .dst = btf };
1875 int data_sz, sz, cnt, i, err, old_strs_len;
1876 __u32 *off;
1877 void *t;
1878
1879 /* appending split BTF isn't supported yet */
1880 if (src_btf->base_btf)
1881 return libbpf_err(-ENOTSUP);
1882
1883 /* deconstruct BTF, if necessary, and invalidate raw_data */
1884 if (btf_ensure_modifiable(btf))
1885 return libbpf_err(-ENOMEM);
1886
1887 /* remember original strings section size if we have to roll back
1888 * partial strings section changes
1889 */
1890 old_strs_len = btf->hdr->str_len;
1891
1892 data_sz = src_btf->hdr->type_len;
1893 cnt = btf__type_cnt(src_btf) - 1;
1894
1895 /* pre-allocate enough memory for new types */
1896 t = btf_add_type_mem(btf, data_sz);
1897 if (!t)
1898 return libbpf_err(-ENOMEM);
1899
1900 /* pre-allocate enough memory for type offset index for new types */
1901 off = btf_add_type_offs_mem(btf, cnt);
1902 if (!off)
1903 return libbpf_err(-ENOMEM);
1904
1905 /* Map the string offsets from src_btf to the offsets from btf to improve performance */
1906 p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
1907 if (IS_ERR(p.str_off_map))
1908 return libbpf_err(-ENOMEM);
1909
1910 /* bulk copy types data for all types from src_btf */
1911 memcpy(t, src_btf->types_data, data_sz);
1912
1913 for (i = 0; i < cnt; i++) {
1914 struct btf_field_iter it;
1915 __u32 *type_id, *str_off;
1916
1917 sz = btf_type_size(t);
1918 if (sz < 0) {
1919 /* unlikely, has to be corrupted src_btf */
1920 err = sz;
1921 goto err_out;
1922 }
1923
1924 /* fill out type ID to type offset mapping for lookups by type ID */
1925 *off = t - btf->types_data;
1926
1927 /* add, dedup, and remap strings referenced by this BTF type */
1928 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1929 if (err)
1930 goto err_out;
1931 while ((str_off = btf_field_iter_next(&it))) {
1932 err = btf_rewrite_str(&p, str_off);
1933 if (err)
1934 goto err_out;
1935 }
1936
1937 /* remap all type IDs referenced from this BTF type */
1938 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
1939 if (err)
1940 goto err_out;
1941
1942 while ((type_id = btf_field_iter_next(&it))) {
1943 if (!*type_id) /* nothing to do for VOID references */
1944 continue;
1945
1946 /* we haven't updated btf's type count yet, so
1947 * btf->start_id + btf->nr_types - 1 is the type ID offset we should
1948 * add to all newly added BTF types
1949 */
1950 *type_id += btf->start_id + btf->nr_types - 1;
1951 }
1952
1953 /* go to next type data and type offset index entry */
1954 t += sz;
1955 off++;
1956 }
1957
1958 /* Up until now any of the copied type data was effectively invisible,
1959 * so if we exited early before this point due to error, BTF would be
1960 * effectively unmodified. There would be extra internal memory
1961 * pre-allocated, but it would not be available for querying. But now
1962 * that we've copied and rewritten all the data successfully, we can
1963 * update type count and various internal offsets and sizes to
1964 * "commit" the changes and made them visible to the outside world.
1965 */
1966 btf->hdr->type_len += data_sz;
1967 btf->hdr->str_off += data_sz;
1968 btf->nr_types += cnt;
1969
1970 hashmap__free(p.str_off_map);
1971
1972 /* return type ID of the first added BTF type */
1973 return btf->start_id + btf->nr_types - cnt;
1974err_out:
1975 /* zero out preallocated memory as if it was just allocated with
1976 * libbpf_add_mem()
1977 */
1978 memset(btf->types_data + btf->hdr->type_len, 0, data_sz);
1979 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len);
1980
1981 /* and now restore original strings section size; types data size
1982 * wasn't modified, so doesn't need restoring, see big comment above
1983 */
1984 btf->hdr->str_len = old_strs_len;
1985
1986 hashmap__free(p.str_off_map);
1987
1988 return libbpf_err(err);
1989}
1990
1991/*
1992 * Append new BTF_KIND_INT type with:
1993 * - *name* - non-empty, non-NULL type name;
1994 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
1995 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
1996 * Returns:
1997 * - >0, type ID of newly added BTF type;
1998 * - <0, on error.
1999 */
2000int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
2001{
2002 struct btf_type *t;
2003 int sz, name_off;
2004
2005 /* non-empty name */
2006 if (!name || !name[0])
2007 return libbpf_err(-EINVAL);
2008 /* byte_sz must be power of 2 */
2009 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16)
2010 return libbpf_err(-EINVAL);
2011 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL))
2012 return libbpf_err(-EINVAL);
2013
2014 /* deconstruct BTF, if necessary, and invalidate raw_data */
2015 if (btf_ensure_modifiable(btf))
2016 return libbpf_err(-ENOMEM);
2017
2018 sz = sizeof(struct btf_type) + sizeof(int);
2019 t = btf_add_type_mem(btf, sz);
2020 if (!t)
2021 return libbpf_err(-ENOMEM);
2022
2023 /* if something goes wrong later, we might end up with an extra string,
2024 * but that shouldn't be a problem, because BTF can't be constructed
2025 * completely anyway and will most probably be just discarded
2026 */
2027 name_off = btf__add_str(btf, name);
2028 if (name_off < 0)
2029 return name_off;
2030
2031 t->name_off = name_off;
2032 t->info = btf_type_info(BTF_KIND_INT, 0, 0);
2033 t->size = byte_sz;
2034 /* set INT info, we don't allow setting legacy bit offset/size */
2035 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
2036
2037 return btf_commit_type(btf, sz);
2038}
2039
2040/*
2041 * Append new BTF_KIND_FLOAT type with:
2042 * - *name* - non-empty, non-NULL type name;
2043 * - *sz* - size of the type, in bytes;
2044 * Returns:
2045 * - >0, type ID of newly added BTF type;
2046 * - <0, on error.
2047 */
2048int btf__add_float(struct btf *btf, const char *name, size_t byte_sz)
2049{
2050 struct btf_type *t;
2051 int sz, name_off;
2052
2053 /* non-empty name */
2054 if (!name || !name[0])
2055 return libbpf_err(-EINVAL);
2056
2057 /* byte_sz must be one of the explicitly allowed values */
2058 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 &&
2059 byte_sz != 16)
2060 return libbpf_err(-EINVAL);
2061
2062 if (btf_ensure_modifiable(btf))
2063 return libbpf_err(-ENOMEM);
2064
2065 sz = sizeof(struct btf_type);
2066 t = btf_add_type_mem(btf, sz);
2067 if (!t)
2068 return libbpf_err(-ENOMEM);
2069
2070 name_off = btf__add_str(btf, name);
2071 if (name_off < 0)
2072 return name_off;
2073
2074 t->name_off = name_off;
2075 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0);
2076 t->size = byte_sz;
2077
2078 return btf_commit_type(btf, sz);
2079}
2080
2081/* it's completely legal to append BTF types with type IDs pointing forward to
2082 * types that haven't been appended yet, so we only make sure that id looks
2083 * sane, we can't guarantee that ID will always be valid
2084 */
2085static int validate_type_id(int id)
2086{
2087 if (id < 0 || id > BTF_MAX_NR_TYPES)
2088 return -EINVAL;
2089 return 0;
2090}
2091
2092/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
2093static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
2094{
2095 struct btf_type *t;
2096 int sz, name_off = 0;
2097
2098 if (validate_type_id(ref_type_id))
2099 return libbpf_err(-EINVAL);
2100
2101 if (btf_ensure_modifiable(btf))
2102 return libbpf_err(-ENOMEM);
2103
2104 sz = sizeof(struct btf_type);
2105 t = btf_add_type_mem(btf, sz);
2106 if (!t)
2107 return libbpf_err(-ENOMEM);
2108
2109 if (name && name[0]) {
2110 name_off = btf__add_str(btf, name);
2111 if (name_off < 0)
2112 return name_off;
2113 }
2114
2115 t->name_off = name_off;
2116 t->info = btf_type_info(kind, 0, 0);
2117 t->type = ref_type_id;
2118
2119 return btf_commit_type(btf, sz);
2120}
2121
2122/*
2123 * Append new BTF_KIND_PTR type with:
2124 * - *ref_type_id* - referenced type ID, it might not exist yet;
2125 * Returns:
2126 * - >0, type ID of newly added BTF type;
2127 * - <0, on error.
2128 */
2129int btf__add_ptr(struct btf *btf, int ref_type_id)
2130{
2131 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
2132}
2133
2134/*
2135 * Append new BTF_KIND_ARRAY type with:
2136 * - *index_type_id* - type ID of the type describing array index;
2137 * - *elem_type_id* - type ID of the type describing array element;
2138 * - *nr_elems* - the size of the array;
2139 * Returns:
2140 * - >0, type ID of newly added BTF type;
2141 * - <0, on error.
2142 */
2143int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems)
2144{
2145 struct btf_type *t;
2146 struct btf_array *a;
2147 int sz;
2148
2149 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
2150 return libbpf_err(-EINVAL);
2151
2152 if (btf_ensure_modifiable(btf))
2153 return libbpf_err(-ENOMEM);
2154
2155 sz = sizeof(struct btf_type) + sizeof(struct btf_array);
2156 t = btf_add_type_mem(btf, sz);
2157 if (!t)
2158 return libbpf_err(-ENOMEM);
2159
2160 t->name_off = 0;
2161 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);
2162 t->size = 0;
2163
2164 a = btf_array(t);
2165 a->type = elem_type_id;
2166 a->index_type = index_type_id;
2167 a->nelems = nr_elems;
2168
2169 return btf_commit_type(btf, sz);
2170}
2171
2172/* generic STRUCT/UNION append function */
2173static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
2174{
2175 struct btf_type *t;
2176 int sz, name_off = 0;
2177
2178 if (btf_ensure_modifiable(btf))
2179 return libbpf_err(-ENOMEM);
2180
2181 sz = sizeof(struct btf_type);
2182 t = btf_add_type_mem(btf, sz);
2183 if (!t)
2184 return libbpf_err(-ENOMEM);
2185
2186 if (name && name[0]) {
2187 name_off = btf__add_str(btf, name);
2188 if (name_off < 0)
2189 return name_off;
2190 }
2191
2192 /* start out with vlen=0 and no kflag; this will be adjusted when
2193 * adding each member
2194 */
2195 t->name_off = name_off;
2196 t->info = btf_type_info(kind, 0, 0);
2197 t->size = bytes_sz;
2198
2199 return btf_commit_type(btf, sz);
2200}
2201
2202/*
2203 * Append new BTF_KIND_STRUCT type with:
2204 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
2205 * - *byte_sz* - size of the struct, in bytes;
2206 *
2207 * Struct initially has no fields in it. Fields can be added by
2208 * btf__add_field() right after btf__add_struct() succeeds.
2209 *
2210 * Returns:
2211 * - >0, type ID of newly added BTF type;
2212 * - <0, on error.
2213 */
2214int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz)
2215{
2216 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz);
2217}
2218
2219/*
2220 * Append new BTF_KIND_UNION type with:
2221 * - *name* - name of the union, can be NULL or empty for anonymous union;
2222 * - *byte_sz* - size of the union, in bytes;
2223 *
2224 * Union initially has no fields in it. Fields can be added by
2225 * btf__add_field() right after btf__add_union() succeeds. All fields
2226 * should have *bit_offset* of 0.
2227 *
2228 * Returns:
2229 * - >0, type ID of newly added BTF type;
2230 * - <0, on error.
2231 */
2232int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
2233{
2234 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
2235}
2236
2237static struct btf_type *btf_last_type(struct btf *btf)
2238{
2239 return btf_type_by_id(btf, btf__type_cnt(btf) - 1);
2240}
2241
2242/*
2243 * Append new field for the current STRUCT/UNION type with:
2244 * - *name* - name of the field, can be NULL or empty for anonymous field;
2245 * - *type_id* - type ID for the type describing field type;
2246 * - *bit_offset* - bit offset of the start of the field within struct/union;
2247 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
2248 * Returns:
2249 * - 0, on success;
2250 * - <0, on error.
2251 */
2252int btf__add_field(struct btf *btf, const char *name, int type_id,
2253 __u32 bit_offset, __u32 bit_size)
2254{
2255 struct btf_type *t;
2256 struct btf_member *m;
2257 bool is_bitfield;
2258 int sz, name_off = 0;
2259
2260 /* last type should be union/struct */
2261 if (btf->nr_types == 0)
2262 return libbpf_err(-EINVAL);
2263 t = btf_last_type(btf);
2264 if (!btf_is_composite(t))
2265 return libbpf_err(-EINVAL);
2266
2267 if (validate_type_id(type_id))
2268 return libbpf_err(-EINVAL);
2269 /* best-effort bit field offset/size enforcement */
2270 is_bitfield = bit_size || (bit_offset % 8 != 0);
2271 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff))
2272 return libbpf_err(-EINVAL);
2273
2274 /* only offset 0 is allowed for unions */
2275 if (btf_is_union(t) && bit_offset)
2276 return libbpf_err(-EINVAL);
2277
2278 /* decompose and invalidate raw data */
2279 if (btf_ensure_modifiable(btf))
2280 return libbpf_err(-ENOMEM);
2281
2282 sz = sizeof(struct btf_member);
2283 m = btf_add_type_mem(btf, sz);
2284 if (!m)
2285 return libbpf_err(-ENOMEM);
2286
2287 if (name && name[0]) {
2288 name_off = btf__add_str(btf, name);
2289 if (name_off < 0)
2290 return name_off;
2291 }
2292
2293 m->name_off = name_off;
2294 m->type = type_id;
2295 m->offset = bit_offset | (bit_size << 24);
2296
2297 /* btf_add_type_mem can invalidate t pointer */
2298 t = btf_last_type(btf);
2299 /* update parent type's vlen and kflag */
2300 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
2301
2302 btf->hdr->type_len += sz;
2303 btf->hdr->str_off += sz;
2304 return 0;
2305}
2306
2307static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz,
2308 bool is_signed, __u8 kind)
2309{
2310 struct btf_type *t;
2311 int sz, name_off = 0;
2312
2313 /* byte_sz must be power of 2 */
2314 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
2315 return libbpf_err(-EINVAL);
2316
2317 if (btf_ensure_modifiable(btf))
2318 return libbpf_err(-ENOMEM);
2319
2320 sz = sizeof(struct btf_type);
2321 t = btf_add_type_mem(btf, sz);
2322 if (!t)
2323 return libbpf_err(-ENOMEM);
2324
2325 if (name && name[0]) {
2326 name_off = btf__add_str(btf, name);
2327 if (name_off < 0)
2328 return name_off;
2329 }
2330
2331 /* start out with vlen=0; it will be adjusted when adding enum values */
2332 t->name_off = name_off;
2333 t->info = btf_type_info(kind, 0, is_signed);
2334 t->size = byte_sz;
2335
2336 return btf_commit_type(btf, sz);
2337}
2338
2339/*
2340 * Append new BTF_KIND_ENUM type with:
2341 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2342 * - *byte_sz* - size of the enum, in bytes.
2343 *
2344 * Enum initially has no enum values in it (and corresponds to enum forward
2345 * declaration). Enumerator values can be added by btf__add_enum_value()
2346 * immediately after btf__add_enum() succeeds.
2347 *
2348 * Returns:
2349 * - >0, type ID of newly added BTF type;
2350 * - <0, on error.
2351 */
2352int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
2353{
2354 /*
2355 * set the signedness to be unsigned, it will change to signed
2356 * if any later enumerator is negative.
2357 */
2358 return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM);
2359}
2360
2361/*
2362 * Append new enum value for the current ENUM type with:
2363 * - *name* - name of the enumerator value, can't be NULL or empty;
2364 * - *value* - integer value corresponding to enum value *name*;
2365 * Returns:
2366 * - 0, on success;
2367 * - <0, on error.
2368 */
2369int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
2370{
2371 struct btf_type *t;
2372 struct btf_enum *v;
2373 int sz, name_off;
2374
2375 /* last type should be BTF_KIND_ENUM */
2376 if (btf->nr_types == 0)
2377 return libbpf_err(-EINVAL);
2378 t = btf_last_type(btf);
2379 if (!btf_is_enum(t))
2380 return libbpf_err(-EINVAL);
2381
2382 /* non-empty name */
2383 if (!name || !name[0])
2384 return libbpf_err(-EINVAL);
2385 if (value < INT_MIN || value > UINT_MAX)
2386 return libbpf_err(-E2BIG);
2387
2388 /* decompose and invalidate raw data */
2389 if (btf_ensure_modifiable(btf))
2390 return libbpf_err(-ENOMEM);
2391
2392 sz = sizeof(struct btf_enum);
2393 v = btf_add_type_mem(btf, sz);
2394 if (!v)
2395 return libbpf_err(-ENOMEM);
2396
2397 name_off = btf__add_str(btf, name);
2398 if (name_off < 0)
2399 return name_off;
2400
2401 v->name_off = name_off;
2402 v->val = value;
2403
2404 /* update parent type's vlen */
2405 t = btf_last_type(btf);
2406 btf_type_inc_vlen(t);
2407
2408 /* if negative value, set signedness to signed */
2409 if (value < 0)
2410 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true);
2411
2412 btf->hdr->type_len += sz;
2413 btf->hdr->str_off += sz;
2414 return 0;
2415}
2416
2417/*
2418 * Append new BTF_KIND_ENUM64 type with:
2419 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2420 * - *byte_sz* - size of the enum, in bytes.
2421 * - *is_signed* - whether the enum values are signed or not;
2422 *
2423 * Enum initially has no enum values in it (and corresponds to enum forward
2424 * declaration). Enumerator values can be added by btf__add_enum64_value()
2425 * immediately after btf__add_enum64() succeeds.
2426 *
2427 * Returns:
2428 * - >0, type ID of newly added BTF type;
2429 * - <0, on error.
2430 */
2431int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz,
2432 bool is_signed)
2433{
2434 return btf_add_enum_common(btf, name, byte_sz, is_signed,
2435 BTF_KIND_ENUM64);
2436}
2437
2438/*
2439 * Append new enum value for the current ENUM64 type with:
2440 * - *name* - name of the enumerator value, can't be NULL or empty;
2441 * - *value* - integer value corresponding to enum value *name*;
2442 * Returns:
2443 * - 0, on success;
2444 * - <0, on error.
2445 */
2446int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value)
2447{
2448 struct btf_enum64 *v;
2449 struct btf_type *t;
2450 int sz, name_off;
2451
2452 /* last type should be BTF_KIND_ENUM64 */
2453 if (btf->nr_types == 0)
2454 return libbpf_err(-EINVAL);
2455 t = btf_last_type(btf);
2456 if (!btf_is_enum64(t))
2457 return libbpf_err(-EINVAL);
2458
2459 /* non-empty name */
2460 if (!name || !name[0])
2461 return libbpf_err(-EINVAL);
2462
2463 /* decompose and invalidate raw data */
2464 if (btf_ensure_modifiable(btf))
2465 return libbpf_err(-ENOMEM);
2466
2467 sz = sizeof(struct btf_enum64);
2468 v = btf_add_type_mem(btf, sz);
2469 if (!v)
2470 return libbpf_err(-ENOMEM);
2471
2472 name_off = btf__add_str(btf, name);
2473 if (name_off < 0)
2474 return name_off;
2475
2476 v->name_off = name_off;
2477 v->val_lo32 = (__u32)value;
2478 v->val_hi32 = value >> 32;
2479
2480 /* update parent type's vlen */
2481 t = btf_last_type(btf);
2482 btf_type_inc_vlen(t);
2483
2484 btf->hdr->type_len += sz;
2485 btf->hdr->str_off += sz;
2486 return 0;
2487}
2488
2489/*
2490 * Append new BTF_KIND_FWD type with:
2491 * - *name*, non-empty/non-NULL name;
2492 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2493 * BTF_FWD_UNION, or BTF_FWD_ENUM;
2494 * Returns:
2495 * - >0, type ID of newly added BTF type;
2496 * - <0, on error.
2497 */
2498int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind)
2499{
2500 if (!name || !name[0])
2501 return libbpf_err(-EINVAL);
2502
2503 switch (fwd_kind) {
2504 case BTF_FWD_STRUCT:
2505 case BTF_FWD_UNION: {
2506 struct btf_type *t;
2507 int id;
2508
2509 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
2510 if (id <= 0)
2511 return id;
2512 t = btf_type_by_id(btf, id);
2513 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);
2514 return id;
2515 }
2516 case BTF_FWD_ENUM:
2517 /* enum forward in BTF currently is just an enum with no enum
2518 * values; we also assume a standard 4-byte size for it
2519 */
2520 return btf__add_enum(btf, name, sizeof(int));
2521 default:
2522 return libbpf_err(-EINVAL);
2523 }
2524}
2525
2526/*
2527 * Append new BTF_KING_TYPEDEF type with:
2528 * - *name*, non-empty/non-NULL name;
2529 * - *ref_type_id* - referenced type ID, it might not exist yet;
2530 * Returns:
2531 * - >0, type ID of newly added BTF type;
2532 * - <0, on error.
2533 */
2534int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
2535{
2536 if (!name || !name[0])
2537 return libbpf_err(-EINVAL);
2538
2539 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
2540}
2541
2542/*
2543 * Append new BTF_KIND_VOLATILE type with:
2544 * - *ref_type_id* - referenced type ID, it might not exist yet;
2545 * Returns:
2546 * - >0, type ID of newly added BTF type;
2547 * - <0, on error.
2548 */
2549int btf__add_volatile(struct btf *btf, int ref_type_id)
2550{
2551 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
2552}
2553
2554/*
2555 * Append new BTF_KIND_CONST type with:
2556 * - *ref_type_id* - referenced type ID, it might not exist yet;
2557 * Returns:
2558 * - >0, type ID of newly added BTF type;
2559 * - <0, on error.
2560 */
2561int btf__add_const(struct btf *btf, int ref_type_id)
2562{
2563 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
2564}
2565
2566/*
2567 * Append new BTF_KIND_RESTRICT type with:
2568 * - *ref_type_id* - referenced type ID, it might not exist yet;
2569 * Returns:
2570 * - >0, type ID of newly added BTF type;
2571 * - <0, on error.
2572 */
2573int btf__add_restrict(struct btf *btf, int ref_type_id)
2574{
2575 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
2576}
2577
2578/*
2579 * Append new BTF_KIND_TYPE_TAG type with:
2580 * - *value*, non-empty/non-NULL tag value;
2581 * - *ref_type_id* - referenced type ID, it might not exist yet;
2582 * Returns:
2583 * - >0, type ID of newly added BTF type;
2584 * - <0, on error.
2585 */
2586int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id)
2587{
2588 if (!value || !value[0])
2589 return libbpf_err(-EINVAL);
2590
2591 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id);
2592}
2593
2594/*
2595 * Append new BTF_KIND_FUNC type with:
2596 * - *name*, non-empty/non-NULL name;
2597 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2598 * Returns:
2599 * - >0, type ID of newly added BTF type;
2600 * - <0, on error.
2601 */
2602int btf__add_func(struct btf *btf, const char *name,
2603 enum btf_func_linkage linkage, int proto_type_id)
2604{
2605 int id;
2606
2607 if (!name || !name[0])
2608 return libbpf_err(-EINVAL);
2609 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL &&
2610 linkage != BTF_FUNC_EXTERN)
2611 return libbpf_err(-EINVAL);
2612
2613 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
2614 if (id > 0) {
2615 struct btf_type *t = btf_type_by_id(btf, id);
2616
2617 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);
2618 }
2619 return libbpf_err(id);
2620}
2621
2622/*
2623 * Append new BTF_KIND_FUNC_PROTO with:
2624 * - *ret_type_id* - type ID for return result of a function.
2625 *
2626 * Function prototype initially has no arguments, but they can be added by
2627 * btf__add_func_param() one by one, immediately after
2628 * btf__add_func_proto() succeeded.
2629 *
2630 * Returns:
2631 * - >0, type ID of newly added BTF type;
2632 * - <0, on error.
2633 */
2634int btf__add_func_proto(struct btf *btf, int ret_type_id)
2635{
2636 struct btf_type *t;
2637 int sz;
2638
2639 if (validate_type_id(ret_type_id))
2640 return libbpf_err(-EINVAL);
2641
2642 if (btf_ensure_modifiable(btf))
2643 return libbpf_err(-ENOMEM);
2644
2645 sz = sizeof(struct btf_type);
2646 t = btf_add_type_mem(btf, sz);
2647 if (!t)
2648 return libbpf_err(-ENOMEM);
2649
2650 /* start out with vlen=0; this will be adjusted when adding enum
2651 * values, if necessary
2652 */
2653 t->name_off = 0;
2654 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
2655 t->type = ret_type_id;
2656
2657 return btf_commit_type(btf, sz);
2658}
2659
2660/*
2661 * Append new function parameter for current FUNC_PROTO type with:
2662 * - *name* - parameter name, can be NULL or empty;
2663 * - *type_id* - type ID describing the type of the parameter.
2664 * Returns:
2665 * - 0, on success;
2666 * - <0, on error.
2667 */
2668int btf__add_func_param(struct btf *btf, const char *name, int type_id)
2669{
2670 struct btf_type *t;
2671 struct btf_param *p;
2672 int sz, name_off = 0;
2673
2674 if (validate_type_id(type_id))
2675 return libbpf_err(-EINVAL);
2676
2677 /* last type should be BTF_KIND_FUNC_PROTO */
2678 if (btf->nr_types == 0)
2679 return libbpf_err(-EINVAL);
2680 t = btf_last_type(btf);
2681 if (!btf_is_func_proto(t))
2682 return libbpf_err(-EINVAL);
2683
2684 /* decompose and invalidate raw data */
2685 if (btf_ensure_modifiable(btf))
2686 return libbpf_err(-ENOMEM);
2687
2688 sz = sizeof(struct btf_param);
2689 p = btf_add_type_mem(btf, sz);
2690 if (!p)
2691 return libbpf_err(-ENOMEM);
2692
2693 if (name && name[0]) {
2694 name_off = btf__add_str(btf, name);
2695 if (name_off < 0)
2696 return name_off;
2697 }
2698
2699 p->name_off = name_off;
2700 p->type = type_id;
2701
2702 /* update parent type's vlen */
2703 t = btf_last_type(btf);
2704 btf_type_inc_vlen(t);
2705
2706 btf->hdr->type_len += sz;
2707 btf->hdr->str_off += sz;
2708 return 0;
2709}
2710
2711/*
2712 * Append new BTF_KIND_VAR type with:
2713 * - *name* - non-empty/non-NULL name;
2714 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2715 * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN;
2716 * - *type_id* - type ID of the type describing the type of the variable.
2717 * Returns:
2718 * - >0, type ID of newly added BTF type;
2719 * - <0, on error.
2720 */
2721int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
2722{
2723 struct btf_type *t;
2724 struct btf_var *v;
2725 int sz, name_off;
2726
2727 /* non-empty name */
2728 if (!name || !name[0])
2729 return libbpf_err(-EINVAL);
2730 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2731 linkage != BTF_VAR_GLOBAL_EXTERN)
2732 return libbpf_err(-EINVAL);
2733 if (validate_type_id(type_id))
2734 return libbpf_err(-EINVAL);
2735
2736 /* deconstruct BTF, if necessary, and invalidate raw_data */
2737 if (btf_ensure_modifiable(btf))
2738 return libbpf_err(-ENOMEM);
2739
2740 sz = sizeof(struct btf_type) + sizeof(struct btf_var);
2741 t = btf_add_type_mem(btf, sz);
2742 if (!t)
2743 return libbpf_err(-ENOMEM);
2744
2745 name_off = btf__add_str(btf, name);
2746 if (name_off < 0)
2747 return name_off;
2748
2749 t->name_off = name_off;
2750 t->info = btf_type_info(BTF_KIND_VAR, 0, 0);
2751 t->type = type_id;
2752
2753 v = btf_var(t);
2754 v->linkage = linkage;
2755
2756 return btf_commit_type(btf, sz);
2757}
2758
2759/*
2760 * Append new BTF_KIND_DATASEC type with:
2761 * - *name* - non-empty/non-NULL name;
2762 * - *byte_sz* - data section size, in bytes.
2763 *
2764 * Data section is initially empty. Variables info can be added with
2765 * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds.
2766 *
2767 * Returns:
2768 * - >0, type ID of newly added BTF type;
2769 * - <0, on error.
2770 */
2771int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
2772{
2773 struct btf_type *t;
2774 int sz, name_off;
2775
2776 /* non-empty name */
2777 if (!name || !name[0])
2778 return libbpf_err(-EINVAL);
2779
2780 if (btf_ensure_modifiable(btf))
2781 return libbpf_err(-ENOMEM);
2782
2783 sz = sizeof(struct btf_type);
2784 t = btf_add_type_mem(btf, sz);
2785 if (!t)
2786 return libbpf_err(-ENOMEM);
2787
2788 name_off = btf__add_str(btf, name);
2789 if (name_off < 0)
2790 return name_off;
2791
2792 /* start with vlen=0, which will be update as var_secinfos are added */
2793 t->name_off = name_off;
2794 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
2795 t->size = byte_sz;
2796
2797 return btf_commit_type(btf, sz);
2798}
2799
2800/*
2801 * Append new data section variable information entry for current DATASEC type:
2802 * - *var_type_id* - type ID, describing type of the variable;
2803 * - *offset* - variable offset within data section, in bytes;
2804 * - *byte_sz* - variable size, in bytes.
2805 *
2806 * Returns:
2807 * - 0, on success;
2808 * - <0, on error.
2809 */
2810int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz)
2811{
2812 struct btf_type *t;
2813 struct btf_var_secinfo *v;
2814 int sz;
2815
2816 /* last type should be BTF_KIND_DATASEC */
2817 if (btf->nr_types == 0)
2818 return libbpf_err(-EINVAL);
2819 t = btf_last_type(btf);
2820 if (!btf_is_datasec(t))
2821 return libbpf_err(-EINVAL);
2822
2823 if (validate_type_id(var_type_id))
2824 return libbpf_err(-EINVAL);
2825
2826 /* decompose and invalidate raw data */
2827 if (btf_ensure_modifiable(btf))
2828 return libbpf_err(-ENOMEM);
2829
2830 sz = sizeof(struct btf_var_secinfo);
2831 v = btf_add_type_mem(btf, sz);
2832 if (!v)
2833 return libbpf_err(-ENOMEM);
2834
2835 v->type = var_type_id;
2836 v->offset = offset;
2837 v->size = byte_sz;
2838
2839 /* update parent type's vlen */
2840 t = btf_last_type(btf);
2841 btf_type_inc_vlen(t);
2842
2843 btf->hdr->type_len += sz;
2844 btf->hdr->str_off += sz;
2845 return 0;
2846}
2847
2848/*
2849 * Append new BTF_KIND_DECL_TAG type with:
2850 * - *value* - non-empty/non-NULL string;
2851 * - *ref_type_id* - referenced type ID, it might not exist yet;
2852 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
2853 * member or function argument index;
2854 * Returns:
2855 * - >0, type ID of newly added BTF type;
2856 * - <0, on error.
2857 */
2858int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
2859 int component_idx)
2860{
2861 struct btf_type *t;
2862 int sz, value_off;
2863
2864 if (!value || !value[0] || component_idx < -1)
2865 return libbpf_err(-EINVAL);
2866
2867 if (validate_type_id(ref_type_id))
2868 return libbpf_err(-EINVAL);
2869
2870 if (btf_ensure_modifiable(btf))
2871 return libbpf_err(-ENOMEM);
2872
2873 sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag);
2874 t = btf_add_type_mem(btf, sz);
2875 if (!t)
2876 return libbpf_err(-ENOMEM);
2877
2878 value_off = btf__add_str(btf, value);
2879 if (value_off < 0)
2880 return value_off;
2881
2882 t->name_off = value_off;
2883 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false);
2884 t->type = ref_type_id;
2885 btf_decl_tag(t)->component_idx = component_idx;
2886
2887 return btf_commit_type(btf, sz);
2888}
2889
2890struct btf_ext_sec_info_param {
2891 __u32 off;
2892 __u32 len;
2893 __u32 min_rec_size;
2894 struct btf_ext_info *ext_info;
2895 const char *desc;
2896};
2897
2898/*
2899 * Parse a single info subsection of the BTF.ext info data:
2900 * - validate subsection structure and elements
2901 * - save info subsection start and sizing details in struct btf_ext
2902 * - endian-independent operation, for calling before byte-swapping
2903 */
2904static int btf_ext_parse_sec_info(struct btf_ext *btf_ext,
2905 struct btf_ext_sec_info_param *ext_sec,
2906 bool is_native)
2907{
2908 const struct btf_ext_info_sec *sinfo;
2909 struct btf_ext_info *ext_info;
2910 __u32 info_left, record_size;
2911 size_t sec_cnt = 0;
2912 void *info;
2913
2914 if (ext_sec->len == 0)
2915 return 0;
2916
2917 if (ext_sec->off & 0x03) {
2918 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n",
2919 ext_sec->desc);
2920 return -EINVAL;
2921 }
2922
2923 /* The start of the info sec (including the __u32 record_size). */
2924 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
2925 info_left = ext_sec->len;
2926
2927 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {
2928 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
2929 ext_sec->desc, ext_sec->off, ext_sec->len);
2930 return -EINVAL;
2931 }
2932
2933 /* At least a record size */
2934 if (info_left < sizeof(__u32)) {
2935 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);
2936 return -EINVAL;
2937 }
2938
2939 /* The record size needs to meet either the minimum standard or, when
2940 * handling non-native endianness data, the exact standard so as
2941 * to allow safe byte-swapping.
2942 */
2943 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info);
2944 if (record_size < ext_sec->min_rec_size ||
2945 (!is_native && record_size != ext_sec->min_rec_size) ||
2946 record_size & 0x03) {
2947 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
2948 ext_sec->desc, record_size);
2949 return -EINVAL;
2950 }
2951
2952 sinfo = info + sizeof(__u32);
2953 info_left -= sizeof(__u32);
2954
2955 /* If no records, return failure now so .BTF.ext won't be used. */
2956 if (!info_left) {
2957 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc);
2958 return -EINVAL;
2959 }
2960
2961 while (info_left) {
2962 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
2963 __u64 total_record_size;
2964 __u32 num_records;
2965
2966 if (info_left < sec_hdrlen) {
2967 pr_debug("%s section header is not found in .BTF.ext\n",
2968 ext_sec->desc);
2969 return -EINVAL;
2970 }
2971
2972 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info);
2973 if (num_records == 0) {
2974 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2975 ext_sec->desc);
2976 return -EINVAL;
2977 }
2978
2979 total_record_size = sec_hdrlen + (__u64)num_records * record_size;
2980 if (info_left < total_record_size) {
2981 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2982 ext_sec->desc);
2983 return -EINVAL;
2984 }
2985
2986 info_left -= total_record_size;
2987 sinfo = (void *)sinfo + total_record_size;
2988 sec_cnt++;
2989 }
2990
2991 ext_info = ext_sec->ext_info;
2992 ext_info->len = ext_sec->len - sizeof(__u32);
2993 ext_info->rec_size = record_size;
2994 ext_info->info = info + sizeof(__u32);
2995 ext_info->sec_cnt = sec_cnt;
2996
2997 return 0;
2998}
2999
3000/* Parse all info secs in the BTF.ext info data */
3001static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
3002{
3003 struct btf_ext_sec_info_param func_info = {
3004 .off = btf_ext->hdr->func_info_off,
3005 .len = btf_ext->hdr->func_info_len,
3006 .min_rec_size = sizeof(struct bpf_func_info_min),
3007 .ext_info = &btf_ext->func_info,
3008 .desc = "func_info"
3009 };
3010 struct btf_ext_sec_info_param line_info = {
3011 .off = btf_ext->hdr->line_info_off,
3012 .len = btf_ext->hdr->line_info_len,
3013 .min_rec_size = sizeof(struct bpf_line_info_min),
3014 .ext_info = &btf_ext->line_info,
3015 .desc = "line_info",
3016 };
3017 struct btf_ext_sec_info_param core_relo = {
3018 .off = btf_ext->hdr->core_relo_off,
3019 .len = btf_ext->hdr->core_relo_len,
3020 .min_rec_size = sizeof(struct bpf_core_relo),
3021 .ext_info = &btf_ext->core_relo_info,
3022 .desc = "core_relo",
3023 };
3024 int err;
3025
3026 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native);
3027 if (err)
3028 return err;
3029
3030 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native);
3031 if (err)
3032 return err;
3033
3034 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3035 return 0; /* skip core relos parsing */
3036
3037 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
3038 if (err)
3039 return err;
3040
3041 return 0;
3042}
3043
3044/* Swap byte-order of BTF.ext header with any endianness */
3045static void btf_ext_bswap_hdr(struct btf_ext_header *h)
3046{
3047 bool is_native = h->magic == BTF_MAGIC;
3048 __u32 hdr_len;
3049
3050 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len);
3051
3052 h->magic = bswap_16(h->magic);
3053 h->hdr_len = bswap_32(h->hdr_len);
3054 h->func_info_off = bswap_32(h->func_info_off);
3055 h->func_info_len = bswap_32(h->func_info_len);
3056 h->line_info_off = bswap_32(h->line_info_off);
3057 h->line_info_len = bswap_32(h->line_info_len);
3058
3059 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3060 return;
3061
3062 h->core_relo_off = bswap_32(h->core_relo_off);
3063 h->core_relo_len = bswap_32(h->core_relo_len);
3064}
3065
3066/* Swap byte-order of generic info subsection */
3067static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native,
3068 info_rec_bswap_fn bswap_fn)
3069{
3070 struct btf_ext_info_sec *sec;
3071 __u32 info_left, rec_size, *rs;
3072
3073 if (len == 0)
3074 return;
3075
3076 rs = info; /* info record size */
3077 rec_size = is_native ? *rs : bswap_32(*rs);
3078 *rs = bswap_32(*rs);
3079
3080 sec = info + sizeof(__u32); /* info sec #1 */
3081 info_left = len - sizeof(__u32);
3082 while (info_left) {
3083 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
3084 __u32 i, num_recs;
3085 void *p;
3086
3087 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info);
3088 sec->sec_name_off = bswap_32(sec->sec_name_off);
3089 sec->num_info = bswap_32(sec->num_info);
3090 p = sec->data; /* info rec #1 */
3091 for (i = 0; i < num_recs; i++, p += rec_size)
3092 bswap_fn(p);
3093 sec = p;
3094 info_left -= sec_hdrlen + (__u64)rec_size * num_recs;
3095 }
3096}
3097
3098/*
3099 * Swap byte-order of all info data in a BTF.ext section
3100 * - requires BTF.ext hdr in native endianness
3101 */
3102static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data)
3103{
3104 const bool is_native = btf_ext->swapped_endian;
3105 const struct btf_ext_header *h = data;
3106 void *info;
3107
3108 /* Swap func_info subsection byte-order */
3109 info = data + h->hdr_len + h->func_info_off;
3110 btf_ext_bswap_info_sec(info, h->func_info_len, is_native,
3111 (info_rec_bswap_fn)bpf_func_info_bswap);
3112
3113 /* Swap line_info subsection byte-order */
3114 info = data + h->hdr_len + h->line_info_off;
3115 btf_ext_bswap_info_sec(info, h->line_info_len, is_native,
3116 (info_rec_bswap_fn)bpf_line_info_bswap);
3117
3118 /* Swap core_relo subsection byte-order (if present) */
3119 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3120 return;
3121
3122 info = data + h->hdr_len + h->core_relo_off;
3123 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native,
3124 (info_rec_bswap_fn)bpf_core_relo_bswap);
3125}
3126
3127/* Parse hdr data and info sections: check and convert to native endianness */
3128static int btf_ext_parse(struct btf_ext *btf_ext)
3129{
3130 __u32 hdr_len, data_size = btf_ext->data_size;
3131 struct btf_ext_header *hdr = btf_ext->hdr;
3132 bool swapped_endian = false;
3133 int err;
3134
3135 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) {
3136 pr_debug("BTF.ext header too short\n");
3137 return -EINVAL;
3138 }
3139
3140 hdr_len = hdr->hdr_len;
3141 if (hdr->magic == bswap_16(BTF_MAGIC)) {
3142 swapped_endian = true;
3143 hdr_len = bswap_32(hdr_len);
3144 } else if (hdr->magic != BTF_MAGIC) {
3145 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
3146 return -EINVAL;
3147 }
3148
3149 /* Ensure known version of structs, current BTF_VERSION == 1 */
3150 if (hdr->version != 1) {
3151 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
3152 return -ENOTSUP;
3153 }
3154
3155 if (hdr->flags) {
3156 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);
3157 return -ENOTSUP;
3158 }
3159
3160 if (data_size < hdr_len) {
3161 pr_debug("BTF.ext header not found\n");
3162 return -EINVAL;
3163 } else if (data_size == hdr_len) {
3164 pr_debug("BTF.ext has no data\n");
3165 return -EINVAL;
3166 }
3167
3168 /* Verify mandatory hdr info details present */
3169 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3170 pr_warn("BTF.ext header missing func_info, line_info\n");
3171 return -EINVAL;
3172 }
3173
3174 /* Keep hdr native byte-order in memory for introspection */
3175 if (swapped_endian)
3176 btf_ext_bswap_hdr(btf_ext->hdr);
3177
3178 /* Validate info subsections and cache key metadata */
3179 err = btf_ext_parse_info(btf_ext, !swapped_endian);
3180 if (err)
3181 return err;
3182
3183 /* Keep infos native byte-order in memory for introspection */
3184 if (swapped_endian)
3185 btf_ext_bswap_info(btf_ext, btf_ext->data);
3186
3187 /*
3188 * Set btf_ext->swapped_endian only after all header and info data has
3189 * been swapped, helping bswap functions determine if their data are
3190 * in native byte-order when called.
3191 */
3192 btf_ext->swapped_endian = swapped_endian;
3193 return 0;
3194}
3195
3196void btf_ext__free(struct btf_ext *btf_ext)
3197{
3198 if (IS_ERR_OR_NULL(btf_ext))
3199 return;
3200 free(btf_ext->func_info.sec_idxs);
3201 free(btf_ext->line_info.sec_idxs);
3202 free(btf_ext->core_relo_info.sec_idxs);
3203 free(btf_ext->data);
3204 free(btf_ext->data_swapped);
3205 free(btf_ext);
3206}
3207
3208struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
3209{
3210 struct btf_ext *btf_ext;
3211 int err;
3212
3213 btf_ext = calloc(1, sizeof(struct btf_ext));
3214 if (!btf_ext)
3215 return libbpf_err_ptr(-ENOMEM);
3216
3217 btf_ext->data_size = size;
3218 btf_ext->data = malloc(size);
3219 if (!btf_ext->data) {
3220 err = -ENOMEM;
3221 goto done;
3222 }
3223 memcpy(btf_ext->data, data, size);
3224
3225 err = btf_ext_parse(btf_ext);
3226
3227done:
3228 if (err) {
3229 btf_ext__free(btf_ext);
3230 return libbpf_err_ptr(err);
3231 }
3232
3233 return btf_ext;
3234}
3235
3236static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian)
3237{
3238 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro;
3239 const __u32 data_sz = btf_ext->data_size;
3240 void *data;
3241
3242 /* Return native data (always present) or swapped data if present */
3243 if (!swap_endian)
3244 return btf_ext->data;
3245 else if (btf_ext->data_swapped)
3246 return btf_ext->data_swapped;
3247
3248 /* Recreate missing swapped data, then cache and return */
3249 data = calloc(1, data_sz);
3250 if (!data)
3251 return NULL;
3252 memcpy(data, btf_ext->data, data_sz);
3253
3254 btf_ext_bswap_info(btf_ext, data);
3255 btf_ext_bswap_hdr(data);
3256 btf_ext->data_swapped = data;
3257 return data;
3258}
3259
3260const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)
3261{
3262 void *data;
3263
3264 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian);
3265 if (!data)
3266 return errno = ENOMEM, NULL;
3267
3268 *size = btf_ext->data_size;
3269 return data;
3270}
3271
3272__attribute__((alias("btf_ext__raw_data")))
3273const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);
3274
3275enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext)
3276{
3277 if (is_host_big_endian())
3278 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
3279 else
3280 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
3281}
3282
3283int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian)
3284{
3285 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
3286 return libbpf_err(-EINVAL);
3287
3288 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
3289
3290 if (!btf_ext->swapped_endian) {
3291 free(btf_ext->data_swapped);
3292 btf_ext->data_swapped = NULL;
3293 }
3294 return 0;
3295}
3296
3297struct btf_dedup;
3298
3299static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts);
3300static void btf_dedup_free(struct btf_dedup *d);
3301static int btf_dedup_prep(struct btf_dedup *d);
3302static int btf_dedup_strings(struct btf_dedup *d);
3303static int btf_dedup_prim_types(struct btf_dedup *d);
3304static int btf_dedup_struct_types(struct btf_dedup *d);
3305static int btf_dedup_ref_types(struct btf_dedup *d);
3306static int btf_dedup_resolve_fwds(struct btf_dedup *d);
3307static int btf_dedup_compact_types(struct btf_dedup *d);
3308static int btf_dedup_remap_types(struct btf_dedup *d);
3309
3310/*
3311 * Deduplicate BTF types and strings.
3312 *
3313 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF
3314 * section with all BTF type descriptors and string data. It overwrites that
3315 * memory in-place with deduplicated types and strings without any loss of
3316 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section
3317 * is provided, all the strings referenced from .BTF.ext section are honored
3318 * and updated to point to the right offsets after deduplication.
3319 *
3320 * If function returns with error, type/string data might be garbled and should
3321 * be discarded.
3322 *
3323 * More verbose and detailed description of both problem btf_dedup is solving,
3324 * as well as solution could be found at:
3325 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
3326 *
3327 * Problem description and justification
3328 * =====================================
3329 *
3330 * BTF type information is typically emitted either as a result of conversion
3331 * from DWARF to BTF or directly by compiler. In both cases, each compilation
3332 * unit contains information about a subset of all the types that are used
3333 * in an application. These subsets are frequently overlapping and contain a lot
3334 * of duplicated information when later concatenated together into a single
3335 * binary. This algorithm ensures that each unique type is represented by single
3336 * BTF type descriptor, greatly reducing resulting size of BTF data.
3337 *
3338 * Compilation unit isolation and subsequent duplication of data is not the only
3339 * problem. The same type hierarchy (e.g., struct and all the type that struct
3340 * references) in different compilation units can be represented in BTF to
3341 * various degrees of completeness (or, rather, incompleteness) due to
3342 * struct/union forward declarations.
3343 *
3344 * Let's take a look at an example, that we'll use to better understand the
3345 * problem (and solution). Suppose we have two compilation units, each using
3346 * same `struct S`, but each of them having incomplete type information about
3347 * struct's fields:
3348 *
3349 * // CU #1:
3350 * struct S;
3351 * struct A {
3352 * int a;
3353 * struct A* self;
3354 * struct S* parent;
3355 * };
3356 * struct B;
3357 * struct S {
3358 * struct A* a_ptr;
3359 * struct B* b_ptr;
3360 * };
3361 *
3362 * // CU #2:
3363 * struct S;
3364 * struct A;
3365 * struct B {
3366 * int b;
3367 * struct B* self;
3368 * struct S* parent;
3369 * };
3370 * struct S {
3371 * struct A* a_ptr;
3372 * struct B* b_ptr;
3373 * };
3374 *
3375 * In case of CU #1, BTF data will know only that `struct B` exist (but no
3376 * more), but will know the complete type information about `struct A`. While
3377 * for CU #2, it will know full type information about `struct B`, but will
3378 * only know about forward declaration of `struct A` (in BTF terms, it will
3379 * have `BTF_KIND_FWD` type descriptor with name `B`).
3380 *
3381 * This compilation unit isolation means that it's possible that there is no
3382 * single CU with complete type information describing structs `S`, `A`, and
3383 * `B`. Also, we might get tons of duplicated and redundant type information.
3384 *
3385 * Additional complication we need to keep in mind comes from the fact that
3386 * types, in general, can form graphs containing cycles, not just DAGs.
3387 *
3388 * While algorithm does deduplication, it also merges and resolves type
3389 * information (unless disabled throught `struct btf_opts`), whenever possible.
3390 * E.g., in the example above with two compilation units having partial type
3391 * information for structs `A` and `B`, the output of algorithm will emit
3392 * a single copy of each BTF type that describes structs `A`, `B`, and `S`
3393 * (as well as type information for `int` and pointers), as if they were defined
3394 * in a single compilation unit as:
3395 *
3396 * struct A {
3397 * int a;
3398 * struct A* self;
3399 * struct S* parent;
3400 * };
3401 * struct B {
3402 * int b;
3403 * struct B* self;
3404 * struct S* parent;
3405 * };
3406 * struct S {
3407 * struct A* a_ptr;
3408 * struct B* b_ptr;
3409 * };
3410 *
3411 * Algorithm summary
3412 * =================
3413 *
3414 * Algorithm completes its work in 7 separate passes:
3415 *
3416 * 1. Strings deduplication.
3417 * 2. Primitive types deduplication (int, enum, fwd).
3418 * 3. Struct/union types deduplication.
3419 * 4. Resolve unambiguous forward declarations.
3420 * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func
3421 * protos, and const/volatile/restrict modifiers).
3422 * 6. Types compaction.
3423 * 7. Types remapping.
3424 *
3425 * Algorithm determines canonical type descriptor, which is a single
3426 * representative type for each truly unique type. This canonical type is the
3427 * one that will go into final deduplicated BTF type information. For
3428 * struct/unions, it is also the type that algorithm will merge additional type
3429 * information into (while resolving FWDs), as it discovers it from data in
3430 * other CUs. Each input BTF type eventually gets either mapped to itself, if
3431 * that type is canonical, or to some other type, if that type is equivalent
3432 * and was chosen as canonical representative. This mapping is stored in
3433 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
3434 * FWD type got resolved to.
3435 *
3436 * To facilitate fast discovery of canonical types, we also maintain canonical
3437 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
3438 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types
3439 * that match that signature. With sufficiently good choice of type signature
3440 * hashing function, we can limit number of canonical types for each unique type
3441 * signature to a very small number, allowing to find canonical type for any
3442 * duplicated type very quickly.
3443 *
3444 * Struct/union deduplication is the most critical part and algorithm for
3445 * deduplicating structs/unions is described in greater details in comments for
3446 * `btf_dedup_is_equiv` function.
3447 */
3448int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts)
3449{
3450 struct btf_dedup *d;
3451 int err;
3452
3453 if (!OPTS_VALID(opts, btf_dedup_opts))
3454 return libbpf_err(-EINVAL);
3455
3456 d = btf_dedup_new(btf, opts);
3457 if (IS_ERR(d)) {
3458 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d));
3459 return libbpf_err(-EINVAL);
3460 }
3461
3462 if (btf_ensure_modifiable(btf)) {
3463 err = -ENOMEM;
3464 goto done;
3465 }
3466
3467 err = btf_dedup_prep(d);
3468 if (err) {
3469 pr_debug("btf_dedup_prep failed: %s\n", errstr(err));
3470 goto done;
3471 }
3472 err = btf_dedup_strings(d);
3473 if (err < 0) {
3474 pr_debug("btf_dedup_strings failed: %s\n", errstr(err));
3475 goto done;
3476 }
3477 err = btf_dedup_prim_types(d);
3478 if (err < 0) {
3479 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err));
3480 goto done;
3481 }
3482 err = btf_dedup_struct_types(d);
3483 if (err < 0) {
3484 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err));
3485 goto done;
3486 }
3487 err = btf_dedup_resolve_fwds(d);
3488 if (err < 0) {
3489 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err));
3490 goto done;
3491 }
3492 err = btf_dedup_ref_types(d);
3493 if (err < 0) {
3494 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err));
3495 goto done;
3496 }
3497 err = btf_dedup_compact_types(d);
3498 if (err < 0) {
3499 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err));
3500 goto done;
3501 }
3502 err = btf_dedup_remap_types(d);
3503 if (err < 0) {
3504 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err));
3505 goto done;
3506 }
3507
3508done:
3509 btf_dedup_free(d);
3510 return libbpf_err(err);
3511}
3512
3513#define BTF_UNPROCESSED_ID ((__u32)-1)
3514#define BTF_IN_PROGRESS_ID ((__u32)-2)
3515
3516struct btf_dedup {
3517 /* .BTF section to be deduped in-place */
3518 struct btf *btf;
3519 /*
3520 * Optional .BTF.ext section. When provided, any strings referenced
3521 * from it will be taken into account when deduping strings
3522 */
3523 struct btf_ext *btf_ext;
3524 /*
3525 * This is a map from any type's signature hash to a list of possible
3526 * canonical representative type candidates. Hash collisions are
3527 * ignored, so even types of various kinds can share same list of
3528 * candidates, which is fine because we rely on subsequent
3529 * btf_xxx_equal() checks to authoritatively verify type equality.
3530 */
3531 struct hashmap *dedup_table;
3532 /* Canonical types map */
3533 __u32 *map;
3534 /* Hypothetical mapping, used during type graph equivalence checks */
3535 __u32 *hypot_map;
3536 __u32 *hypot_list;
3537 size_t hypot_cnt;
3538 size_t hypot_cap;
3539 /* Whether hypothetical mapping, if successful, would need to adjust
3540 * already canonicalized types (due to a new forward declaration to
3541 * concrete type resolution). In such case, during split BTF dedup
3542 * candidate type would still be considered as different, because base
3543 * BTF is considered to be immutable.
3544 */
3545 bool hypot_adjust_canon;
3546 /* Various option modifying behavior of algorithm */
3547 struct btf_dedup_opts opts;
3548 /* temporary strings deduplication state */
3549 struct strset *strs_set;
3550};
3551
3552static unsigned long hash_combine(unsigned long h, unsigned long value)
3553{
3554 return h * 31 + value;
3555}
3556
3557#define for_each_dedup_cand(d, node, hash) \
3558 hashmap__for_each_key_entry(d->dedup_table, node, hash)
3559
3560static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id)
3561{
3562 return hashmap__append(d->dedup_table, hash, type_id);
3563}
3564
3565static int btf_dedup_hypot_map_add(struct btf_dedup *d,
3566 __u32 from_id, __u32 to_id)
3567{
3568 if (d->hypot_cnt == d->hypot_cap) {
3569 __u32 *new_list;
3570
3571 d->hypot_cap += max((size_t)16, d->hypot_cap / 2);
3572 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32));
3573 if (!new_list)
3574 return -ENOMEM;
3575 d->hypot_list = new_list;
3576 }
3577 d->hypot_list[d->hypot_cnt++] = from_id;
3578 d->hypot_map[from_id] = to_id;
3579 return 0;
3580}
3581
3582static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
3583{
3584 int i;
3585
3586 for (i = 0; i < d->hypot_cnt; i++)
3587 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
3588 d->hypot_cnt = 0;
3589 d->hypot_adjust_canon = false;
3590}
3591
3592static void btf_dedup_free(struct btf_dedup *d)
3593{
3594 hashmap__free(d->dedup_table);
3595 d->dedup_table = NULL;
3596
3597 free(d->map);
3598 d->map = NULL;
3599
3600 free(d->hypot_map);
3601 d->hypot_map = NULL;
3602
3603 free(d->hypot_list);
3604 d->hypot_list = NULL;
3605
3606 free(d);
3607}
3608
3609static size_t btf_dedup_identity_hash_fn(long key, void *ctx)
3610{
3611 return key;
3612}
3613
3614static size_t btf_dedup_collision_hash_fn(long key, void *ctx)
3615{
3616 return 0;
3617}
3618
3619static bool btf_dedup_equal_fn(long k1, long k2, void *ctx)
3620{
3621 return k1 == k2;
3622}
3623
3624static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts)
3625{
3626 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
3627 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
3628 int i, err = 0, type_cnt;
3629
3630 if (!d)
3631 return ERR_PTR(-ENOMEM);
3632
3633 if (OPTS_GET(opts, force_collisions, false))
3634 hash_fn = btf_dedup_collision_hash_fn;
3635
3636 d->btf = btf;
3637 d->btf_ext = OPTS_GET(opts, btf_ext, NULL);
3638
3639 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);
3640 if (IS_ERR(d->dedup_table)) {
3641 err = PTR_ERR(d->dedup_table);
3642 d->dedup_table = NULL;
3643 goto done;
3644 }
3645
3646 type_cnt = btf__type_cnt(btf);
3647 d->map = malloc(sizeof(__u32) * type_cnt);
3648 if (!d->map) {
3649 err = -ENOMEM;
3650 goto done;
3651 }
3652 /* special BTF "void" type is made canonical immediately */
3653 d->map[0] = 0;
3654 for (i = 1; i < type_cnt; i++) {
3655 struct btf_type *t = btf_type_by_id(d->btf, i);
3656
3657 /* VAR and DATASEC are never deduped and are self-canonical */
3658 if (btf_is_var(t) || btf_is_datasec(t))
3659 d->map[i] = i;
3660 else
3661 d->map[i] = BTF_UNPROCESSED_ID;
3662 }
3663
3664 d->hypot_map = malloc(sizeof(__u32) * type_cnt);
3665 if (!d->hypot_map) {
3666 err = -ENOMEM;
3667 goto done;
3668 }
3669 for (i = 0; i < type_cnt; i++)
3670 d->hypot_map[i] = BTF_UNPROCESSED_ID;
3671
3672done:
3673 if (err) {
3674 btf_dedup_free(d);
3675 return ERR_PTR(err);
3676 }
3677
3678 return d;
3679}
3680
3681/*
3682 * Iterate over all possible places in .BTF and .BTF.ext that can reference
3683 * string and pass pointer to it to a provided callback `fn`.
3684 */
3685static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx)
3686{
3687 int i, r;
3688
3689 for (i = 0; i < d->btf->nr_types; i++) {
3690 struct btf_field_iter it;
3691 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
3692 __u32 *str_off;
3693
3694 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
3695 if (r)
3696 return r;
3697
3698 while ((str_off = btf_field_iter_next(&it))) {
3699 r = fn(str_off, ctx);
3700 if (r)
3701 return r;
3702 }
3703 }
3704
3705 if (!d->btf_ext)
3706 return 0;
3707
3708 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx);
3709 if (r)
3710 return r;
3711
3712 return 0;
3713}
3714
3715static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
3716{
3717 struct btf_dedup *d = ctx;
3718 __u32 str_off = *str_off_ptr;
3719 const char *s;
3720 int off, err;
3721
3722 /* don't touch empty string or string in main BTF */
3723 if (str_off == 0 || str_off < d->btf->start_str_off)
3724 return 0;
3725
3726 s = btf__str_by_offset(d->btf, str_off);
3727 if (d->btf->base_btf) {
3728 err = btf__find_str(d->btf->base_btf, s);
3729 if (err >= 0) {
3730 *str_off_ptr = err;
3731 return 0;
3732 }
3733 if (err != -ENOENT)
3734 return err;
3735 }
3736
3737 off = strset__add_str(d->strs_set, s);
3738 if (off < 0)
3739 return off;
3740
3741 *str_off_ptr = d->btf->start_str_off + off;
3742 return 0;
3743}
3744
3745/*
3746 * Dedup string and filter out those that are not referenced from either .BTF
3747 * or .BTF.ext (if provided) sections.
3748 *
3749 * This is done by building index of all strings in BTF's string section,
3750 * then iterating over all entities that can reference strings (e.g., type
3751 * names, struct field names, .BTF.ext line info, etc) and marking corresponding
3752 * strings as used. After that all used strings are deduped and compacted into
3753 * sequential blob of memory and new offsets are calculated. Then all the string
3754 * references are iterated again and rewritten using new offsets.
3755 */
3756static int btf_dedup_strings(struct btf_dedup *d)
3757{
3758 int err;
3759
3760 if (d->btf->strs_deduped)
3761 return 0;
3762
3763 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0);
3764 if (IS_ERR(d->strs_set)) {
3765 err = PTR_ERR(d->strs_set);
3766 goto err_out;
3767 }
3768
3769 if (!d->btf->base_btf) {
3770 /* insert empty string; we won't be looking it up during strings
3771 * dedup, but it's good to have it for generic BTF string lookups
3772 */
3773 err = strset__add_str(d->strs_set, "");
3774 if (err < 0)
3775 goto err_out;
3776 }
3777
3778 /* remap string offsets */
3779 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
3780 if (err)
3781 goto err_out;
3782
3783 /* replace BTF string data and hash with deduped ones */
3784 strset__free(d->btf->strs_set);
3785 d->btf->hdr->str_len = strset__data_size(d->strs_set);
3786 d->btf->strs_set = d->strs_set;
3787 d->strs_set = NULL;
3788 d->btf->strs_deduped = true;
3789 return 0;
3790
3791err_out:
3792 strset__free(d->strs_set);
3793 d->strs_set = NULL;
3794
3795 return err;
3796}
3797
3798static long btf_hash_common(struct btf_type *t)
3799{
3800 long h;
3801
3802 h = hash_combine(0, t->name_off);
3803 h = hash_combine(h, t->info);
3804 h = hash_combine(h, t->size);
3805 return h;
3806}
3807
3808static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2)
3809{
3810 return t1->name_off == t2->name_off &&
3811 t1->info == t2->info &&
3812 t1->size == t2->size;
3813}
3814
3815/* Calculate type signature hash of INT or TAG. */
3816static long btf_hash_int_decl_tag(struct btf_type *t)
3817{
3818 __u32 info = *(__u32 *)(t + 1);
3819 long h;
3820
3821 h = btf_hash_common(t);
3822 h = hash_combine(h, info);
3823 return h;
3824}
3825
3826/* Check structural equality of two INTs or TAGs. */
3827static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2)
3828{
3829 __u32 info1, info2;
3830
3831 if (!btf_equal_common(t1, t2))
3832 return false;
3833 info1 = *(__u32 *)(t1 + 1);
3834 info2 = *(__u32 *)(t2 + 1);
3835 return info1 == info2;
3836}
3837
3838/* Calculate type signature hash of ENUM/ENUM64. */
3839static long btf_hash_enum(struct btf_type *t)
3840{
3841 long h;
3842
3843 /* don't hash vlen, enum members and size to support enum fwd resolving */
3844 h = hash_combine(0, t->name_off);
3845 return h;
3846}
3847
3848static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2)
3849{
3850 const struct btf_enum *m1, *m2;
3851 __u16 vlen;
3852 int i;
3853
3854 vlen = btf_vlen(t1);
3855 m1 = btf_enum(t1);
3856 m2 = btf_enum(t2);
3857 for (i = 0; i < vlen; i++) {
3858 if (m1->name_off != m2->name_off || m1->val != m2->val)
3859 return false;
3860 m1++;
3861 m2++;
3862 }
3863 return true;
3864}
3865
3866static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2)
3867{
3868 const struct btf_enum64 *m1, *m2;
3869 __u16 vlen;
3870 int i;
3871
3872 vlen = btf_vlen(t1);
3873 m1 = btf_enum64(t1);
3874 m2 = btf_enum64(t2);
3875 for (i = 0; i < vlen; i++) {
3876 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 ||
3877 m1->val_hi32 != m2->val_hi32)
3878 return false;
3879 m1++;
3880 m2++;
3881 }
3882 return true;
3883}
3884
3885/* Check structural equality of two ENUMs or ENUM64s. */
3886static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
3887{
3888 if (!btf_equal_common(t1, t2))
3889 return false;
3890
3891 /* t1 & t2 kinds are identical because of btf_equal_common */
3892 if (btf_kind(t1) == BTF_KIND_ENUM)
3893 return btf_equal_enum_members(t1, t2);
3894 else
3895 return btf_equal_enum64_members(t1, t2);
3896}
3897
3898static inline bool btf_is_enum_fwd(struct btf_type *t)
3899{
3900 return btf_is_any_enum(t) && btf_vlen(t) == 0;
3901}
3902
3903static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
3904{
3905 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
3906 return btf_equal_enum(t1, t2);
3907 /* At this point either t1 or t2 or both are forward declarations, thus:
3908 * - skip comparing vlen because it is zero for forward declarations;
3909 * - skip comparing size to allow enum forward declarations
3910 * to be compatible with enum64 full declarations;
3911 * - skip comparing kind for the same reason.
3912 */
3913 return t1->name_off == t2->name_off &&
3914 btf_is_any_enum(t1) && btf_is_any_enum(t2);
3915}
3916
3917/*
3918 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
3919 * as referenced type IDs equivalence is established separately during type
3920 * graph equivalence check algorithm.
3921 */
3922static long btf_hash_struct(struct btf_type *t)
3923{
3924 const struct btf_member *member = btf_members(t);
3925 __u32 vlen = btf_vlen(t);
3926 long h = btf_hash_common(t);
3927 int i;
3928
3929 for (i = 0; i < vlen; i++) {
3930 h = hash_combine(h, member->name_off);
3931 h = hash_combine(h, member->offset);
3932 /* no hashing of referenced type ID, it can be unresolved yet */
3933 member++;
3934 }
3935 return h;
3936}
3937
3938/*
3939 * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced
3940 * type IDs. This check is performed during type graph equivalence check and
3941 * referenced types equivalence is checked separately.
3942 */
3943static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
3944{
3945 const struct btf_member *m1, *m2;
3946 __u16 vlen;
3947 int i;
3948
3949 if (!btf_equal_common(t1, t2))
3950 return false;
3951
3952 vlen = btf_vlen(t1);
3953 m1 = btf_members(t1);
3954 m2 = btf_members(t2);
3955 for (i = 0; i < vlen; i++) {
3956 if (m1->name_off != m2->name_off || m1->offset != m2->offset)
3957 return false;
3958 m1++;
3959 m2++;
3960 }
3961 return true;
3962}
3963
3964/*
3965 * Calculate type signature hash of ARRAY, including referenced type IDs,
3966 * under assumption that they were already resolved to canonical type IDs and
3967 * are not going to change.
3968 */
3969static long btf_hash_array(struct btf_type *t)
3970{
3971 const struct btf_array *info = btf_array(t);
3972 long h = btf_hash_common(t);
3973
3974 h = hash_combine(h, info->type);
3975 h = hash_combine(h, info->index_type);
3976 h = hash_combine(h, info->nelems);
3977 return h;
3978}
3979
3980/*
3981 * Check exact equality of two ARRAYs, taking into account referenced
3982 * type IDs, under assumption that they were already resolved to canonical
3983 * type IDs and are not going to change.
3984 * This function is called during reference types deduplication to compare
3985 * ARRAY to potential canonical representative.
3986 */
3987static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2)
3988{
3989 const struct btf_array *info1, *info2;
3990
3991 if (!btf_equal_common(t1, t2))
3992 return false;
3993
3994 info1 = btf_array(t1);
3995 info2 = btf_array(t2);
3996 return info1->type == info2->type &&
3997 info1->index_type == info2->index_type &&
3998 info1->nelems == info2->nelems;
3999}
4000
4001/*
4002 * Check structural compatibility of two ARRAYs, ignoring referenced type
4003 * IDs. This check is performed during type graph equivalence check and
4004 * referenced types equivalence is checked separately.
4005 */
4006static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2)
4007{
4008 if (!btf_equal_common(t1, t2))
4009 return false;
4010
4011 return btf_array(t1)->nelems == btf_array(t2)->nelems;
4012}
4013
4014/*
4015 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs,
4016 * under assumption that they were already resolved to canonical type IDs and
4017 * are not going to change.
4018 */
4019static long btf_hash_fnproto(struct btf_type *t)
4020{
4021 const struct btf_param *member = btf_params(t);
4022 __u16 vlen = btf_vlen(t);
4023 long h = btf_hash_common(t);
4024 int i;
4025
4026 for (i = 0; i < vlen; i++) {
4027 h = hash_combine(h, member->name_off);
4028 h = hash_combine(h, member->type);
4029 member++;
4030 }
4031 return h;
4032}
4033
4034/*
4035 * Check exact equality of two FUNC_PROTOs, taking into account referenced
4036 * type IDs, under assumption that they were already resolved to canonical
4037 * type IDs and are not going to change.
4038 * This function is called during reference types deduplication to compare
4039 * FUNC_PROTO to potential canonical representative.
4040 */
4041static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2)
4042{
4043 const struct btf_param *m1, *m2;
4044 __u16 vlen;
4045 int i;
4046
4047 if (!btf_equal_common(t1, t2))
4048 return false;
4049
4050 vlen = btf_vlen(t1);
4051 m1 = btf_params(t1);
4052 m2 = btf_params(t2);
4053 for (i = 0; i < vlen; i++) {
4054 if (m1->name_off != m2->name_off || m1->type != m2->type)
4055 return false;
4056 m1++;
4057 m2++;
4058 }
4059 return true;
4060}
4061
4062/*
4063 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
4064 * IDs. This check is performed during type graph equivalence check and
4065 * referenced types equivalence is checked separately.
4066 */
4067static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
4068{
4069 const struct btf_param *m1, *m2;
4070 __u16 vlen;
4071 int i;
4072
4073 /* skip return type ID */
4074 if (t1->name_off != t2->name_off || t1->info != t2->info)
4075 return false;
4076
4077 vlen = btf_vlen(t1);
4078 m1 = btf_params(t1);
4079 m2 = btf_params(t2);
4080 for (i = 0; i < vlen; i++) {
4081 if (m1->name_off != m2->name_off)
4082 return false;
4083 m1++;
4084 m2++;
4085 }
4086 return true;
4087}
4088
4089/* Prepare split BTF for deduplication by calculating hashes of base BTF's
4090 * types and initializing the rest of the state (canonical type mapping) for
4091 * the fixed base BTF part.
4092 */
4093static int btf_dedup_prep(struct btf_dedup *d)
4094{
4095 struct btf_type *t;
4096 int type_id;
4097 long h;
4098
4099 if (!d->btf->base_btf)
4100 return 0;
4101
4102 for (type_id = 1; type_id < d->btf->start_id; type_id++) {
4103 t = btf_type_by_id(d->btf, type_id);
4104
4105 /* all base BTF types are self-canonical by definition */
4106 d->map[type_id] = type_id;
4107
4108 switch (btf_kind(t)) {
4109 case BTF_KIND_VAR:
4110 case BTF_KIND_DATASEC:
4111 /* VAR and DATASEC are never hash/deduplicated */
4112 continue;
4113 case BTF_KIND_CONST:
4114 case BTF_KIND_VOLATILE:
4115 case BTF_KIND_RESTRICT:
4116 case BTF_KIND_PTR:
4117 case BTF_KIND_FWD:
4118 case BTF_KIND_TYPEDEF:
4119 case BTF_KIND_FUNC:
4120 case BTF_KIND_FLOAT:
4121 case BTF_KIND_TYPE_TAG:
4122 h = btf_hash_common(t);
4123 break;
4124 case BTF_KIND_INT:
4125 case BTF_KIND_DECL_TAG:
4126 h = btf_hash_int_decl_tag(t);
4127 break;
4128 case BTF_KIND_ENUM:
4129 case BTF_KIND_ENUM64:
4130 h = btf_hash_enum(t);
4131 break;
4132 case BTF_KIND_STRUCT:
4133 case BTF_KIND_UNION:
4134 h = btf_hash_struct(t);
4135 break;
4136 case BTF_KIND_ARRAY:
4137 h = btf_hash_array(t);
4138 break;
4139 case BTF_KIND_FUNC_PROTO:
4140 h = btf_hash_fnproto(t);
4141 break;
4142 default:
4143 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
4144 return -EINVAL;
4145 }
4146 if (btf_dedup_table_add(d, h, type_id))
4147 return -ENOMEM;
4148 }
4149
4150 return 0;
4151}
4152
4153/*
4154 * Deduplicate primitive types, that can't reference other types, by calculating
4155 * their type signature hash and comparing them with any possible canonical
4156 * candidate. If no canonical candidate matches, type itself is marked as
4157 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
4158 */
4159static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
4160{
4161 struct btf_type *t = btf_type_by_id(d->btf, type_id);
4162 struct hashmap_entry *hash_entry;
4163 struct btf_type *cand;
4164 /* if we don't find equivalent type, then we are canonical */
4165 __u32 new_id = type_id;
4166 __u32 cand_id;
4167 long h;
4168
4169 switch (btf_kind(t)) {
4170 case BTF_KIND_CONST:
4171 case BTF_KIND_VOLATILE:
4172 case BTF_KIND_RESTRICT:
4173 case BTF_KIND_PTR:
4174 case BTF_KIND_TYPEDEF:
4175 case BTF_KIND_ARRAY:
4176 case BTF_KIND_STRUCT:
4177 case BTF_KIND_UNION:
4178 case BTF_KIND_FUNC:
4179 case BTF_KIND_FUNC_PROTO:
4180 case BTF_KIND_VAR:
4181 case BTF_KIND_DATASEC:
4182 case BTF_KIND_DECL_TAG:
4183 case BTF_KIND_TYPE_TAG:
4184 return 0;
4185
4186 case BTF_KIND_INT:
4187 h = btf_hash_int_decl_tag(t);
4188 for_each_dedup_cand(d, hash_entry, h) {
4189 cand_id = hash_entry->value;
4190 cand = btf_type_by_id(d->btf, cand_id);
4191 if (btf_equal_int_tag(t, cand)) {
4192 new_id = cand_id;
4193 break;
4194 }
4195 }
4196 break;
4197
4198 case BTF_KIND_ENUM:
4199 case BTF_KIND_ENUM64:
4200 h = btf_hash_enum(t);
4201 for_each_dedup_cand(d, hash_entry, h) {
4202 cand_id = hash_entry->value;
4203 cand = btf_type_by_id(d->btf, cand_id);
4204 if (btf_equal_enum(t, cand)) {
4205 new_id = cand_id;
4206 break;
4207 }
4208 if (btf_compat_enum(t, cand)) {
4209 if (btf_is_enum_fwd(t)) {
4210 /* resolve fwd to full enum */
4211 new_id = cand_id;
4212 break;
4213 }
4214 /* resolve canonical enum fwd to full enum */
4215 d->map[cand_id] = type_id;
4216 }
4217 }
4218 break;
4219
4220 case BTF_KIND_FWD:
4221 case BTF_KIND_FLOAT:
4222 h = btf_hash_common(t);
4223 for_each_dedup_cand(d, hash_entry, h) {
4224 cand_id = hash_entry->value;
4225 cand = btf_type_by_id(d->btf, cand_id);
4226 if (btf_equal_common(t, cand)) {
4227 new_id = cand_id;
4228 break;
4229 }
4230 }
4231 break;
4232
4233 default:
4234 return -EINVAL;
4235 }
4236
4237 d->map[type_id] = new_id;
4238 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4239 return -ENOMEM;
4240
4241 return 0;
4242}
4243
4244static int btf_dedup_prim_types(struct btf_dedup *d)
4245{
4246 int i, err;
4247
4248 for (i = 0; i < d->btf->nr_types; i++) {
4249 err = btf_dedup_prim_type(d, d->btf->start_id + i);
4250 if (err)
4251 return err;
4252 }
4253 return 0;
4254}
4255
4256/*
4257 * Check whether type is already mapped into canonical one (could be to itself).
4258 */
4259static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id)
4260{
4261 return d->map[type_id] <= BTF_MAX_NR_TYPES;
4262}
4263
4264/*
4265 * Resolve type ID into its canonical type ID, if any; otherwise return original
4266 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow
4267 * STRUCT/UNION link and resolve it into canonical type ID as well.
4268 */
4269static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id)
4270{
4271 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
4272 type_id = d->map[type_id];
4273 return type_id;
4274}
4275
4276/*
4277 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original
4278 * type ID.
4279 */
4280static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id)
4281{
4282 __u32 orig_type_id = type_id;
4283
4284 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
4285 return type_id;
4286
4287 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
4288 type_id = d->map[type_id];
4289
4290 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
4291 return type_id;
4292
4293 return orig_type_id;
4294}
4295
4296
4297static inline __u16 btf_fwd_kind(struct btf_type *t)
4298{
4299 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
4300}
4301
4302/* Check if given two types are identical ARRAY definitions */
4303static bool btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
4304{
4305 struct btf_type *t1, *t2;
4306
4307 t1 = btf_type_by_id(d->btf, id1);
4308 t2 = btf_type_by_id(d->btf, id2);
4309 if (!btf_is_array(t1) || !btf_is_array(t2))
4310 return false;
4311
4312 return btf_equal_array(t1, t2);
4313}
4314
4315/* Check if given two types are identical STRUCT/UNION definitions */
4316static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2)
4317{
4318 const struct btf_member *m1, *m2;
4319 struct btf_type *t1, *t2;
4320 int n, i;
4321
4322 t1 = btf_type_by_id(d->btf, id1);
4323 t2 = btf_type_by_id(d->btf, id2);
4324
4325 if (!btf_is_composite(t1) || btf_kind(t1) != btf_kind(t2))
4326 return false;
4327
4328 if (!btf_shallow_equal_struct(t1, t2))
4329 return false;
4330
4331 m1 = btf_members(t1);
4332 m2 = btf_members(t2);
4333 for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
4334 if (m1->type != m2->type &&
4335 !btf_dedup_identical_arrays(d, m1->type, m2->type) &&
4336 !btf_dedup_identical_structs(d, m1->type, m2->type))
4337 return false;
4338 }
4339 return true;
4340}
4341
4342/*
4343 * Check equivalence of BTF type graph formed by candidate struct/union (we'll
4344 * call it "candidate graph" in this description for brevity) to a type graph
4345 * formed by (potential) canonical struct/union ("canonical graph" for brevity
4346 * here, though keep in mind that not all types in canonical graph are
4347 * necessarily canonical representatives themselves, some of them might be
4348 * duplicates or its uniqueness might not have been established yet).
4349 * Returns:
4350 * - >0, if type graphs are equivalent;
4351 * - 0, if not equivalent;
4352 * - <0, on error.
4353 *
4354 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
4355 * equivalence of BTF types at each step. If at any point BTF types in candidate
4356 * and canonical graphs are not compatible structurally, whole graphs are
4357 * incompatible. If types are structurally equivalent (i.e., all information
4358 * except referenced type IDs is exactly the same), a mapping from `canon_id` to
4359 * a `cand_id` is recoded in hypothetical mapping (`btf_dedup->hypot_map`).
4360 * If a type references other types, then those referenced types are checked
4361 * for equivalence recursively.
4362 *
4363 * During DFS traversal, if we find that for current `canon_id` type we
4364 * already have some mapping in hypothetical map, we check for two possible
4365 * situations:
4366 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
4367 * happen when type graphs have cycles. In this case we assume those two
4368 * types are equivalent.
4369 * - `canon_id` is mapped to different type. This is contradiction in our
4370 * hypothetical mapping, because same graph in canonical graph corresponds
4371 * to two different types in candidate graph, which for equivalent type
4372 * graphs shouldn't happen. This condition terminates equivalence check
4373 * with negative result.
4374 *
4375 * If type graphs traversal exhausts types to check and find no contradiction,
4376 * then type graphs are equivalent.
4377 *
4378 * When checking types for equivalence, there is one special case: FWD types.
4379 * If FWD type resolution is allowed and one of the types (either from canonical
4380 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind
4381 * flag) and their names match, hypothetical mapping is updated to point from
4382 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully,
4383 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
4384 *
4385 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution,
4386 * if there are two exactly named (or anonymous) structs/unions that are
4387 * compatible structurally, one of which has FWD field, while other is concrete
4388 * STRUCT/UNION, but according to C sources they are different structs/unions
4389 * that are referencing different types with the same name. This is extremely
4390 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if
4391 * this logic is causing problems.
4392 *
4393 * Doing FWD resolution means that both candidate and/or canonical graphs can
4394 * consists of portions of the graph that come from multiple compilation units.
4395 * This is due to the fact that types within single compilation unit are always
4396 * deduplicated and FWDs are already resolved, if referenced struct/union
4397 * definition is available. So, if we had unresolved FWD and found corresponding
4398 * STRUCT/UNION, they will be from different compilation units. This
4399 * consequently means that when we "link" FWD to corresponding STRUCT/UNION,
4400 * type graph will likely have at least two different BTF types that describe
4401 * same type (e.g., most probably there will be two different BTF types for the
4402 * same 'int' primitive type) and could even have "overlapping" parts of type
4403 * graph that describe same subset of types.
4404 *
4405 * This in turn means that our assumption that each type in canonical graph
4406 * must correspond to exactly one type in candidate graph might not hold
4407 * anymore and will make it harder to detect contradictions using hypothetical
4408 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
4409 * resolution only in canonical graph. FWDs in candidate graphs are never
4410 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs
4411 * that can occur:
4412 * - Both types in canonical and candidate graphs are FWDs. If they are
4413 * structurally equivalent, then they can either be both resolved to the
4414 * same STRUCT/UNION or not resolved at all. In both cases they are
4415 * equivalent and there is no need to resolve FWD on candidate side.
4416 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
4417 * so nothing to resolve as well, algorithm will check equivalence anyway.
4418 * - Type in canonical graph is FWD, while type in candidate is concrete
4419 * STRUCT/UNION. In this case candidate graph comes from single compilation
4420 * unit, so there is exactly one BTF type for each unique C type. After
4421 * resolving FWD into STRUCT/UNION, there might be more than one BTF type
4422 * in canonical graph mapping to single BTF type in candidate graph, but
4423 * because hypothetical mapping maps from canonical to candidate types, it's
4424 * alright, and we still maintain the property of having single `canon_id`
4425 * mapping to single `cand_id` (there could be two different `canon_id`
4426 * mapped to the same `cand_id`, but it's not contradictory).
4427 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
4428 * graph is FWD. In this case we are just going to check compatibility of
4429 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll
4430 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to
4431 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs
4432 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
4433 * canonical graph.
4434 */
4435static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
4436 __u32 canon_id)
4437{
4438 struct btf_type *cand_type;
4439 struct btf_type *canon_type;
4440 __u32 hypot_type_id;
4441 __u16 cand_kind;
4442 __u16 canon_kind;
4443 int i, eq;
4444
4445 /* if both resolve to the same canonical, they must be equivalent */
4446 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id))
4447 return 1;
4448
4449 canon_id = resolve_fwd_id(d, canon_id);
4450
4451 hypot_type_id = d->hypot_map[canon_id];
4452 if (hypot_type_id <= BTF_MAX_NR_TYPES) {
4453 if (hypot_type_id == cand_id)
4454 return 1;
4455 /* In some cases compiler will generate different DWARF types
4456 * for *identical* array type definitions and use them for
4457 * different fields within the *same* struct. This breaks type
4458 * equivalence check, which makes an assumption that candidate
4459 * types sub-graph has a consistent and deduped-by-compiler
4460 * types within a single CU. So work around that by explicitly
4461 * allowing identical array types here.
4462 */
4463 if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
4464 return 1;
4465 /* It turns out that similar situation can happen with
4466 * struct/union sometimes, sigh... Handle the case where
4467 * structs/unions are exactly the same, down to the referenced
4468 * type IDs. Anything more complicated (e.g., if referenced
4469 * types are different, but equivalent) is *way more*
4470 * complicated and requires a many-to-many equivalence mapping.
4471 */
4472 if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
4473 return 1;
4474 return 0;
4475 }
4476
4477 if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
4478 return -ENOMEM;
4479
4480 cand_type = btf_type_by_id(d->btf, cand_id);
4481 canon_type = btf_type_by_id(d->btf, canon_id);
4482 cand_kind = btf_kind(cand_type);
4483 canon_kind = btf_kind(canon_type);
4484
4485 if (cand_type->name_off != canon_type->name_off)
4486 return 0;
4487
4488 /* FWD <--> STRUCT/UNION equivalence check, if enabled */
4489 if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD)
4490 && cand_kind != canon_kind) {
4491 __u16 real_kind;
4492 __u16 fwd_kind;
4493
4494 if (cand_kind == BTF_KIND_FWD) {
4495 real_kind = canon_kind;
4496 fwd_kind = btf_fwd_kind(cand_type);
4497 } else {
4498 real_kind = cand_kind;
4499 fwd_kind = btf_fwd_kind(canon_type);
4500 /* we'd need to resolve base FWD to STRUCT/UNION */
4501 if (fwd_kind == real_kind && canon_id < d->btf->start_id)
4502 d->hypot_adjust_canon = true;
4503 }
4504 return fwd_kind == real_kind;
4505 }
4506
4507 if (cand_kind != canon_kind)
4508 return 0;
4509
4510 switch (cand_kind) {
4511 case BTF_KIND_INT:
4512 return btf_equal_int_tag(cand_type, canon_type);
4513
4514 case BTF_KIND_ENUM:
4515 case BTF_KIND_ENUM64:
4516 return btf_compat_enum(cand_type, canon_type);
4517
4518 case BTF_KIND_FWD:
4519 case BTF_KIND_FLOAT:
4520 return btf_equal_common(cand_type, canon_type);
4521
4522 case BTF_KIND_CONST:
4523 case BTF_KIND_VOLATILE:
4524 case BTF_KIND_RESTRICT:
4525 case BTF_KIND_PTR:
4526 case BTF_KIND_TYPEDEF:
4527 case BTF_KIND_FUNC:
4528 case BTF_KIND_TYPE_TAG:
4529 if (cand_type->info != canon_type->info)
4530 return 0;
4531 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
4532
4533 case BTF_KIND_ARRAY: {
4534 const struct btf_array *cand_arr, *canon_arr;
4535
4536 if (!btf_compat_array(cand_type, canon_type))
4537 return 0;
4538 cand_arr = btf_array(cand_type);
4539 canon_arr = btf_array(canon_type);
4540 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
4541 if (eq <= 0)
4542 return eq;
4543 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
4544 }
4545
4546 case BTF_KIND_STRUCT:
4547 case BTF_KIND_UNION: {
4548 const struct btf_member *cand_m, *canon_m;
4549 __u16 vlen;
4550
4551 if (!btf_shallow_equal_struct(cand_type, canon_type))
4552 return 0;
4553 vlen = btf_vlen(cand_type);
4554 cand_m = btf_members(cand_type);
4555 canon_m = btf_members(canon_type);
4556 for (i = 0; i < vlen; i++) {
4557 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
4558 if (eq <= 0)
4559 return eq;
4560 cand_m++;
4561 canon_m++;
4562 }
4563
4564 return 1;
4565 }
4566
4567 case BTF_KIND_FUNC_PROTO: {
4568 const struct btf_param *cand_p, *canon_p;
4569 __u16 vlen;
4570
4571 if (!btf_compat_fnproto(cand_type, canon_type))
4572 return 0;
4573 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
4574 if (eq <= 0)
4575 return eq;
4576 vlen = btf_vlen(cand_type);
4577 cand_p = btf_params(cand_type);
4578 canon_p = btf_params(canon_type);
4579 for (i = 0; i < vlen; i++) {
4580 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);
4581 if (eq <= 0)
4582 return eq;
4583 cand_p++;
4584 canon_p++;
4585 }
4586 return 1;
4587 }
4588
4589 default:
4590 return -EINVAL;
4591 }
4592 return 0;
4593}
4594
4595/*
4596 * Use hypothetical mapping, produced by successful type graph equivalence
4597 * check, to augment existing struct/union canonical mapping, where possible.
4598 *
4599 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record
4600 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
4601 * it doesn't matter if FWD type was part of canonical graph or candidate one,
4602 * we are recording the mapping anyway. As opposed to carefulness required
4603 * for struct/union correspondence mapping (described below), for FWD resolution
4604 * it's not important, as by the time that FWD type (reference type) will be
4605 * deduplicated all structs/unions will be deduped already anyway.
4606 *
4607 * Recording STRUCT/UNION mapping is purely a performance optimization and is
4608 * not required for correctness. It needs to be done carefully to ensure that
4609 * struct/union from candidate's type graph is not mapped into corresponding
4610 * struct/union from canonical type graph that itself hasn't been resolved into
4611 * canonical representative. The only guarantee we have is that canonical
4612 * struct/union was determined as canonical and that won't change. But any
4613 * types referenced through that struct/union fields could have been not yet
4614 * resolved, so in case like that it's too early to establish any kind of
4615 * correspondence between structs/unions.
4616 *
4617 * No canonical correspondence is derived for primitive types (they are already
4618 * deduplicated completely already anyway) or reference types (they rely on
4619 * stability of struct/union canonical relationship for equivalence checks).
4620 */
4621static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
4622{
4623 __u32 canon_type_id, targ_type_id;
4624 __u16 t_kind, c_kind;
4625 __u32 t_id, c_id;
4626 int i;
4627
4628 for (i = 0; i < d->hypot_cnt; i++) {
4629 canon_type_id = d->hypot_list[i];
4630 targ_type_id = d->hypot_map[canon_type_id];
4631 t_id = resolve_type_id(d, targ_type_id);
4632 c_id = resolve_type_id(d, canon_type_id);
4633 t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
4634 c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
4635 /*
4636 * Resolve FWD into STRUCT/UNION.
4637 * It's ok to resolve FWD into STRUCT/UNION that's not yet
4638 * mapped to canonical representative (as opposed to
4639 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because
4640 * eventually that struct is going to be mapped and all resolved
4641 * FWDs will automatically resolve to correct canonical
4642 * representative. This will happen before ref type deduping,
4643 * which critically depends on stability of these mapping. This
4644 * stability is not a requirement for STRUCT/UNION equivalence
4645 * checks, though.
4646 */
4647
4648 /* if it's the split BTF case, we still need to point base FWD
4649 * to STRUCT/UNION in a split BTF, because FWDs from split BTF
4650 * will be resolved against base FWD. If we don't point base
4651 * canonical FWD to the resolved STRUCT/UNION, then all the
4652 * FWDs in split BTF won't be correctly resolved to a proper
4653 * STRUCT/UNION.
4654 */
4655 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
4656 d->map[c_id] = t_id;
4657
4658 /* if graph equivalence determined that we'd need to adjust
4659 * base canonical types, then we need to only point base FWDs
4660 * to STRUCTs/UNIONs and do no more modifications. For all
4661 * other purposes the type graphs were not equivalent.
4662 */
4663 if (d->hypot_adjust_canon)
4664 continue;
4665
4666 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
4667 d->map[t_id] = c_id;
4668
4669 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
4670 c_kind != BTF_KIND_FWD &&
4671 is_type_mapped(d, c_id) &&
4672 !is_type_mapped(d, t_id)) {
4673 /*
4674 * as a perf optimization, we can map struct/union
4675 * that's part of type graph we just verified for
4676 * equivalence. We can do that for struct/union that has
4677 * canonical representative only, though.
4678 */
4679 d->map[t_id] = c_id;
4680 }
4681 }
4682}
4683
4684/*
4685 * Deduplicate struct/union types.
4686 *
4687 * For each struct/union type its type signature hash is calculated, taking
4688 * into account type's name, size, number, order and names of fields, but
4689 * ignoring type ID's referenced from fields, because they might not be deduped
4690 * completely until after reference types deduplication phase. This type hash
4691 * is used to iterate over all potential canonical types, sharing same hash.
4692 * For each canonical candidate we check whether type graphs that they form
4693 * (through referenced types in fields and so on) are equivalent using algorithm
4694 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and
4695 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping
4696 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4697 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4698 * potentially map other structs/unions to their canonical representatives,
4699 * if such relationship hasn't yet been established. This speeds up algorithm
4700 * by eliminating some of the duplicate work.
4701 *
4702 * If no matching canonical representative was found, struct/union is marked
4703 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4704 * for further look ups.
4705 */
4706static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
4707{
4708 struct btf_type *cand_type, *t;
4709 struct hashmap_entry *hash_entry;
4710 /* if we don't find equivalent type, then we are canonical */
4711 __u32 new_id = type_id;
4712 __u16 kind;
4713 long h;
4714
4715 /* already deduped or is in process of deduping (loop detected) */
4716 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4717 return 0;
4718
4719 t = btf_type_by_id(d->btf, type_id);
4720 kind = btf_kind(t);
4721
4722 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
4723 return 0;
4724
4725 h = btf_hash_struct(t);
4726 for_each_dedup_cand(d, hash_entry, h) {
4727 __u32 cand_id = hash_entry->value;
4728 int eq;
4729
4730 /*
4731 * Even though btf_dedup_is_equiv() checks for
4732 * btf_shallow_equal_struct() internally when checking two
4733 * structs (unions) for equivalence, we need to guard here
4734 * from picking matching FWD type as a dedup candidate.
4735 * This can happen due to hash collision. In such case just
4736 * relying on btf_dedup_is_equiv() would lead to potentially
4737 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because
4738 * FWD and compatible STRUCT/UNION are considered equivalent.
4739 */
4740 cand_type = btf_type_by_id(d->btf, cand_id);
4741 if (!btf_shallow_equal_struct(t, cand_type))
4742 continue;
4743
4744 btf_dedup_clear_hypot_map(d);
4745 eq = btf_dedup_is_equiv(d, type_id, cand_id);
4746 if (eq < 0)
4747 return eq;
4748 if (!eq)
4749 continue;
4750 btf_dedup_merge_hypot_map(d);
4751 if (d->hypot_adjust_canon) /* not really equivalent */
4752 continue;
4753 new_id = cand_id;
4754 break;
4755 }
4756
4757 d->map[type_id] = new_id;
4758 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4759 return -ENOMEM;
4760
4761 return 0;
4762}
4763
4764static int btf_dedup_struct_types(struct btf_dedup *d)
4765{
4766 int i, err;
4767
4768 for (i = 0; i < d->btf->nr_types; i++) {
4769 err = btf_dedup_struct_type(d, d->btf->start_id + i);
4770 if (err)
4771 return err;
4772 }
4773 return 0;
4774}
4775
4776/*
4777 * Deduplicate reference type.
4778 *
4779 * Once all primitive and struct/union types got deduplicated, we can easily
4780 * deduplicate all other (reference) BTF types. This is done in two steps:
4781 *
4782 * 1. Resolve all referenced type IDs into their canonical type IDs. This
4783 * resolution can be done either immediately for primitive or struct/union types
4784 * (because they were deduped in previous two phases) or recursively for
4785 * reference types. Recursion will always terminate at either primitive or
4786 * struct/union type, at which point we can "unwind" chain of reference types
4787 * one by one. There is no danger of encountering cycles because in C type
4788 * system the only way to form type cycle is through struct/union, so any chain
4789 * of reference types, even those taking part in a type cycle, will inevitably
4790 * reach struct/union at some point.
4791 *
4792 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type
4793 * becomes "stable", in the sense that no further deduplication will cause
4794 * any changes to it. With that, it's now possible to calculate type's signature
4795 * hash (this time taking into account referenced type IDs) and loop over all
4796 * potential canonical representatives. If no match was found, current type
4797 * will become canonical representative of itself and will be added into
4798 * btf_dedup->dedup_table as another possible canonical representative.
4799 */
4800static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id)
4801{
4802 struct hashmap_entry *hash_entry;
4803 __u32 new_id = type_id, cand_id;
4804 struct btf_type *t, *cand;
4805 /* if we don't find equivalent type, then we are representative type */
4806 int ref_type_id;
4807 long h;
4808
4809 if (d->map[type_id] == BTF_IN_PROGRESS_ID)
4810 return -ELOOP;
4811 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4812 return resolve_type_id(d, type_id);
4813
4814 t = btf_type_by_id(d->btf, type_id);
4815 d->map[type_id] = BTF_IN_PROGRESS_ID;
4816
4817 switch (btf_kind(t)) {
4818 case BTF_KIND_CONST:
4819 case BTF_KIND_VOLATILE:
4820 case BTF_KIND_RESTRICT:
4821 case BTF_KIND_PTR:
4822 case BTF_KIND_TYPEDEF:
4823 case BTF_KIND_FUNC:
4824 case BTF_KIND_TYPE_TAG:
4825 ref_type_id = btf_dedup_ref_type(d, t->type);
4826 if (ref_type_id < 0)
4827 return ref_type_id;
4828 t->type = ref_type_id;
4829
4830 h = btf_hash_common(t);
4831 for_each_dedup_cand(d, hash_entry, h) {
4832 cand_id = hash_entry->value;
4833 cand = btf_type_by_id(d->btf, cand_id);
4834 if (btf_equal_common(t, cand)) {
4835 new_id = cand_id;
4836 break;
4837 }
4838 }
4839 break;
4840
4841 case BTF_KIND_DECL_TAG:
4842 ref_type_id = btf_dedup_ref_type(d, t->type);
4843 if (ref_type_id < 0)
4844 return ref_type_id;
4845 t->type = ref_type_id;
4846
4847 h = btf_hash_int_decl_tag(t);
4848 for_each_dedup_cand(d, hash_entry, h) {
4849 cand_id = hash_entry->value;
4850 cand = btf_type_by_id(d->btf, cand_id);
4851 if (btf_equal_int_tag(t, cand)) {
4852 new_id = cand_id;
4853 break;
4854 }
4855 }
4856 break;
4857
4858 case BTF_KIND_ARRAY: {
4859 struct btf_array *info = btf_array(t);
4860
4861 ref_type_id = btf_dedup_ref_type(d, info->type);
4862 if (ref_type_id < 0)
4863 return ref_type_id;
4864 info->type = ref_type_id;
4865
4866 ref_type_id = btf_dedup_ref_type(d, info->index_type);
4867 if (ref_type_id < 0)
4868 return ref_type_id;
4869 info->index_type = ref_type_id;
4870
4871 h = btf_hash_array(t);
4872 for_each_dedup_cand(d, hash_entry, h) {
4873 cand_id = hash_entry->value;
4874 cand = btf_type_by_id(d->btf, cand_id);
4875 if (btf_equal_array(t, cand)) {
4876 new_id = cand_id;
4877 break;
4878 }
4879 }
4880 break;
4881 }
4882
4883 case BTF_KIND_FUNC_PROTO: {
4884 struct btf_param *param;
4885 __u16 vlen;
4886 int i;
4887
4888 ref_type_id = btf_dedup_ref_type(d, t->type);
4889 if (ref_type_id < 0)
4890 return ref_type_id;
4891 t->type = ref_type_id;
4892
4893 vlen = btf_vlen(t);
4894 param = btf_params(t);
4895 for (i = 0; i < vlen; i++) {
4896 ref_type_id = btf_dedup_ref_type(d, param->type);
4897 if (ref_type_id < 0)
4898 return ref_type_id;
4899 param->type = ref_type_id;
4900 param++;
4901 }
4902
4903 h = btf_hash_fnproto(t);
4904 for_each_dedup_cand(d, hash_entry, h) {
4905 cand_id = hash_entry->value;
4906 cand = btf_type_by_id(d->btf, cand_id);
4907 if (btf_equal_fnproto(t, cand)) {
4908 new_id = cand_id;
4909 break;
4910 }
4911 }
4912 break;
4913 }
4914
4915 default:
4916 return -EINVAL;
4917 }
4918
4919 d->map[type_id] = new_id;
4920 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4921 return -ENOMEM;
4922
4923 return new_id;
4924}
4925
4926static int btf_dedup_ref_types(struct btf_dedup *d)
4927{
4928 int i, err;
4929
4930 for (i = 0; i < d->btf->nr_types; i++) {
4931 err = btf_dedup_ref_type(d, d->btf->start_id + i);
4932 if (err < 0)
4933 return err;
4934 }
4935 /* we won't need d->dedup_table anymore */
4936 hashmap__free(d->dedup_table);
4937 d->dedup_table = NULL;
4938 return 0;
4939}
4940
4941/*
4942 * Collect a map from type names to type ids for all canonical structs
4943 * and unions. If the same name is shared by several canonical types
4944 * use a special value 0 to indicate this fact.
4945 */
4946static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map)
4947{
4948 __u32 nr_types = btf__type_cnt(d->btf);
4949 struct btf_type *t;
4950 __u32 type_id;
4951 __u16 kind;
4952 int err;
4953
4954 /*
4955 * Iterate over base and split module ids in order to get all
4956 * available structs in the map.
4957 */
4958 for (type_id = 1; type_id < nr_types; ++type_id) {
4959 t = btf_type_by_id(d->btf, type_id);
4960 kind = btf_kind(t);
4961
4962 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
4963 continue;
4964
4965 /* Skip non-canonical types */
4966 if (type_id != d->map[type_id])
4967 continue;
4968
4969 err = hashmap__add(names_map, t->name_off, type_id);
4970 if (err == -EEXIST)
4971 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL);
4972
4973 if (err)
4974 return err;
4975 }
4976
4977 return 0;
4978}
4979
4980static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id)
4981{
4982 struct btf_type *t = btf_type_by_id(d->btf, type_id);
4983 enum btf_fwd_kind fwd_kind = btf_kflag(t);
4984 __u16 cand_kind, kind = btf_kind(t);
4985 struct btf_type *cand_t;
4986 uintptr_t cand_id;
4987
4988 if (kind != BTF_KIND_FWD)
4989 return 0;
4990
4991 /* Skip if this FWD already has a mapping */
4992 if (type_id != d->map[type_id])
4993 return 0;
4994
4995 if (!hashmap__find(names_map, t->name_off, &cand_id))
4996 return 0;
4997
4998 /* Zero is a special value indicating that name is not unique */
4999 if (!cand_id)
5000 return 0;
5001
5002 cand_t = btf_type_by_id(d->btf, cand_id);
5003 cand_kind = btf_kind(cand_t);
5004 if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) ||
5005 (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION))
5006 return 0;
5007
5008 d->map[type_id] = cand_id;
5009
5010 return 0;
5011}
5012
5013/*
5014 * Resolve unambiguous forward declarations.
5015 *
5016 * The lion's share of all FWD declarations is resolved during
5017 * `btf_dedup_struct_types` phase when different type graphs are
5018 * compared against each other. However, if in some compilation unit a
5019 * FWD declaration is not a part of a type graph compared against
5020 * another type graph that declaration's canonical type would not be
5021 * changed. Example:
5022 *
5023 * CU #1:
5024 *
5025 * struct foo;
5026 * struct foo *some_global;
5027 *
5028 * CU #2:
5029 *
5030 * struct foo { int u; };
5031 * struct foo *another_global;
5032 *
5033 * After `btf_dedup_struct_types` the BTF looks as follows:
5034 *
5035 * [1] STRUCT 'foo' size=4 vlen=1 ...
5036 * [2] INT 'int' size=4 ...
5037 * [3] PTR '(anon)' type_id=1
5038 * [4] FWD 'foo' fwd_kind=struct
5039 * [5] PTR '(anon)' type_id=4
5040 *
5041 * This pass assumes that such FWD declarations should be mapped to
5042 * structs or unions with identical name in case if the name is not
5043 * ambiguous.
5044 */
5045static int btf_dedup_resolve_fwds(struct btf_dedup *d)
5046{
5047 int i, err;
5048 struct hashmap *names_map;
5049
5050 names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
5051 if (IS_ERR(names_map))
5052 return PTR_ERR(names_map);
5053
5054 err = btf_dedup_fill_unique_names_map(d, names_map);
5055 if (err < 0)
5056 goto exit;
5057
5058 for (i = 0; i < d->btf->nr_types; i++) {
5059 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i);
5060 if (err < 0)
5061 break;
5062 }
5063
5064exit:
5065 hashmap__free(names_map);
5066 return err;
5067}
5068
5069/*
5070 * Compact types.
5071 *
5072 * After we established for each type its corresponding canonical representative
5073 * type, we now can eliminate types that are not canonical and leave only
5074 * canonical ones layed out sequentially in memory by copying them over
5075 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
5076 * a map from original type ID to a new compacted type ID, which will be used
5077 * during next phase to "fix up" type IDs, referenced from struct/union and
5078 * reference types.
5079 */
5080static int btf_dedup_compact_types(struct btf_dedup *d)
5081{
5082 __u32 *new_offs;
5083 __u32 next_type_id = d->btf->start_id;
5084 const struct btf_type *t;
5085 void *p;
5086 int i, id, len;
5087
5088 /* we are going to reuse hypot_map to store compaction remapping */
5089 d->hypot_map[0] = 0;
5090 /* base BTF types are not renumbered */
5091 for (id = 1; id < d->btf->start_id; id++)
5092 d->hypot_map[id] = id;
5093 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
5094 d->hypot_map[id] = BTF_UNPROCESSED_ID;
5095
5096 p = d->btf->types_data;
5097
5098 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
5099 if (d->map[id] != id)
5100 continue;
5101
5102 t = btf__type_by_id(d->btf, id);
5103 len = btf_type_size(t);
5104 if (len < 0)
5105 return len;
5106
5107 memmove(p, t, len);
5108 d->hypot_map[id] = next_type_id;
5109 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
5110 p += len;
5111 next_type_id++;
5112 }
5113
5114 /* shrink struct btf's internal types index and update btf_header */
5115 d->btf->nr_types = next_type_id - d->btf->start_id;
5116 d->btf->type_offs_cap = d->btf->nr_types;
5117 d->btf->hdr->type_len = p - d->btf->types_data;
5118 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
5119 sizeof(*new_offs));
5120 if (d->btf->type_offs_cap && !new_offs)
5121 return -ENOMEM;
5122 d->btf->type_offs = new_offs;
5123 d->btf->hdr->str_off = d->btf->hdr->type_len;
5124 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;
5125 return 0;
5126}
5127
5128/*
5129 * Figure out final (deduplicated and compacted) type ID for provided original
5130 * `type_id` by first resolving it into corresponding canonical type ID and
5131 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
5132 * which is populated during compaction phase.
5133 */
5134static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx)
5135{
5136 struct btf_dedup *d = ctx;
5137 __u32 resolved_type_id, new_type_id;
5138
5139 resolved_type_id = resolve_type_id(d, *type_id);
5140 new_type_id = d->hypot_map[resolved_type_id];
5141 if (new_type_id > BTF_MAX_NR_TYPES)
5142 return -EINVAL;
5143
5144 *type_id = new_type_id;
5145 return 0;
5146}
5147
5148/*
5149 * Remap referenced type IDs into deduped type IDs.
5150 *
5151 * After BTF types are deduplicated and compacted, their final type IDs may
5152 * differ from original ones. The map from original to a corresponding
5153 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
5154 * compaction phase. During remapping phase we are rewriting all type IDs
5155 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to
5156 * their final deduped type IDs.
5157 */
5158static int btf_dedup_remap_types(struct btf_dedup *d)
5159{
5160 int i, r;
5161
5162 for (i = 0; i < d->btf->nr_types; i++) {
5163 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
5164 struct btf_field_iter it;
5165 __u32 *type_id;
5166
5167 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
5168 if (r)
5169 return r;
5170
5171 while ((type_id = btf_field_iter_next(&it))) {
5172 __u32 resolved_id, new_id;
5173
5174 resolved_id = resolve_type_id(d, *type_id);
5175 new_id = d->hypot_map[resolved_id];
5176 if (new_id > BTF_MAX_NR_TYPES)
5177 return -EINVAL;
5178
5179 *type_id = new_id;
5180 }
5181 }
5182
5183 if (!d->btf_ext)
5184 return 0;
5185
5186 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d);
5187 if (r)
5188 return r;
5189
5190 return 0;
5191}
5192
5193/*
5194 * Probe few well-known locations for vmlinux kernel image and try to load BTF
5195 * data out of it to use for target BTF.
5196 */
5197struct btf *btf__load_vmlinux_btf(void)
5198{
5199 const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux";
5200 /* fall back locations, trying to find vmlinux on disk */
5201 const char *locations[] = {
5202 "/boot/vmlinux-%1$s",
5203 "/lib/modules/%1$s/vmlinux-%1$s",
5204 "/lib/modules/%1$s/build/vmlinux",
5205 "/usr/lib/modules/%1$s/kernel/vmlinux",
5206 "/usr/lib/debug/boot/vmlinux-%1$s",
5207 "/usr/lib/debug/boot/vmlinux-%1$s.debug",
5208 "/usr/lib/debug/lib/modules/%1$s/vmlinux",
5209 };
5210 char path[PATH_MAX + 1];
5211 struct utsname buf;
5212 struct btf *btf;
5213 int i, err;
5214
5215 /* is canonical sysfs location accessible? */
5216 if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) {
5217 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n",
5218 sysfs_btf_path);
5219 } else {
5220 btf = btf__parse(sysfs_btf_path, NULL);
5221 if (!btf) {
5222 err = -errno;
5223 pr_warn("failed to read kernel BTF from '%s': %s\n",
5224 sysfs_btf_path, errstr(err));
5225 return libbpf_err_ptr(err);
5226 }
5227 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path);
5228 return btf;
5229 }
5230
5231 /* try fallback locations */
5232 uname(&buf);
5233 for (i = 0; i < ARRAY_SIZE(locations); i++) {
5234 snprintf(path, PATH_MAX, locations[i], buf.release);
5235
5236 if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS))
5237 continue;
5238
5239 btf = btf__parse(path, NULL);
5240 err = libbpf_get_error(btf);
5241 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err));
5242 if (err)
5243 continue;
5244
5245 return btf;
5246 }
5247
5248 pr_warn("failed to find valid kernel BTF\n");
5249 return libbpf_err_ptr(-ESRCH);
5250}
5251
5252struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf")));
5253
5254struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf)
5255{
5256 char path[80];
5257
5258 snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name);
5259 return btf__parse_split(path, vmlinux_btf);
5260}
5261
5262int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx)
5263{
5264 const struct btf_ext_info *seg;
5265 struct btf_ext_info_sec *sec;
5266 int i, err;
5267
5268 seg = &btf_ext->func_info;
5269 for_each_btf_ext_sec(seg, sec) {
5270 struct bpf_func_info_min *rec;
5271
5272 for_each_btf_ext_rec(seg, sec, i, rec) {
5273 err = visit(&rec->type_id, ctx);
5274 if (err < 0)
5275 return err;
5276 }
5277 }
5278
5279 seg = &btf_ext->core_relo_info;
5280 for_each_btf_ext_sec(seg, sec) {
5281 struct bpf_core_relo *rec;
5282
5283 for_each_btf_ext_rec(seg, sec, i, rec) {
5284 err = visit(&rec->type_id, ctx);
5285 if (err < 0)
5286 return err;
5287 }
5288 }
5289
5290 return 0;
5291}
5292
5293int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx)
5294{
5295 const struct btf_ext_info *seg;
5296 struct btf_ext_info_sec *sec;
5297 int i, err;
5298
5299 seg = &btf_ext->func_info;
5300 for_each_btf_ext_sec(seg, sec) {
5301 err = visit(&sec->sec_name_off, ctx);
5302 if (err)
5303 return err;
5304 }
5305
5306 seg = &btf_ext->line_info;
5307 for_each_btf_ext_sec(seg, sec) {
5308 struct bpf_line_info_min *rec;
5309
5310 err = visit(&sec->sec_name_off, ctx);
5311 if (err)
5312 return err;
5313
5314 for_each_btf_ext_rec(seg, sec, i, rec) {
5315 err = visit(&rec->file_name_off, ctx);
5316 if (err)
5317 return err;
5318 err = visit(&rec->line_off, ctx);
5319 if (err)
5320 return err;
5321 }
5322 }
5323
5324 seg = &btf_ext->core_relo_info;
5325 for_each_btf_ext_sec(seg, sec) {
5326 struct bpf_core_relo *rec;
5327
5328 err = visit(&sec->sec_name_off, ctx);
5329 if (err)
5330 return err;
5331
5332 for_each_btf_ext_rec(seg, sec, i, rec) {
5333 err = visit(&rec->access_str_off, ctx);
5334 if (err)
5335 return err;
5336 }
5337 }
5338
5339 return 0;
5340}
5341
5342struct btf_distill {
5343 struct btf_pipe pipe;
5344 int *id_map;
5345 unsigned int split_start_id;
5346 unsigned int split_start_str;
5347 int diff_id;
5348};
5349
5350static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i)
5351{
5352 struct btf_type *split_t = btf_type_by_id(dist->pipe.src, i);
5353 struct btf_field_iter it;
5354 __u32 *id;
5355 int err;
5356
5357 err = btf_field_iter_init(&it, split_t, BTF_FIELD_ITER_IDS);
5358 if (err)
5359 return err;
5360 while ((id = btf_field_iter_next(&it))) {
5361 struct btf_type *base_t;
5362
5363 if (!*id)
5364 continue;
5365 /* split BTF id, not needed */
5366 if (*id >= dist->split_start_id)
5367 continue;
5368 /* already added ? */
5369 if (dist->id_map[*id] > 0)
5370 continue;
5371
5372 /* only a subset of base BTF types should be referenced from
5373 * split BTF; ensure nothing unexpected is referenced.
5374 */
5375 base_t = btf_type_by_id(dist->pipe.src, *id);
5376 switch (btf_kind(base_t)) {
5377 case BTF_KIND_INT:
5378 case BTF_KIND_FLOAT:
5379 case BTF_KIND_FWD:
5380 case BTF_KIND_ARRAY:
5381 case BTF_KIND_STRUCT:
5382 case BTF_KIND_UNION:
5383 case BTF_KIND_TYPEDEF:
5384 case BTF_KIND_ENUM:
5385 case BTF_KIND_ENUM64:
5386 case BTF_KIND_PTR:
5387 case BTF_KIND_CONST:
5388 case BTF_KIND_RESTRICT:
5389 case BTF_KIND_VOLATILE:
5390 case BTF_KIND_FUNC_PROTO:
5391 case BTF_KIND_TYPE_TAG:
5392 dist->id_map[*id] = *id;
5393 break;
5394 default:
5395 pr_warn("unexpected reference to base type[%u] of kind [%u] when creating distilled base BTF.\n",
5396 *id, btf_kind(base_t));
5397 return -EINVAL;
5398 }
5399 /* If a base type is used, ensure types it refers to are
5400 * marked as used also; so for example if we find a PTR to INT
5401 * we need both the PTR and INT.
5402 *
5403 * The only exception is named struct/unions, since distilled
5404 * base BTF composite types have no members.
5405 */
5406 if (btf_is_composite(base_t) && base_t->name_off)
5407 continue;
5408 err = btf_add_distilled_type_ids(dist, *id);
5409 if (err)
5410 return err;
5411 }
5412 return 0;
5413}
5414
5415static int btf_add_distilled_types(struct btf_distill *dist)
5416{
5417 bool adding_to_base = dist->pipe.dst->start_id == 1;
5418 int id = btf__type_cnt(dist->pipe.dst);
5419 struct btf_type *t;
5420 int i, err = 0;
5421
5422
5423 /* Add types for each of the required references to either distilled
5424 * base or split BTF, depending on type characteristics.
5425 */
5426 for (i = 1; i < dist->split_start_id; i++) {
5427 const char *name;
5428 int kind;
5429
5430 if (!dist->id_map[i])
5431 continue;
5432 t = btf_type_by_id(dist->pipe.src, i);
5433 kind = btf_kind(t);
5434 name = btf__name_by_offset(dist->pipe.src, t->name_off);
5435
5436 switch (kind) {
5437 case BTF_KIND_INT:
5438 case BTF_KIND_FLOAT:
5439 case BTF_KIND_FWD:
5440 /* Named int, float, fwd are added to base. */
5441 if (!adding_to_base)
5442 continue;
5443 err = btf_add_type(&dist->pipe, t);
5444 break;
5445 case BTF_KIND_STRUCT:
5446 case BTF_KIND_UNION:
5447 /* Named struct/union are added to base as 0-vlen
5448 * struct/union of same size. Anonymous struct/unions
5449 * are added to split BTF as-is.
5450 */
5451 if (adding_to_base) {
5452 if (!t->name_off)
5453 continue;
5454 err = btf_add_composite(dist->pipe.dst, kind, name, t->size);
5455 } else {
5456 if (t->name_off)
5457 continue;
5458 err = btf_add_type(&dist->pipe, t);
5459 }
5460 break;
5461 case BTF_KIND_ENUM:
5462 case BTF_KIND_ENUM64:
5463 /* Named enum[64]s are added to base as a sized
5464 * enum; relocation will match with appropriately-named
5465 * and sized enum or enum64.
5466 *
5467 * Anonymous enums are added to split BTF as-is.
5468 */
5469 if (adding_to_base) {
5470 if (!t->name_off)
5471 continue;
5472 err = btf__add_enum(dist->pipe.dst, name, t->size);
5473 } else {
5474 if (t->name_off)
5475 continue;
5476 err = btf_add_type(&dist->pipe, t);
5477 }
5478 break;
5479 case BTF_KIND_ARRAY:
5480 case BTF_KIND_TYPEDEF:
5481 case BTF_KIND_PTR:
5482 case BTF_KIND_CONST:
5483 case BTF_KIND_RESTRICT:
5484 case BTF_KIND_VOLATILE:
5485 case BTF_KIND_FUNC_PROTO:
5486 case BTF_KIND_TYPE_TAG:
5487 /* All other types are added to split BTF. */
5488 if (adding_to_base)
5489 continue;
5490 err = btf_add_type(&dist->pipe, t);
5491 break;
5492 default:
5493 pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n",
5494 name, i, kind);
5495 return -EINVAL;
5496
5497 }
5498 if (err < 0)
5499 break;
5500 dist->id_map[i] = id++;
5501 }
5502 return err;
5503}
5504
5505/* Split BTF ids without a mapping will be shifted downwards since distilled
5506 * base BTF is smaller than the original base BTF. For those that have a
5507 * mapping (either to base or updated split BTF), update the id based on
5508 * that mapping.
5509 */
5510static int btf_update_distilled_type_ids(struct btf_distill *dist, __u32 i)
5511{
5512 struct btf_type *t = btf_type_by_id(dist->pipe.dst, i);
5513 struct btf_field_iter it;
5514 __u32 *id;
5515 int err;
5516
5517 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
5518 if (err)
5519 return err;
5520 while ((id = btf_field_iter_next(&it))) {
5521 if (dist->id_map[*id])
5522 *id = dist->id_map[*id];
5523 else if (*id >= dist->split_start_id)
5524 *id -= dist->diff_id;
5525 }
5526 return 0;
5527}
5528
5529/* Create updated split BTF with distilled base BTF; distilled base BTF
5530 * consists of BTF information required to clarify the types that split
5531 * BTF refers to, omitting unneeded details. Specifically it will contain
5532 * base types and memberless definitions of named structs, unions and enumerated
5533 * types. Associated reference types like pointers, arrays and anonymous
5534 * structs, unions and enumerated types will be added to split BTF.
5535 * Size is recorded for named struct/unions to help guide matching to the
5536 * target base BTF during later relocation.
5537 *
5538 * The only case where structs, unions or enumerated types are fully represented
5539 * is when they are anonymous; in such cases, the anonymous type is added to
5540 * split BTF in full.
5541 *
5542 * We return newly-created split BTF where the split BTF refers to a newly-created
5543 * distilled base BTF. Both must be freed separately by the caller.
5544 */
5545int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf,
5546 struct btf **new_split_btf)
5547{
5548 struct btf *new_base = NULL, *new_split = NULL;
5549 const struct btf *old_base;
5550 unsigned int n = btf__type_cnt(src_btf);
5551 struct btf_distill dist = {};
5552 struct btf_type *t;
5553 int i, err = 0;
5554
5555 /* src BTF must be split BTF. */
5556 old_base = btf__base_btf(src_btf);
5557 if (!new_base_btf || !new_split_btf || !old_base)
5558 return libbpf_err(-EINVAL);
5559
5560 new_base = btf__new_empty();
5561 if (!new_base)
5562 return libbpf_err(-ENOMEM);
5563
5564 btf__set_endianness(new_base, btf__endianness(src_btf));
5565
5566 dist.id_map = calloc(n, sizeof(*dist.id_map));
5567 if (!dist.id_map) {
5568 err = -ENOMEM;
5569 goto done;
5570 }
5571 dist.pipe.src = src_btf;
5572 dist.pipe.dst = new_base;
5573 dist.pipe.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
5574 if (IS_ERR(dist.pipe.str_off_map)) {
5575 err = -ENOMEM;
5576 goto done;
5577 }
5578 dist.split_start_id = btf__type_cnt(old_base);
5579 dist.split_start_str = old_base->hdr->str_len;
5580
5581 /* Pass over src split BTF; generate the list of base BTF type ids it
5582 * references; these will constitute our distilled BTF set to be
5583 * distributed over base and split BTF as appropriate.
5584 */
5585 for (i = src_btf->start_id; i < n; i++) {
5586 err = btf_add_distilled_type_ids(&dist, i);
5587 if (err < 0)
5588 goto done;
5589 }
5590 /* Next add types for each of the required references to base BTF and split BTF
5591 * in turn.
5592 */
5593 err = btf_add_distilled_types(&dist);
5594 if (err < 0)
5595 goto done;
5596
5597 /* Create new split BTF with distilled base BTF as its base; the final
5598 * state is split BTF with distilled base BTF that represents enough
5599 * about its base references to allow it to be relocated with the base
5600 * BTF available.
5601 */
5602 new_split = btf__new_empty_split(new_base);
5603 if (!new_split) {
5604 err = -errno;
5605 goto done;
5606 }
5607 dist.pipe.dst = new_split;
5608 /* First add all split types */
5609 for (i = src_btf->start_id; i < n; i++) {
5610 t = btf_type_by_id(src_btf, i);
5611 err = btf_add_type(&dist.pipe, t);
5612 if (err < 0)
5613 goto done;
5614 }
5615 /* Now add distilled types to split BTF that are not added to base. */
5616 err = btf_add_distilled_types(&dist);
5617 if (err < 0)
5618 goto done;
5619
5620 /* All split BTF ids will be shifted downwards since there are less base
5621 * BTF ids in distilled base BTF.
5622 */
5623 dist.diff_id = dist.split_start_id - btf__type_cnt(new_base);
5624
5625 n = btf__type_cnt(new_split);
5626 /* Now update base/split BTF ids. */
5627 for (i = 1; i < n; i++) {
5628 err = btf_update_distilled_type_ids(&dist, i);
5629 if (err < 0)
5630 break;
5631 }
5632done:
5633 free(dist.id_map);
5634 hashmap__free(dist.pipe.str_off_map);
5635 if (err) {
5636 btf__free(new_split);
5637 btf__free(new_base);
5638 return libbpf_err(err);
5639 }
5640 *new_base_btf = new_base;
5641 *new_split_btf = new_split;
5642
5643 return 0;
5644}
5645
5646const struct btf_header *btf_header(const struct btf *btf)
5647{
5648 return btf->hdr;
5649}
5650
5651void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
5652{
5653 btf->base_btf = (struct btf *)base_btf;
5654 btf->start_id = btf__type_cnt(base_btf);
5655 btf->start_str_off = base_btf->hdr->str_len;
5656}
5657
5658int btf__relocate(struct btf *btf, const struct btf *base_btf)
5659{
5660 int err = btf_relocate(btf, base_btf, NULL);
5661
5662 if (!err)
5663 btf->owns_base = false;
5664 return libbpf_err(err);
5665}