1/* SPDX-License-Identifier: GPL-2.0 */
   2/* Copyright (c) 2018 Facebook */
   3
   4#include <uapi/linux/btf.h>
   5#include <uapi/linux/bpf.h>
   6#include <uapi/linux/bpf_perf_event.h>
   7#include <uapi/linux/types.h>
   8#include <linux/seq_file.h>
   9#include <linux/compiler.h>
  10#include <linux/ctype.h>
  11#include <linux/errno.h>
  12#include <linux/slab.h>
  13#include <linux/anon_inodes.h>
  14#include <linux/file.h>
  15#include <linux/uaccess.h>
  16#include <linux/kernel.h>
  17#include <linux/idr.h>
  18#include <linux/sort.h>
  19#include <linux/bpf_verifier.h>
  20#include <linux/btf.h>
  21#include <linux/btf_ids.h>
 
 
  22#include <linux/skmsg.h>
  23#include <linux/perf_event.h>
  24#include <linux/bsearch.h>
  25#include <linux/kobject.h>
  26#include <linux/sysfs.h>
 
 
 
  27#include <net/sock.h>
 
 
  28
  29/* BTF (BPF Type Format) is the meta data format which describes
  30 * the data types of BPF program/map.  Hence, it basically focus
  31 * on the C programming language which the modern BPF is primary
  32 * using.
  33 *
  34 * ELF Section:
  35 * ~~~~~~~~~~~
  36 * The BTF data is stored under the ".BTF" ELF section
  37 *
  38 * struct btf_type:
  39 * ~~~~~~~~~~~~~~~
  40 * Each 'struct btf_type' object describes a C data type.
  41 * Depending on the type it is describing, a 'struct btf_type'
  42 * object may be followed by more data.  F.e.
  43 * To describe an array, 'struct btf_type' is followed by
  44 * 'struct btf_array'.
  45 *
  46 * 'struct btf_type' and any extra data following it are
  47 * 4 bytes aligned.
  48 *
  49 * Type section:
  50 * ~~~~~~~~~~~~~
  51 * The BTF type section contains a list of 'struct btf_type' objects.
  52 * Each one describes a C type.  Recall from the above section
  53 * that a 'struct btf_type' object could be immediately followed by extra
  54 * data in order to describe some particular C types.
  55 *
  56 * type_id:
  57 * ~~~~~~~
  58 * Each btf_type object is identified by a type_id.  The type_id
  59 * is implicitly implied by the location of the btf_type object in
  60 * the BTF type section.  The first one has type_id 1.  The second
  61 * one has type_id 2...etc.  Hence, an earlier btf_type has
  62 * a smaller type_id.
  63 *
  64 * A btf_type object may refer to another btf_type object by using
  65 * type_id (i.e. the "type" in the "struct btf_type").
  66 *
  67 * NOTE that we cannot assume any reference-order.
  68 * A btf_type object can refer to an earlier btf_type object
  69 * but it can also refer to a later btf_type object.
  70 *
  71 * For example, to describe "const void *".  A btf_type
  72 * object describing "const" may refer to another btf_type
  73 * object describing "void *".  This type-reference is done
  74 * by specifying type_id:
  75 *
  76 * [1] CONST (anon) type_id=2
  77 * [2] PTR (anon) type_id=0
  78 *
  79 * The above is the btf_verifier debug log:
  80 *   - Each line started with "[?]" is a btf_type object
  81 *   - [?] is the type_id of the btf_type object.
  82 *   - CONST/PTR is the BTF_KIND_XXX
  83 *   - "(anon)" is the name of the type.  It just
  84 *     happens that CONST and PTR has no name.
  85 *   - type_id=XXX is the 'u32 type' in btf_type
  86 *
  87 * NOTE: "void" has type_id 0
  88 *
  89 * String section:
  90 * ~~~~~~~~~~~~~~
  91 * The BTF string section contains the names used by the type section.
  92 * Each string is referred by an "offset" from the beginning of the
  93 * string section.
  94 *
  95 * Each string is '\0' terminated.
  96 *
  97 * The first character in the string section must be '\0'
  98 * which is used to mean 'anonymous'. Some btf_type may not
  99 * have a name.
 100 */
 101
 102/* BTF verification:
 103 *
 104 * To verify BTF data, two passes are needed.
 105 *
 106 * Pass #1
 107 * ~~~~~~~
 108 * The first pass is to collect all btf_type objects to
 109 * an array: "btf->types".
 110 *
 111 * Depending on the C type that a btf_type is describing,
 112 * a btf_type may be followed by extra data.  We don't know
 113 * how many btf_type is there, and more importantly we don't
 114 * know where each btf_type is located in the type section.
 115 *
 116 * Without knowing the location of each type_id, most verifications
 117 * cannot be done.  e.g. an earlier btf_type may refer to a later
 118 * btf_type (recall the "const void *" above), so we cannot
 119 * check this type-reference in the first pass.
 120 *
 121 * In the first pass, it still does some verifications (e.g.
 122 * checking the name is a valid offset to the string section).
 123 *
 124 * Pass #2
 125 * ~~~~~~~
 126 * The main focus is to resolve a btf_type that is referring
 127 * to another type.
 128 *
 129 * We have to ensure the referring type:
 130 * 1) does exist in the BTF (i.e. in btf->types[])
 131 * 2) does not cause a loop:
 132 *	struct A {
 133 *		struct B b;
 134 *	};
 135 *
 136 *	struct B {
 137 *		struct A a;
 138 *	};
 139 *
 140 * btf_type_needs_resolve() decides if a btf_type needs
 141 * to be resolved.
 142 *
 143 * The needs_resolve type implements the "resolve()" ops which
 144 * essentially does a DFS and detects backedge.
 145 *
 146 * During resolve (or DFS), different C types have different
 147 * "RESOLVED" conditions.
 148 *
 149 * When resolving a BTF_KIND_STRUCT, we need to resolve all its
 150 * members because a member is always referring to another
 151 * type.  A struct's member can be treated as "RESOLVED" if
 152 * it is referring to a BTF_KIND_PTR.  Otherwise, the
 153 * following valid C struct would be rejected:
 154 *
 155 *	struct A {
 156 *		int m;
 157 *		struct A *a;
 158 *	};
 159 *
 160 * When resolving a BTF_KIND_PTR, it needs to keep resolving if
 161 * it is referring to another BTF_KIND_PTR.  Otherwise, we cannot
 162 * detect a pointer loop, e.g.:
 163 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
 164 *                        ^                                         |
 165 *                        +-----------------------------------------+
 166 *
 167 */
 168
 169#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2)
 170#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
 171#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
 172#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
 173#define BITS_ROUNDUP_BYTES(bits) \
 174	(BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
 175
 176#define BTF_INFO_MASK 0x9f00ffff
 177#define BTF_INT_MASK 0x0fffffff
 178#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
 179#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
 180
 181/* 16MB for 64k structs and each has 16 members and
 182 * a few MB spaces for the string section.
 183 * The hard limit is S32_MAX.
 184 */
 185#define BTF_MAX_SIZE (16 * 1024 * 1024)
 186
 187#define for_each_member_from(i, from, struct_type, member)		\
 188	for (i = from, member = btf_type_member(struct_type) + from;	\
 189	     i < btf_type_vlen(struct_type);				\
 190	     i++, member++)
 191
 192#define for_each_vsi_from(i, from, struct_type, member)				\
 193	for (i = from, member = btf_type_var_secinfo(struct_type) + from;	\
 194	     i < btf_type_vlen(struct_type);					\
 195	     i++, member++)
 196
 197DEFINE_IDR(btf_idr);
 198DEFINE_SPINLOCK(btf_idr_lock);
 199
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 200struct btf {
 201	void *data;
 202	struct btf_type **types;
 203	u32 *resolved_ids;
 204	u32 *resolved_sizes;
 205	const char *strings;
 206	void *nohdr_data;
 207	struct btf_header hdr;
 208	u32 nr_types; /* includes VOID for base BTF */
 209	u32 types_size;
 210	u32 data_size;
 211	refcount_t refcnt;
 212	u32 id;
 213	struct rcu_head rcu;
 
 
 
 
 214
 215	/* split BTF support */
 216	struct btf *base_btf;
 217	u32 start_id; /* first type ID in this BTF (0 for base BTF) */
 218	u32 start_str_off; /* first string offset (0 for base BTF) */
 219	char name[MODULE_NAME_LEN];
 220	bool kernel_btf;
 221};
 222
 223enum verifier_phase {
 224	CHECK_META,
 225	CHECK_TYPE,
 226};
 227
 228struct resolve_vertex {
 229	const struct btf_type *t;
 230	u32 type_id;
 231	u16 next_member;
 232};
 233
 234enum visit_state {
 235	NOT_VISITED,
 236	VISITED,
 237	RESOLVED,
 238};
 239
 240enum resolve_mode {
 241	RESOLVE_TBD,	/* To Be Determined */
 242	RESOLVE_PTR,	/* Resolving for Pointer */
 243	RESOLVE_STRUCT_OR_ARRAY,	/* Resolving for struct/union
 244					 * or array
 245					 */
 246};
 247
 248#define MAX_RESOLVE_DEPTH 32
 249
 250struct btf_sec_info {
 251	u32 off;
 252	u32 len;
 253};
 254
 255struct btf_verifier_env {
 256	struct btf *btf;
 257	u8 *visit_states;
 258	struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
 259	struct bpf_verifier_log log;
 260	u32 log_type_id;
 261	u32 top_stack;
 262	enum verifier_phase phase;
 263	enum resolve_mode resolve_mode;
 264};
 265
 266static const char * const btf_kind_str[NR_BTF_KINDS] = {
 267	[BTF_KIND_UNKN]		= "UNKNOWN",
 268	[BTF_KIND_INT]		= "INT",
 269	[BTF_KIND_PTR]		= "PTR",
 270	[BTF_KIND_ARRAY]	= "ARRAY",
 271	[BTF_KIND_STRUCT]	= "STRUCT",
 272	[BTF_KIND_UNION]	= "UNION",
 273	[BTF_KIND_ENUM]		= "ENUM",
 274	[BTF_KIND_FWD]		= "FWD",
 275	[BTF_KIND_TYPEDEF]	= "TYPEDEF",
 276	[BTF_KIND_VOLATILE]	= "VOLATILE",
 277	[BTF_KIND_CONST]	= "CONST",
 278	[BTF_KIND_RESTRICT]	= "RESTRICT",
 279	[BTF_KIND_FUNC]		= "FUNC",
 280	[BTF_KIND_FUNC_PROTO]	= "FUNC_PROTO",
 281	[BTF_KIND_VAR]		= "VAR",
 282	[BTF_KIND_DATASEC]	= "DATASEC",
 283	[BTF_KIND_FLOAT]	= "FLOAT",
 
 
 
 284};
 285
 286const char *btf_type_str(const struct btf_type *t)
 287{
 288	return btf_kind_str[BTF_INFO_KIND(t->info)];
 289}
 290
 291/* Chunk size we use in safe copy of data to be shown. */
 292#define BTF_SHOW_OBJ_SAFE_SIZE		32
 293
 294/*
 295 * This is the maximum size of a base type value (equivalent to a
 296 * 128-bit int); if we are at the end of our safe buffer and have
 297 * less than 16 bytes space we can't be assured of being able
 298 * to copy the next type safely, so in such cases we will initiate
 299 * a new copy.
 300 */
 301#define BTF_SHOW_OBJ_BASE_TYPE_SIZE	16
 302
 303/* Type name size */
 304#define BTF_SHOW_NAME_SIZE		80
 305
 306/*
 
 
 
 
 
 
 307 * Common data to all BTF show operations. Private show functions can add
 308 * their own data to a structure containing a struct btf_show and consult it
 309 * in the show callback.  See btf_type_show() below.
 310 *
 311 * One challenge with showing nested data is we want to skip 0-valued
 312 * data, but in order to figure out whether a nested object is all zeros
 313 * we need to walk through it.  As a result, we need to make two passes
 314 * when handling structs, unions and arrays; the first path simply looks
 315 * for nonzero data, while the second actually does the display.  The first
 316 * pass is signalled by show->state.depth_check being set, and if we
 317 * encounter a non-zero value we set show->state.depth_to_show to
 318 * the depth at which we encountered it.  When we have completed the
 319 * first pass, we will know if anything needs to be displayed if
 320 * depth_to_show > depth.  See btf_[struct,array]_show() for the
 321 * implementation of this.
 322 *
 323 * Another problem is we want to ensure the data for display is safe to
 324 * access.  To support this, the anonymous "struct {} obj" tracks the data
 325 * object and our safe copy of it.  We copy portions of the data needed
 326 * to the object "copy" buffer, but because its size is limited to
 327 * BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we
 328 * traverse larger objects for display.
 329 *
 330 * The various data type show functions all start with a call to
 331 * btf_show_start_type() which returns a pointer to the safe copy
 332 * of the data needed (or if BTF_SHOW_UNSAFE is specified, to the
 333 * raw data itself).  btf_show_obj_safe() is responsible for
 334 * using copy_from_kernel_nofault() to update the safe data if necessary
 335 * as we traverse the object's data.  skbuff-like semantics are
 336 * used:
 337 *
 338 * - obj.head points to the start of the toplevel object for display
 339 * - obj.size is the size of the toplevel object
 340 * - obj.data points to the current point in the original data at
 341 *   which our safe data starts.  obj.data will advance as we copy
 342 *   portions of the data.
 343 *
 344 * In most cases a single copy will suffice, but larger data structures
 345 * such as "struct task_struct" will require many copies.  The logic in
 346 * btf_show_obj_safe() handles the logic that determines if a new
 347 * copy_from_kernel_nofault() is needed.
 348 */
 349struct btf_show {
 350	u64 flags;
 351	void *target;	/* target of show operation (seq file, buffer) */
 352	void (*showfn)(struct btf_show *show, const char *fmt, va_list args);
 353	const struct btf *btf;
 354	/* below are used during iteration */
 355	struct {
 356		u8 depth;
 357		u8 depth_to_show;
 358		u8 depth_check;
 359		u8 array_member:1,
 360		   array_terminated:1;
 361		u16 array_encoding;
 362		u32 type_id;
 363		int status;			/* non-zero for error */
 364		const struct btf_type *type;
 365		const struct btf_member *member;
 366		char name[BTF_SHOW_NAME_SIZE];	/* space for member name/type */
 367	} state;
 368	struct {
 369		u32 size;
 370		void *head;
 371		void *data;
 372		u8 safe[BTF_SHOW_OBJ_SAFE_SIZE];
 373	} obj;
 374};
 375
 376struct btf_kind_operations {
 377	s32 (*check_meta)(struct btf_verifier_env *env,
 378			  const struct btf_type *t,
 379			  u32 meta_left);
 380	int (*resolve)(struct btf_verifier_env *env,
 381		       const struct resolve_vertex *v);
 382	int (*check_member)(struct btf_verifier_env *env,
 383			    const struct btf_type *struct_type,
 384			    const struct btf_member *member,
 385			    const struct btf_type *member_type);
 386	int (*check_kflag_member)(struct btf_verifier_env *env,
 387				  const struct btf_type *struct_type,
 388				  const struct btf_member *member,
 389				  const struct btf_type *member_type);
 390	void (*log_details)(struct btf_verifier_env *env,
 391			    const struct btf_type *t);
 392	void (*show)(const struct btf *btf, const struct btf_type *t,
 393			 u32 type_id, void *data, u8 bits_offsets,
 394			 struct btf_show *show);
 395};
 396
 397static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
 398static struct btf_type btf_void;
 399
 400static int btf_resolve(struct btf_verifier_env *env,
 401		       const struct btf_type *t, u32 type_id);
 402
 
 
 
 403static bool btf_type_is_modifier(const struct btf_type *t)
 404{
 405	/* Some of them is not strictly a C modifier
 406	 * but they are grouped into the same bucket
 407	 * for BTF concern:
 408	 *   A type (t) that refers to another
 409	 *   type through t->type AND its size cannot
 410	 *   be determined without following the t->type.
 411	 *
 412	 * ptr does not fall into this bucket
 413	 * because its size is always sizeof(void *).
 414	 */
 415	switch (BTF_INFO_KIND(t->info)) {
 416	case BTF_KIND_TYPEDEF:
 417	case BTF_KIND_VOLATILE:
 418	case BTF_KIND_CONST:
 419	case BTF_KIND_RESTRICT:
 
 420		return true;
 421	}
 422
 423	return false;
 424}
 425
 426bool btf_type_is_void(const struct btf_type *t)
 427{
 428	return t == &btf_void;
 429}
 430
 431static bool btf_type_is_fwd(const struct btf_type *t)
 432{
 433	return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
 434}
 435
 436static bool btf_type_nosize(const struct btf_type *t)
 437{
 438	return btf_type_is_void(t) || btf_type_is_fwd(t) ||
 439	       btf_type_is_func(t) || btf_type_is_func_proto(t);
 440}
 441
 442static bool btf_type_nosize_or_null(const struct btf_type *t)
 443{
 444	return !t || btf_type_nosize(t);
 445}
 446
 447static bool __btf_type_is_struct(const struct btf_type *t)
 448{
 449	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
 
 
 450}
 451
 452static bool btf_type_is_array(const struct btf_type *t)
 453{
 454	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
 455}
 456
 457static bool btf_type_is_datasec(const struct btf_type *t)
 458{
 459	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
 
 460}
 461
 462u32 btf_nr_types(const struct btf *btf)
 463{
 464	u32 total = 0;
 465
 466	while (btf) {
 467		total += btf->nr_types;
 468		btf = btf->base_btf;
 469	}
 470
 471	return total;
 472}
 473
 474s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind)
 475{
 476	const struct btf_type *t;
 477	const char *tname;
 478	u32 i, total;
 479
 480	total = btf_nr_types(btf);
 481	for (i = 1; i < total; i++) {
 482		t = btf_type_by_id(btf, i);
 483		if (BTF_INFO_KIND(t->info) != kind)
 484			continue;
 485
 486		tname = btf_name_by_offset(btf, t->name_off);
 487		if (!strcmp(tname, name))
 488			return i;
 489	}
 490
 491	return -ENOENT;
 492}
 493
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 494const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
 495					       u32 id, u32 *res_id)
 496{
 497	const struct btf_type *t = btf_type_by_id(btf, id);
 498
 499	while (btf_type_is_modifier(t)) {
 500		id = t->type;
 501		t = btf_type_by_id(btf, t->type);
 502	}
 503
 504	if (res_id)
 505		*res_id = id;
 506
 507	return t;
 508}
 509
 510const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
 511					    u32 id, u32 *res_id)
 512{
 513	const struct btf_type *t;
 514
 515	t = btf_type_skip_modifiers(btf, id, NULL);
 516	if (!btf_type_is_ptr(t))
 517		return NULL;
 518
 519	return btf_type_skip_modifiers(btf, t->type, res_id);
 520}
 521
 522const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf,
 523						 u32 id, u32 *res_id)
 524{
 525	const struct btf_type *ptype;
 526
 527	ptype = btf_type_resolve_ptr(btf, id, res_id);
 528	if (ptype && btf_type_is_func_proto(ptype))
 529		return ptype;
 530
 531	return NULL;
 532}
 533
 534/* Types that act only as a source, not sink or intermediate
 535 * type when resolving.
 536 */
 537static bool btf_type_is_resolve_source_only(const struct btf_type *t)
 538{
 539	return btf_type_is_var(t) ||
 
 540	       btf_type_is_datasec(t);
 541}
 542
 543/* What types need to be resolved?
 544 *
 545 * btf_type_is_modifier() is an obvious one.
 546 *
 547 * btf_type_is_struct() because its member refers to
 548 * another type (through member->type).
 549 *
 550 * btf_type_is_var() because the variable refers to
 551 * another type. btf_type_is_datasec() holds multiple
 552 * btf_type_is_var() types that need resolving.
 553 *
 554 * btf_type_is_array() because its element (array->type)
 555 * refers to another type.  Array can be thought of a
 556 * special case of struct while array just has the same
 557 * member-type repeated by array->nelems of times.
 558 */
 559static bool btf_type_needs_resolve(const struct btf_type *t)
 560{
 561	return btf_type_is_modifier(t) ||
 562	       btf_type_is_ptr(t) ||
 563	       btf_type_is_struct(t) ||
 564	       btf_type_is_array(t) ||
 565	       btf_type_is_var(t) ||
 
 
 566	       btf_type_is_datasec(t);
 567}
 568
 569/* t->size can be used */
 570static bool btf_type_has_size(const struct btf_type *t)
 571{
 572	switch (BTF_INFO_KIND(t->info)) {
 573	case BTF_KIND_INT:
 574	case BTF_KIND_STRUCT:
 575	case BTF_KIND_UNION:
 576	case BTF_KIND_ENUM:
 577	case BTF_KIND_DATASEC:
 578	case BTF_KIND_FLOAT:
 
 579		return true;
 580	}
 581
 582	return false;
 583}
 584
 585static const char *btf_int_encoding_str(u8 encoding)
 586{
 587	if (encoding == 0)
 588		return "(none)";
 589	else if (encoding == BTF_INT_SIGNED)
 590		return "SIGNED";
 591	else if (encoding == BTF_INT_CHAR)
 592		return "CHAR";
 593	else if (encoding == BTF_INT_BOOL)
 594		return "BOOL";
 595	else
 596		return "UNKN";
 597}
 598
 599static u32 btf_type_int(const struct btf_type *t)
 600{
 601	return *(u32 *)(t + 1);
 602}
 603
 604static const struct btf_array *btf_type_array(const struct btf_type *t)
 605{
 606	return (const struct btf_array *)(t + 1);
 607}
 608
 609static const struct btf_enum *btf_type_enum(const struct btf_type *t)
 610{
 611	return (const struct btf_enum *)(t + 1);
 612}
 613
 614static const struct btf_var *btf_type_var(const struct btf_type *t)
 615{
 616	return (const struct btf_var *)(t + 1);
 617}
 618
 
 
 
 
 
 
 
 
 
 
 619static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
 620{
 621	return kind_ops[BTF_INFO_KIND(t->info)];
 622}
 623
 624static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
 625{
 626	if (!BTF_STR_OFFSET_VALID(offset))
 627		return false;
 628
 629	while (offset < btf->start_str_off)
 630		btf = btf->base_btf;
 631
 632	offset -= btf->start_str_off;
 633	return offset < btf->hdr.str_len;
 634}
 635
 636static bool __btf_name_char_ok(char c, bool first, bool dot_ok)
 637{
 638	if ((first ? !isalpha(c) :
 639		     !isalnum(c)) &&
 640	    c != '_' &&
 641	    ((c == '.' && !dot_ok) ||
 642	      c != '.'))
 643		return false;
 644	return true;
 645}
 646
 647static const char *btf_str_by_offset(const struct btf *btf, u32 offset)
 648{
 649	while (offset < btf->start_str_off)
 650		btf = btf->base_btf;
 651
 652	offset -= btf->start_str_off;
 653	if (offset < btf->hdr.str_len)
 654		return &btf->strings[offset];
 655
 656	return NULL;
 657}
 658
 659static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok)
 660{
 661	/* offset must be valid */
 662	const char *src = btf_str_by_offset(btf, offset);
 663	const char *src_limit;
 664
 665	if (!__btf_name_char_ok(*src, true, dot_ok))
 666		return false;
 667
 668	/* set a limit on identifier length */
 669	src_limit = src + KSYM_NAME_LEN;
 670	src++;
 671	while (*src && src < src_limit) {
 672		if (!__btf_name_char_ok(*src, false, dot_ok))
 673			return false;
 674		src++;
 675	}
 676
 677	return !*src;
 678}
 679
 680/* Only C-style identifier is permitted. This can be relaxed if
 681 * necessary.
 682 */
 683static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
 684{
 685	return __btf_name_valid(btf, offset, false);
 686}
 687
 
 688static bool btf_name_valid_section(const struct btf *btf, u32 offset)
 689{
 690	return __btf_name_valid(btf, offset, true);
 
 
 
 
 
 
 
 
 
 
 
 
 
 691}
 692
 693static const char *__btf_name_by_offset(const struct btf *btf, u32 offset)
 694{
 695	const char *name;
 696
 697	if (!offset)
 698		return "(anon)";
 699
 700	name = btf_str_by_offset(btf, offset);
 701	return name ?: "(invalid-name-offset)";
 702}
 703
 704const char *btf_name_by_offset(const struct btf *btf, u32 offset)
 705{
 706	return btf_str_by_offset(btf, offset);
 707}
 708
 709const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
 710{
 711	while (type_id < btf->start_id)
 712		btf = btf->base_btf;
 713
 714	type_id -= btf->start_id;
 715	if (type_id >= btf->nr_types)
 716		return NULL;
 717	return btf->types[type_id];
 718}
 
 719
 720/*
 721 * Regular int is not a bit field and it must be either
 722 * u8/u16/u32/u64 or __int128.
 723 */
 724static bool btf_type_int_is_regular(const struct btf_type *t)
 725{
 726	u8 nr_bits, nr_bytes;
 727	u32 int_data;
 728
 729	int_data = btf_type_int(t);
 730	nr_bits = BTF_INT_BITS(int_data);
 731	nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
 732	if (BITS_PER_BYTE_MASKED(nr_bits) ||
 733	    BTF_INT_OFFSET(int_data) ||
 734	    (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
 735	     nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) &&
 736	     nr_bytes != (2 * sizeof(u64)))) {
 737		return false;
 738	}
 739
 740	return true;
 741}
 742
 743/*
 744 * Check that given struct member is a regular int with expected
 745 * offset and size.
 746 */
 747bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
 748			   const struct btf_member *m,
 749			   u32 expected_offset, u32 expected_size)
 750{
 751	const struct btf_type *t;
 752	u32 id, int_data;
 753	u8 nr_bits;
 754
 755	id = m->type;
 756	t = btf_type_id_size(btf, &id, NULL);
 757	if (!t || !btf_type_is_int(t))
 758		return false;
 759
 760	int_data = btf_type_int(t);
 761	nr_bits = BTF_INT_BITS(int_data);
 762	if (btf_type_kflag(s)) {
 763		u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset);
 764		u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset);
 765
 766		/* if kflag set, int should be a regular int and
 767		 * bit offset should be at byte boundary.
 768		 */
 769		return !bitfield_size &&
 770		       BITS_ROUNDUP_BYTES(bit_offset) == expected_offset &&
 771		       BITS_ROUNDUP_BYTES(nr_bits) == expected_size;
 772	}
 773
 774	if (BTF_INT_OFFSET(int_data) ||
 775	    BITS_PER_BYTE_MASKED(m->offset) ||
 776	    BITS_ROUNDUP_BYTES(m->offset) != expected_offset ||
 777	    BITS_PER_BYTE_MASKED(nr_bits) ||
 778	    BITS_ROUNDUP_BYTES(nr_bits) != expected_size)
 779		return false;
 780
 781	return true;
 782}
 783
 784/* Similar to btf_type_skip_modifiers() but does not skip typedefs. */
 785static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf,
 786						       u32 id)
 787{
 788	const struct btf_type *t = btf_type_by_id(btf, id);
 789
 790	while (btf_type_is_modifier(t) &&
 791	       BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
 792		t = btf_type_by_id(btf, t->type);
 793	}
 794
 795	return t;
 796}
 797
 798#define BTF_SHOW_MAX_ITER	10
 799
 800#define BTF_KIND_BIT(kind)	(1ULL << kind)
 801
 802/*
 803 * Populate show->state.name with type name information.
 804 * Format of type name is
 805 *
 806 * [.member_name = ] (type_name)
 807 */
 808static const char *btf_show_name(struct btf_show *show)
 809{
 810	/* BTF_MAX_ITER array suffixes "[]" */
 811	const char *array_suffixes = "[][][][][][][][][][]";
 812	const char *array_suffix = &array_suffixes[strlen(array_suffixes)];
 813	/* BTF_MAX_ITER pointer suffixes "*" */
 814	const char *ptr_suffixes = "**********";
 815	const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)];
 816	const char *name = NULL, *prefix = "", *parens = "";
 817	const struct btf_member *m = show->state.member;
 818	const struct btf_type *t = show->state.type;
 819	const struct btf_array *array;
 820	u32 id = show->state.type_id;
 821	const char *member = NULL;
 822	bool show_member = false;
 823	u64 kinds = 0;
 824	int i;
 825
 826	show->state.name[0] = '\0';
 827
 828	/*
 829	 * Don't show type name if we're showing an array member;
 830	 * in that case we show the array type so don't need to repeat
 831	 * ourselves for each member.
 832	 */
 833	if (show->state.array_member)
 834		return "";
 835
 836	/* Retrieve member name, if any. */
 837	if (m) {
 838		member = btf_name_by_offset(show->btf, m->name_off);
 839		show_member = strlen(member) > 0;
 840		id = m->type;
 841	}
 842
 843	/*
 844	 * Start with type_id, as we have resolved the struct btf_type *
 845	 * via btf_modifier_show() past the parent typedef to the child
 846	 * struct, int etc it is defined as.  In such cases, the type_id
 847	 * still represents the starting type while the struct btf_type *
 848	 * in our show->state points at the resolved type of the typedef.
 849	 */
 850	t = btf_type_by_id(show->btf, id);
 851	if (!t)
 852		return "";
 853
 854	/*
 855	 * The goal here is to build up the right number of pointer and
 856	 * array suffixes while ensuring the type name for a typedef
 857	 * is represented.  Along the way we accumulate a list of
 858	 * BTF kinds we have encountered, since these will inform later
 859	 * display; for example, pointer types will not require an
 860	 * opening "{" for struct, we will just display the pointer value.
 861	 *
 862	 * We also want to accumulate the right number of pointer or array
 863	 * indices in the format string while iterating until we get to
 864	 * the typedef/pointee/array member target type.
 865	 *
 866	 * We start by pointing at the end of pointer and array suffix
 867	 * strings; as we accumulate pointers and arrays we move the pointer
 868	 * or array string backwards so it will show the expected number of
 869	 * '*' or '[]' for the type.  BTF_SHOW_MAX_ITER of nesting of pointers
 870	 * and/or arrays and typedefs are supported as a precaution.
 871	 *
 872	 * We also want to get typedef name while proceeding to resolve
 873	 * type it points to so that we can add parentheses if it is a
 874	 * "typedef struct" etc.
 875	 */
 876	for (i = 0; i < BTF_SHOW_MAX_ITER; i++) {
 877
 878		switch (BTF_INFO_KIND(t->info)) {
 879		case BTF_KIND_TYPEDEF:
 880			if (!name)
 881				name = btf_name_by_offset(show->btf,
 882							       t->name_off);
 883			kinds |= BTF_KIND_BIT(BTF_KIND_TYPEDEF);
 884			id = t->type;
 885			break;
 886		case BTF_KIND_ARRAY:
 887			kinds |= BTF_KIND_BIT(BTF_KIND_ARRAY);
 888			parens = "[";
 889			if (!t)
 890				return "";
 891			array = btf_type_array(t);
 892			if (array_suffix > array_suffixes)
 893				array_suffix -= 2;
 894			id = array->type;
 895			break;
 896		case BTF_KIND_PTR:
 897			kinds |= BTF_KIND_BIT(BTF_KIND_PTR);
 898			if (ptr_suffix > ptr_suffixes)
 899				ptr_suffix -= 1;
 900			id = t->type;
 901			break;
 902		default:
 903			id = 0;
 904			break;
 905		}
 906		if (!id)
 907			break;
 908		t = btf_type_skip_qualifiers(show->btf, id);
 909	}
 910	/* We may not be able to represent this type; bail to be safe */
 911	if (i == BTF_SHOW_MAX_ITER)
 912		return "";
 913
 914	if (!name)
 915		name = btf_name_by_offset(show->btf, t->name_off);
 916
 917	switch (BTF_INFO_KIND(t->info)) {
 918	case BTF_KIND_STRUCT:
 919	case BTF_KIND_UNION:
 920		prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ?
 921			 "struct" : "union";
 922		/* if it's an array of struct/union, parens is already set */
 923		if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY))))
 924			parens = "{";
 925		break;
 926	case BTF_KIND_ENUM:
 
 927		prefix = "enum";
 928		break;
 929	default:
 930		break;
 931	}
 932
 933	/* pointer does not require parens */
 934	if (kinds & BTF_KIND_BIT(BTF_KIND_PTR))
 935		parens = "";
 936	/* typedef does not require struct/union/enum prefix */
 937	if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF))
 938		prefix = "";
 939
 940	if (!name)
 941		name = "";
 942
 943	/* Even if we don't want type name info, we want parentheses etc */
 944	if (show->flags & BTF_SHOW_NONAME)
 945		snprintf(show->state.name, sizeof(show->state.name), "%s",
 946			 parens);
 947	else
 948		snprintf(show->state.name, sizeof(show->state.name),
 949			 "%s%s%s(%s%s%s%s%s%s)%s",
 950			 /* first 3 strings comprise ".member = " */
 951			 show_member ? "." : "",
 952			 show_member ? member : "",
 953			 show_member ? " = " : "",
 954			 /* ...next is our prefix (struct, enum, etc) */
 955			 prefix,
 956			 strlen(prefix) > 0 && strlen(name) > 0 ? " " : "",
 957			 /* ...this is the type name itself */
 958			 name,
 959			 /* ...suffixed by the appropriate '*', '[]' suffixes */
 960			 strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix,
 961			 array_suffix, parens);
 962
 963	return show->state.name;
 964}
 965
 966static const char *__btf_show_indent(struct btf_show *show)
 967{
 968	const char *indents = "                                ";
 969	const char *indent = &indents[strlen(indents)];
 970
 971	if ((indent - show->state.depth) >= indents)
 972		return indent - show->state.depth;
 973	return indents;
 974}
 975
 976static const char *btf_show_indent(struct btf_show *show)
 977{
 978	return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show);
 979}
 980
 981static const char *btf_show_newline(struct btf_show *show)
 982{
 983	return show->flags & BTF_SHOW_COMPACT ? "" : "\n";
 984}
 985
 986static const char *btf_show_delim(struct btf_show *show)
 987{
 988	if (show->state.depth == 0)
 989		return "";
 990
 991	if ((show->flags & BTF_SHOW_COMPACT) && show->state.type &&
 992		BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION)
 993		return "|";
 994
 995	return ",";
 996}
 997
 998__printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...)
 999{
1000	va_list args;
1001
1002	if (!show->state.depth_check) {
1003		va_start(args, fmt);
1004		show->showfn(show, fmt, args);
1005		va_end(args);
1006	}
1007}
1008
1009/* Macros are used here as btf_show_type_value[s]() prepends and appends
1010 * format specifiers to the format specifier passed in; these do the work of
1011 * adding indentation, delimiters etc while the caller simply has to specify
1012 * the type value(s) in the format specifier + value(s).
1013 */
1014#define btf_show_type_value(show, fmt, value)				       \
1015	do {								       \
1016		if ((value) != 0 || (show->flags & BTF_SHOW_ZERO) ||	       \
 
1017		    show->state.depth == 0) {				       \
1018			btf_show(show, "%s%s" fmt "%s%s",		       \
1019				 btf_show_indent(show),			       \
1020				 btf_show_name(show),			       \
1021				 value, btf_show_delim(show),		       \
1022				 btf_show_newline(show));		       \
1023			if (show->state.depth > show->state.depth_to_show)     \
1024				show->state.depth_to_show = show->state.depth; \
1025		}							       \
1026	} while (0)
1027
1028#define btf_show_type_values(show, fmt, ...)				       \
1029	do {								       \
1030		btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show),       \
1031			 btf_show_name(show),				       \
1032			 __VA_ARGS__, btf_show_delim(show),		       \
1033			 btf_show_newline(show));			       \
1034		if (show->state.depth > show->state.depth_to_show)	       \
1035			show->state.depth_to_show = show->state.depth;	       \
1036	} while (0)
1037
1038/* How much is left to copy to safe buffer after @data? */
1039static int btf_show_obj_size_left(struct btf_show *show, void *data)
1040{
1041	return show->obj.head + show->obj.size - data;
1042}
1043
1044/* Is object pointed to by @data of @size already copied to our safe buffer? */
1045static bool btf_show_obj_is_safe(struct btf_show *show, void *data, int size)
1046{
1047	return data >= show->obj.data &&
1048	       (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE);
1049}
1050
1051/*
1052 * If object pointed to by @data of @size falls within our safe buffer, return
1053 * the equivalent pointer to the same safe data.  Assumes
1054 * copy_from_kernel_nofault() has already happened and our safe buffer is
1055 * populated.
1056 */
1057static void *__btf_show_obj_safe(struct btf_show *show, void *data, int size)
1058{
1059	if (btf_show_obj_is_safe(show, data, size))
1060		return show->obj.safe + (data - show->obj.data);
1061	return NULL;
1062}
1063
1064/*
1065 * Return a safe-to-access version of data pointed to by @data.
1066 * We do this by copying the relevant amount of information
1067 * to the struct btf_show obj.safe buffer using copy_from_kernel_nofault().
1068 *
1069 * If BTF_SHOW_UNSAFE is specified, just return data as-is; no
1070 * safe copy is needed.
1071 *
1072 * Otherwise we need to determine if we have the required amount
1073 * of data (determined by the @data pointer and the size of the
1074 * largest base type we can encounter (represented by
1075 * BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures
1076 * that we will be able to print some of the current object,
1077 * and if more is needed a copy will be triggered.
1078 * Some objects such as structs will not fit into the buffer;
1079 * in such cases additional copies when we iterate over their
1080 * members may be needed.
1081 *
1082 * btf_show_obj_safe() is used to return a safe buffer for
1083 * btf_show_start_type(); this ensures that as we recurse into
1084 * nested types we always have safe data for the given type.
1085 * This approach is somewhat wasteful; it's possible for example
1086 * that when iterating over a large union we'll end up copying the
1087 * same data repeatedly, but the goal is safety not performance.
1088 * We use stack data as opposed to per-CPU buffers because the
1089 * iteration over a type can take some time, and preemption handling
1090 * would greatly complicate use of the safe buffer.
1091 */
1092static void *btf_show_obj_safe(struct btf_show *show,
1093			       const struct btf_type *t,
1094			       void *data)
1095{
1096	const struct btf_type *rt;
1097	int size_left, size;
1098	void *safe = NULL;
1099
1100	if (show->flags & BTF_SHOW_UNSAFE)
1101		return data;
1102
1103	rt = btf_resolve_size(show->btf, t, &size);
1104	if (IS_ERR(rt)) {
1105		show->state.status = PTR_ERR(rt);
1106		return NULL;
1107	}
1108
1109	/*
1110	 * Is this toplevel object? If so, set total object size and
1111	 * initialize pointers.  Otherwise check if we still fall within
1112	 * our safe object data.
1113	 */
1114	if (show->state.depth == 0) {
1115		show->obj.size = size;
1116		show->obj.head = data;
1117	} else {
1118		/*
1119		 * If the size of the current object is > our remaining
1120		 * safe buffer we _may_ need to do a new copy.  However
1121		 * consider the case of a nested struct; it's size pushes
1122		 * us over the safe buffer limit, but showing any individual
1123		 * struct members does not.  In such cases, we don't need
1124		 * to initiate a fresh copy yet; however we definitely need
1125		 * at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left
1126		 * in our buffer, regardless of the current object size.
1127		 * The logic here is that as we resolve types we will
1128		 * hit a base type at some point, and we need to be sure
1129		 * the next chunk of data is safely available to display
1130		 * that type info safely.  We cannot rely on the size of
1131		 * the current object here because it may be much larger
1132		 * than our current buffer (e.g. task_struct is 8k).
1133		 * All we want to do here is ensure that we can print the
1134		 * next basic type, which we can if either
1135		 * - the current type size is within the safe buffer; or
1136		 * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in
1137		 *   the safe buffer.
1138		 */
1139		safe = __btf_show_obj_safe(show, data,
1140					   min(size,
1141					       BTF_SHOW_OBJ_BASE_TYPE_SIZE));
1142	}
1143
1144	/*
1145	 * We need a new copy to our safe object, either because we haven't
1146	 * yet copied and are initializing safe data, or because the data
1147	 * we want falls outside the boundaries of the safe object.
1148	 */
1149	if (!safe) {
1150		size_left = btf_show_obj_size_left(show, data);
1151		if (size_left > BTF_SHOW_OBJ_SAFE_SIZE)
1152			size_left = BTF_SHOW_OBJ_SAFE_SIZE;
1153		show->state.status = copy_from_kernel_nofault(show->obj.safe,
1154							      data, size_left);
1155		if (!show->state.status) {
1156			show->obj.data = data;
1157			safe = show->obj.safe;
1158		}
1159	}
1160
1161	return safe;
1162}
1163
1164/*
1165 * Set the type we are starting to show and return a safe data pointer
1166 * to be used for showing the associated data.
1167 */
1168static void *btf_show_start_type(struct btf_show *show,
1169				 const struct btf_type *t,
1170				 u32 type_id, void *data)
1171{
1172	show->state.type = t;
1173	show->state.type_id = type_id;
1174	show->state.name[0] = '\0';
1175
1176	return btf_show_obj_safe(show, t, data);
1177}
1178
1179static void btf_show_end_type(struct btf_show *show)
1180{
1181	show->state.type = NULL;
1182	show->state.type_id = 0;
1183	show->state.name[0] = '\0';
1184}
1185
1186static void *btf_show_start_aggr_type(struct btf_show *show,
1187				      const struct btf_type *t,
1188				      u32 type_id, void *data)
1189{
1190	void *safe_data = btf_show_start_type(show, t, type_id, data);
1191
1192	if (!safe_data)
1193		return safe_data;
1194
1195	btf_show(show, "%s%s%s", btf_show_indent(show),
1196		 btf_show_name(show),
1197		 btf_show_newline(show));
1198	show->state.depth++;
1199	return safe_data;
1200}
1201
1202static void btf_show_end_aggr_type(struct btf_show *show,
1203				   const char *suffix)
1204{
1205	show->state.depth--;
1206	btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix,
1207		 btf_show_delim(show), btf_show_newline(show));
1208	btf_show_end_type(show);
1209}
1210
1211static void btf_show_start_member(struct btf_show *show,
1212				  const struct btf_member *m)
1213{
1214	show->state.member = m;
1215}
1216
1217static void btf_show_start_array_member(struct btf_show *show)
1218{
1219	show->state.array_member = 1;
1220	btf_show_start_member(show, NULL);
1221}
1222
1223static void btf_show_end_member(struct btf_show *show)
1224{
1225	show->state.member = NULL;
1226}
1227
1228static void btf_show_end_array_member(struct btf_show *show)
1229{
1230	show->state.array_member = 0;
1231	btf_show_end_member(show);
1232}
1233
1234static void *btf_show_start_array_type(struct btf_show *show,
1235				       const struct btf_type *t,
1236				       u32 type_id,
1237				       u16 array_encoding,
1238				       void *data)
1239{
1240	show->state.array_encoding = array_encoding;
1241	show->state.array_terminated = 0;
1242	return btf_show_start_aggr_type(show, t, type_id, data);
1243}
1244
1245static void btf_show_end_array_type(struct btf_show *show)
1246{
1247	show->state.array_encoding = 0;
1248	show->state.array_terminated = 0;
1249	btf_show_end_aggr_type(show, "]");
1250}
1251
1252static void *btf_show_start_struct_type(struct btf_show *show,
1253					const struct btf_type *t,
1254					u32 type_id,
1255					void *data)
1256{
1257	return btf_show_start_aggr_type(show, t, type_id, data);
1258}
1259
1260static void btf_show_end_struct_type(struct btf_show *show)
1261{
1262	btf_show_end_aggr_type(show, "}");
1263}
1264
1265__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
1266					      const char *fmt, ...)
1267{
1268	va_list args;
1269
1270	va_start(args, fmt);
1271	bpf_verifier_vlog(log, fmt, args);
1272	va_end(args);
1273}
1274
1275__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
1276					    const char *fmt, ...)
1277{
1278	struct bpf_verifier_log *log = &env->log;
1279	va_list args;
1280
1281	if (!bpf_verifier_log_needed(log))
1282		return;
1283
1284	va_start(args, fmt);
1285	bpf_verifier_vlog(log, fmt, args);
1286	va_end(args);
1287}
1288
1289__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
1290						   const struct btf_type *t,
1291						   bool log_details,
1292						   const char *fmt, ...)
1293{
1294	struct bpf_verifier_log *log = &env->log;
1295	u8 kind = BTF_INFO_KIND(t->info);
1296	struct btf *btf = env->btf;
1297	va_list args;
1298
1299	if (!bpf_verifier_log_needed(log))
1300		return;
1301
1302	/* btf verifier prints all types it is processing via
1303	 * btf_verifier_log_type(..., fmt = NULL).
1304	 * Skip those prints for in-kernel BTF verification.
1305	 */
1306	if (log->level == BPF_LOG_KERNEL && !fmt)
1307		return;
 
 
 
 
 
 
1308
1309	__btf_verifier_log(log, "[%u] %s %s%s",
1310			   env->log_type_id,
1311			   btf_kind_str[kind],
1312			   __btf_name_by_offset(btf, t->name_off),
1313			   log_details ? " " : "");
1314
1315	if (log_details)
1316		btf_type_ops(t)->log_details(env, t);
1317
1318	if (fmt && *fmt) {
1319		__btf_verifier_log(log, " ");
1320		va_start(args, fmt);
1321		bpf_verifier_vlog(log, fmt, args);
1322		va_end(args);
1323	}
1324
1325	__btf_verifier_log(log, "\n");
1326}
1327
1328#define btf_verifier_log_type(env, t, ...) \
1329	__btf_verifier_log_type((env), (t), true, __VA_ARGS__)
1330#define btf_verifier_log_basic(env, t, ...) \
1331	__btf_verifier_log_type((env), (t), false, __VA_ARGS__)
1332
1333__printf(4, 5)
1334static void btf_verifier_log_member(struct btf_verifier_env *env,
1335				    const struct btf_type *struct_type,
1336				    const struct btf_member *member,
1337				    const char *fmt, ...)
1338{
1339	struct bpf_verifier_log *log = &env->log;
1340	struct btf *btf = env->btf;
1341	va_list args;
1342
1343	if (!bpf_verifier_log_needed(log))
1344		return;
1345
1346	if (log->level == BPF_LOG_KERNEL && !fmt)
1347		return;
 
 
 
 
 
 
 
1348	/* The CHECK_META phase already did a btf dump.
1349	 *
1350	 * If member is logged again, it must hit an error in
1351	 * parsing this member.  It is useful to print out which
1352	 * struct this member belongs to.
1353	 */
1354	if (env->phase != CHECK_META)
1355		btf_verifier_log_type(env, struct_type, NULL);
1356
1357	if (btf_type_kflag(struct_type))
1358		__btf_verifier_log(log,
1359				   "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
1360				   __btf_name_by_offset(btf, member->name_off),
1361				   member->type,
1362				   BTF_MEMBER_BITFIELD_SIZE(member->offset),
1363				   BTF_MEMBER_BIT_OFFSET(member->offset));
1364	else
1365		__btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
1366				   __btf_name_by_offset(btf, member->name_off),
1367				   member->type, member->offset);
1368
1369	if (fmt && *fmt) {
1370		__btf_verifier_log(log, " ");
1371		va_start(args, fmt);
1372		bpf_verifier_vlog(log, fmt, args);
1373		va_end(args);
1374	}
1375
1376	__btf_verifier_log(log, "\n");
1377}
1378
1379__printf(4, 5)
1380static void btf_verifier_log_vsi(struct btf_verifier_env *env,
1381				 const struct btf_type *datasec_type,
1382				 const struct btf_var_secinfo *vsi,
1383				 const char *fmt, ...)
1384{
1385	struct bpf_verifier_log *log = &env->log;
1386	va_list args;
1387
1388	if (!bpf_verifier_log_needed(log))
1389		return;
1390	if (log->level == BPF_LOG_KERNEL && !fmt)
1391		return;
1392	if (env->phase != CHECK_META)
1393		btf_verifier_log_type(env, datasec_type, NULL);
1394
1395	__btf_verifier_log(log, "\t type_id=%u offset=%u size=%u",
1396			   vsi->type, vsi->offset, vsi->size);
1397	if (fmt && *fmt) {
1398		__btf_verifier_log(log, " ");
1399		va_start(args, fmt);
1400		bpf_verifier_vlog(log, fmt, args);
1401		va_end(args);
1402	}
1403
1404	__btf_verifier_log(log, "\n");
1405}
1406
1407static void btf_verifier_log_hdr(struct btf_verifier_env *env,
1408				 u32 btf_data_size)
1409{
1410	struct bpf_verifier_log *log = &env->log;
1411	const struct btf *btf = env->btf;
1412	const struct btf_header *hdr;
1413
1414	if (!bpf_verifier_log_needed(log))
1415		return;
1416
1417	if (log->level == BPF_LOG_KERNEL)
1418		return;
1419	hdr = &btf->hdr;
1420	__btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
1421	__btf_verifier_log(log, "version: %u\n", hdr->version);
1422	__btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
1423	__btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
1424	__btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
1425	__btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
1426	__btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
1427	__btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
1428	__btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
1429}
1430
1431static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
1432{
1433	struct btf *btf = env->btf;
1434
1435	if (btf->types_size == btf->nr_types) {
1436		/* Expand 'types' array */
1437
1438		struct btf_type **new_types;
1439		u32 expand_by, new_size;
1440
1441		if (btf->start_id + btf->types_size == BTF_MAX_TYPE) {
1442			btf_verifier_log(env, "Exceeded max num of types");
1443			return -E2BIG;
1444		}
1445
1446		expand_by = max_t(u32, btf->types_size >> 2, 16);
1447		new_size = min_t(u32, BTF_MAX_TYPE,
1448				 btf->types_size + expand_by);
1449
1450		new_types = kvcalloc(new_size, sizeof(*new_types),
1451				     GFP_KERNEL | __GFP_NOWARN);
1452		if (!new_types)
1453			return -ENOMEM;
1454
1455		if (btf->nr_types == 0) {
1456			if (!btf->base_btf) {
1457				/* lazily init VOID type */
1458				new_types[0] = &btf_void;
1459				btf->nr_types++;
1460			}
1461		} else {
1462			memcpy(new_types, btf->types,
1463			       sizeof(*btf->types) * btf->nr_types);
1464		}
1465
1466		kvfree(btf->types);
1467		btf->types = new_types;
1468		btf->types_size = new_size;
1469	}
1470
1471	btf->types[btf->nr_types++] = t;
1472
1473	return 0;
1474}
1475
1476static int btf_alloc_id(struct btf *btf)
1477{
1478	int id;
1479
1480	idr_preload(GFP_KERNEL);
1481	spin_lock_bh(&btf_idr_lock);
1482	id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
1483	if (id > 0)
1484		btf->id = id;
1485	spin_unlock_bh(&btf_idr_lock);
1486	idr_preload_end();
1487
1488	if (WARN_ON_ONCE(!id))
1489		return -ENOSPC;
1490
1491	return id > 0 ? 0 : id;
1492}
1493
1494static void btf_free_id(struct btf *btf)
1495{
1496	unsigned long flags;
1497
1498	/*
1499	 * In map-in-map, calling map_delete_elem() on outer
1500	 * map will call bpf_map_put on the inner map.
1501	 * It will then eventually call btf_free_id()
1502	 * on the inner map.  Some of the map_delete_elem()
1503	 * implementation may have irq disabled, so
1504	 * we need to use the _irqsave() version instead
1505	 * of the _bh() version.
1506	 */
1507	spin_lock_irqsave(&btf_idr_lock, flags);
1508	idr_remove(&btf_idr, btf->id);
1509	spin_unlock_irqrestore(&btf_idr_lock, flags);
1510}
1511
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1512static void btf_free(struct btf *btf)
1513{
 
 
 
 
1514	kvfree(btf->types);
1515	kvfree(btf->resolved_sizes);
1516	kvfree(btf->resolved_ids);
1517	kvfree(btf->data);
1518	kfree(btf);
1519}
1520
1521static void btf_free_rcu(struct rcu_head *rcu)
1522{
1523	struct btf *btf = container_of(rcu, struct btf, rcu);
1524
1525	btf_free(btf);
1526}
1527
 
 
 
 
 
1528void btf_get(struct btf *btf)
1529{
1530	refcount_inc(&btf->refcnt);
1531}
1532
1533void btf_put(struct btf *btf)
1534{
1535	if (btf && refcount_dec_and_test(&btf->refcnt)) {
1536		btf_free_id(btf);
1537		call_rcu(&btf->rcu, btf_free_rcu);
1538	}
1539}
1540
1541static int env_resolve_init(struct btf_verifier_env *env)
1542{
1543	struct btf *btf = env->btf;
1544	u32 nr_types = btf->nr_types;
1545	u32 *resolved_sizes = NULL;
1546	u32 *resolved_ids = NULL;
1547	u8 *visit_states = NULL;
1548
1549	resolved_sizes = kvcalloc(nr_types, sizeof(*resolved_sizes),
1550				  GFP_KERNEL | __GFP_NOWARN);
1551	if (!resolved_sizes)
1552		goto nomem;
1553
1554	resolved_ids = kvcalloc(nr_types, sizeof(*resolved_ids),
1555				GFP_KERNEL | __GFP_NOWARN);
1556	if (!resolved_ids)
1557		goto nomem;
1558
1559	visit_states = kvcalloc(nr_types, sizeof(*visit_states),
1560				GFP_KERNEL | __GFP_NOWARN);
1561	if (!visit_states)
1562		goto nomem;
1563
1564	btf->resolved_sizes = resolved_sizes;
1565	btf->resolved_ids = resolved_ids;
1566	env->visit_states = visit_states;
1567
1568	return 0;
1569
1570nomem:
1571	kvfree(resolved_sizes);
1572	kvfree(resolved_ids);
1573	kvfree(visit_states);
1574	return -ENOMEM;
1575}
1576
1577static void btf_verifier_env_free(struct btf_verifier_env *env)
1578{
1579	kvfree(env->visit_states);
1580	kfree(env);
1581}
1582
1583static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
1584				     const struct btf_type *next_type)
1585{
1586	switch (env->resolve_mode) {
1587	case RESOLVE_TBD:
1588		/* int, enum or void is a sink */
1589		return !btf_type_needs_resolve(next_type);
1590	case RESOLVE_PTR:
1591		/* int, enum, void, struct, array, func or func_proto is a sink
1592		 * for ptr
1593		 */
1594		return !btf_type_is_modifier(next_type) &&
1595			!btf_type_is_ptr(next_type);
1596	case RESOLVE_STRUCT_OR_ARRAY:
1597		/* int, enum, void, ptr, func or func_proto is a sink
1598		 * for struct and array
1599		 */
1600		return !btf_type_is_modifier(next_type) &&
1601			!btf_type_is_array(next_type) &&
1602			!btf_type_is_struct(next_type);
1603	default:
1604		BUG();
1605	}
1606}
1607
1608static bool env_type_is_resolved(const struct btf_verifier_env *env,
1609				 u32 type_id)
1610{
1611	/* base BTF types should be resolved by now */
1612	if (type_id < env->btf->start_id)
1613		return true;
1614
1615	return env->visit_states[type_id - env->btf->start_id] == RESOLVED;
1616}
1617
1618static int env_stack_push(struct btf_verifier_env *env,
1619			  const struct btf_type *t, u32 type_id)
1620{
1621	const struct btf *btf = env->btf;
1622	struct resolve_vertex *v;
1623
1624	if (env->top_stack == MAX_RESOLVE_DEPTH)
1625		return -E2BIG;
1626
1627	if (type_id < btf->start_id
1628	    || env->visit_states[type_id - btf->start_id] != NOT_VISITED)
1629		return -EEXIST;
1630
1631	env->visit_states[type_id - btf->start_id] = VISITED;
1632
1633	v = &env->stack[env->top_stack++];
1634	v->t = t;
1635	v->type_id = type_id;
1636	v->next_member = 0;
1637
1638	if (env->resolve_mode == RESOLVE_TBD) {
1639		if (btf_type_is_ptr(t))
1640			env->resolve_mode = RESOLVE_PTR;
1641		else if (btf_type_is_struct(t) || btf_type_is_array(t))
1642			env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
1643	}
1644
1645	return 0;
1646}
1647
1648static void env_stack_set_next_member(struct btf_verifier_env *env,
1649				      u16 next_member)
1650{
1651	env->stack[env->top_stack - 1].next_member = next_member;
1652}
1653
1654static void env_stack_pop_resolved(struct btf_verifier_env *env,
1655				   u32 resolved_type_id,
1656				   u32 resolved_size)
1657{
1658	u32 type_id = env->stack[--(env->top_stack)].type_id;
1659	struct btf *btf = env->btf;
1660
1661	type_id -= btf->start_id; /* adjust to local type id */
1662	btf->resolved_sizes[type_id] = resolved_size;
1663	btf->resolved_ids[type_id] = resolved_type_id;
1664	env->visit_states[type_id] = RESOLVED;
1665}
1666
1667static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
1668{
1669	return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
1670}
1671
1672/* Resolve the size of a passed-in "type"
1673 *
1674 * type: is an array (e.g. u32 array[x][y])
1675 * return type: type "u32[x][y]", i.e. BTF_KIND_ARRAY,
1676 * *type_size: (x * y * sizeof(u32)).  Hence, *type_size always
1677 *             corresponds to the return type.
1678 * *elem_type: u32
1679 * *elem_id: id of u32
1680 * *total_nelems: (x * y).  Hence, individual elem size is
1681 *                (*type_size / *total_nelems)
1682 * *type_id: id of type if it's changed within the function, 0 if not
1683 *
1684 * type: is not an array (e.g. const struct X)
1685 * return type: type "struct X"
1686 * *type_size: sizeof(struct X)
1687 * *elem_type: same as return type ("struct X")
1688 * *elem_id: 0
1689 * *total_nelems: 1
1690 * *type_id: id of type if it's changed within the function, 0 if not
1691 */
1692static const struct btf_type *
1693__btf_resolve_size(const struct btf *btf, const struct btf_type *type,
1694		   u32 *type_size, const struct btf_type **elem_type,
1695		   u32 *elem_id, u32 *total_nelems, u32 *type_id)
1696{
1697	const struct btf_type *array_type = NULL;
1698	const struct btf_array *array = NULL;
1699	u32 i, size, nelems = 1, id = 0;
1700
1701	for (i = 0; i < MAX_RESOLVE_DEPTH; i++) {
1702		switch (BTF_INFO_KIND(type->info)) {
1703		/* type->size can be used */
1704		case BTF_KIND_INT:
1705		case BTF_KIND_STRUCT:
1706		case BTF_KIND_UNION:
1707		case BTF_KIND_ENUM:
1708		case BTF_KIND_FLOAT:
 
1709			size = type->size;
1710			goto resolved;
1711
1712		case BTF_KIND_PTR:
1713			size = sizeof(void *);
1714			goto resolved;
1715
1716		/* Modifiers */
1717		case BTF_KIND_TYPEDEF:
1718		case BTF_KIND_VOLATILE:
1719		case BTF_KIND_CONST:
1720		case BTF_KIND_RESTRICT:
 
1721			id = type->type;
1722			type = btf_type_by_id(btf, type->type);
1723			break;
1724
1725		case BTF_KIND_ARRAY:
1726			if (!array_type)
1727				array_type = type;
1728			array = btf_type_array(type);
1729			if (nelems && array->nelems > U32_MAX / nelems)
1730				return ERR_PTR(-EINVAL);
1731			nelems *= array->nelems;
1732			type = btf_type_by_id(btf, array->type);
1733			break;
1734
1735		/* type without size */
1736		default:
1737			return ERR_PTR(-EINVAL);
1738		}
1739	}
1740
1741	return ERR_PTR(-EINVAL);
1742
1743resolved:
1744	if (nelems && size > U32_MAX / nelems)
1745		return ERR_PTR(-EINVAL);
1746
1747	*type_size = nelems * size;
1748	if (total_nelems)
1749		*total_nelems = nelems;
1750	if (elem_type)
1751		*elem_type = type;
1752	if (elem_id)
1753		*elem_id = array ? array->type : 0;
1754	if (type_id && id)
1755		*type_id = id;
1756
1757	return array_type ? : type;
1758}
1759
1760const struct btf_type *
1761btf_resolve_size(const struct btf *btf, const struct btf_type *type,
1762		 u32 *type_size)
1763{
1764	return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
1765}
1766
1767static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id)
1768{
1769	while (type_id < btf->start_id)
1770		btf = btf->base_btf;
1771
1772	return btf->resolved_ids[type_id - btf->start_id];
1773}
1774
1775/* The input param "type_id" must point to a needs_resolve type */
1776static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
1777						  u32 *type_id)
1778{
1779	*type_id = btf_resolved_type_id(btf, *type_id);
1780	return btf_type_by_id(btf, *type_id);
1781}
1782
1783static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id)
1784{
1785	while (type_id < btf->start_id)
1786		btf = btf->base_btf;
1787
1788	return btf->resolved_sizes[type_id - btf->start_id];
1789}
1790
1791const struct btf_type *btf_type_id_size(const struct btf *btf,
1792					u32 *type_id, u32 *ret_size)
1793{
1794	const struct btf_type *size_type;
1795	u32 size_type_id = *type_id;
1796	u32 size = 0;
1797
1798	size_type = btf_type_by_id(btf, size_type_id);
1799	if (btf_type_nosize_or_null(size_type))
1800		return NULL;
1801
1802	if (btf_type_has_size(size_type)) {
1803		size = size_type->size;
1804	} else if (btf_type_is_array(size_type)) {
1805		size = btf_resolved_type_size(btf, size_type_id);
1806	} else if (btf_type_is_ptr(size_type)) {
1807		size = sizeof(void *);
1808	} else {
1809		if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) &&
1810				 !btf_type_is_var(size_type)))
1811			return NULL;
1812
1813		size_type_id = btf_resolved_type_id(btf, size_type_id);
1814		size_type = btf_type_by_id(btf, size_type_id);
1815		if (btf_type_nosize_or_null(size_type))
1816			return NULL;
1817		else if (btf_type_has_size(size_type))
1818			size = size_type->size;
1819		else if (btf_type_is_array(size_type))
1820			size = btf_resolved_type_size(btf, size_type_id);
1821		else if (btf_type_is_ptr(size_type))
1822			size = sizeof(void *);
1823		else
1824			return NULL;
1825	}
1826
1827	*type_id = size_type_id;
1828	if (ret_size)
1829		*ret_size = size;
1830
1831	return size_type;
1832}
1833
1834static int btf_df_check_member(struct btf_verifier_env *env,
1835			       const struct btf_type *struct_type,
1836			       const struct btf_member *member,
1837			       const struct btf_type *member_type)
1838{
1839	btf_verifier_log_basic(env, struct_type,
1840			       "Unsupported check_member");
1841	return -EINVAL;
1842}
1843
1844static int btf_df_check_kflag_member(struct btf_verifier_env *env,
1845				     const struct btf_type *struct_type,
1846				     const struct btf_member *member,
1847				     const struct btf_type *member_type)
1848{
1849	btf_verifier_log_basic(env, struct_type,
1850			       "Unsupported check_kflag_member");
1851	return -EINVAL;
1852}
1853
1854/* Used for ptr, array struct/union and float type members.
1855 * int, enum and modifier types have their specific callback functions.
1856 */
1857static int btf_generic_check_kflag_member(struct btf_verifier_env *env,
1858					  const struct btf_type *struct_type,
1859					  const struct btf_member *member,
1860					  const struct btf_type *member_type)
1861{
1862	if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) {
1863		btf_verifier_log_member(env, struct_type, member,
1864					"Invalid member bitfield_size");
1865		return -EINVAL;
1866	}
1867
1868	/* bitfield size is 0, so member->offset represents bit offset only.
1869	 * It is safe to call non kflag check_member variants.
1870	 */
1871	return btf_type_ops(member_type)->check_member(env, struct_type,
1872						       member,
1873						       member_type);
1874}
1875
1876static int btf_df_resolve(struct btf_verifier_env *env,
1877			  const struct resolve_vertex *v)
1878{
1879	btf_verifier_log_basic(env, v->t, "Unsupported resolve");
1880	return -EINVAL;
1881}
1882
1883static void btf_df_show(const struct btf *btf, const struct btf_type *t,
1884			u32 type_id, void *data, u8 bits_offsets,
1885			struct btf_show *show)
1886{
1887	btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
1888}
1889
1890static int btf_int_check_member(struct btf_verifier_env *env,
1891				const struct btf_type *struct_type,
1892				const struct btf_member *member,
1893				const struct btf_type *member_type)
1894{
1895	u32 int_data = btf_type_int(member_type);
1896	u32 struct_bits_off = member->offset;
1897	u32 struct_size = struct_type->size;
1898	u32 nr_copy_bits;
1899	u32 bytes_offset;
1900
1901	if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
1902		btf_verifier_log_member(env, struct_type, member,
1903					"bits_offset exceeds U32_MAX");
1904		return -EINVAL;
1905	}
1906
1907	struct_bits_off += BTF_INT_OFFSET(int_data);
1908	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1909	nr_copy_bits = BTF_INT_BITS(int_data) +
1910		BITS_PER_BYTE_MASKED(struct_bits_off);
1911
1912	if (nr_copy_bits > BITS_PER_U128) {
1913		btf_verifier_log_member(env, struct_type, member,
1914					"nr_copy_bits exceeds 128");
1915		return -EINVAL;
1916	}
1917
1918	if (struct_size < bytes_offset ||
1919	    struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1920		btf_verifier_log_member(env, struct_type, member,
1921					"Member exceeds struct_size");
1922		return -EINVAL;
1923	}
1924
1925	return 0;
1926}
1927
1928static int btf_int_check_kflag_member(struct btf_verifier_env *env,
1929				      const struct btf_type *struct_type,
1930				      const struct btf_member *member,
1931				      const struct btf_type *member_type)
1932{
1933	u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset;
1934	u32 int_data = btf_type_int(member_type);
1935	u32 struct_size = struct_type->size;
1936	u32 nr_copy_bits;
1937
1938	/* a regular int type is required for the kflag int member */
1939	if (!btf_type_int_is_regular(member_type)) {
1940		btf_verifier_log_member(env, struct_type, member,
1941					"Invalid member base type");
1942		return -EINVAL;
1943	}
1944
1945	/* check sanity of bitfield size */
1946	nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
1947	struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
1948	nr_int_data_bits = BTF_INT_BITS(int_data);
1949	if (!nr_bits) {
1950		/* Not a bitfield member, member offset must be at byte
1951		 * boundary.
1952		 */
1953		if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1954			btf_verifier_log_member(env, struct_type, member,
1955						"Invalid member offset");
1956			return -EINVAL;
1957		}
1958
1959		nr_bits = nr_int_data_bits;
1960	} else if (nr_bits > nr_int_data_bits) {
1961		btf_verifier_log_member(env, struct_type, member,
1962					"Invalid member bitfield_size");
1963		return -EINVAL;
1964	}
1965
1966	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1967	nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off);
1968	if (nr_copy_bits > BITS_PER_U128) {
1969		btf_verifier_log_member(env, struct_type, member,
1970					"nr_copy_bits exceeds 128");
1971		return -EINVAL;
1972	}
1973
1974	if (struct_size < bytes_offset ||
1975	    struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1976		btf_verifier_log_member(env, struct_type, member,
1977					"Member exceeds struct_size");
1978		return -EINVAL;
1979	}
1980
1981	return 0;
1982}
1983
1984static s32 btf_int_check_meta(struct btf_verifier_env *env,
1985			      const struct btf_type *t,
1986			      u32 meta_left)
1987{
1988	u32 int_data, nr_bits, meta_needed = sizeof(int_data);
1989	u16 encoding;
1990
1991	if (meta_left < meta_needed) {
1992		btf_verifier_log_basic(env, t,
1993				       "meta_left:%u meta_needed:%u",
1994				       meta_left, meta_needed);
1995		return -EINVAL;
1996	}
1997
1998	if (btf_type_vlen(t)) {
1999		btf_verifier_log_type(env, t, "vlen != 0");
2000		return -EINVAL;
2001	}
2002
2003	if (btf_type_kflag(t)) {
2004		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2005		return -EINVAL;
2006	}
2007
2008	int_data = btf_type_int(t);
2009	if (int_data & ~BTF_INT_MASK) {
2010		btf_verifier_log_basic(env, t, "Invalid int_data:%x",
2011				       int_data);
2012		return -EINVAL;
2013	}
2014
2015	nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
2016
2017	if (nr_bits > BITS_PER_U128) {
2018		btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
2019				      BITS_PER_U128);
2020		return -EINVAL;
2021	}
2022
2023	if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
2024		btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
2025		return -EINVAL;
2026	}
2027
2028	/*
2029	 * Only one of the encoding bits is allowed and it
2030	 * should be sufficient for the pretty print purpose (i.e. decoding).
2031	 * Multiple bits can be allowed later if it is found
2032	 * to be insufficient.
2033	 */
2034	encoding = BTF_INT_ENCODING(int_data);
2035	if (encoding &&
2036	    encoding != BTF_INT_SIGNED &&
2037	    encoding != BTF_INT_CHAR &&
2038	    encoding != BTF_INT_BOOL) {
2039		btf_verifier_log_type(env, t, "Unsupported encoding");
2040		return -ENOTSUPP;
2041	}
2042
2043	btf_verifier_log_type(env, t, NULL);
2044
2045	return meta_needed;
2046}
2047
2048static void btf_int_log(struct btf_verifier_env *env,
2049			const struct btf_type *t)
2050{
2051	int int_data = btf_type_int(t);
2052
2053	btf_verifier_log(env,
2054			 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
2055			 t->size, BTF_INT_OFFSET(int_data),
2056			 BTF_INT_BITS(int_data),
2057			 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
2058}
2059
2060static void btf_int128_print(struct btf_show *show, void *data)
2061{
2062	/* data points to a __int128 number.
2063	 * Suppose
2064	 *     int128_num = *(__int128 *)data;
2065	 * The below formulas shows what upper_num and lower_num represents:
2066	 *     upper_num = int128_num >> 64;
2067	 *     lower_num = int128_num & 0xffffffffFFFFFFFFULL;
2068	 */
2069	u64 upper_num, lower_num;
2070
2071#ifdef __BIG_ENDIAN_BITFIELD
2072	upper_num = *(u64 *)data;
2073	lower_num = *(u64 *)(data + 8);
2074#else
2075	upper_num = *(u64 *)(data + 8);
2076	lower_num = *(u64 *)data;
2077#endif
2078	if (upper_num == 0)
2079		btf_show_type_value(show, "0x%llx", lower_num);
2080	else
2081		btf_show_type_values(show, "0x%llx%016llx", upper_num,
2082				     lower_num);
2083}
2084
2085static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,
2086			     u16 right_shift_bits)
2087{
2088	u64 upper_num, lower_num;
2089
2090#ifdef __BIG_ENDIAN_BITFIELD
2091	upper_num = print_num[0];
2092	lower_num = print_num[1];
2093#else
2094	upper_num = print_num[1];
2095	lower_num = print_num[0];
2096#endif
2097
2098	/* shake out un-needed bits by shift/or operations */
2099	if (left_shift_bits >= 64) {
2100		upper_num = lower_num << (left_shift_bits - 64);
2101		lower_num = 0;
2102	} else {
2103		upper_num = (upper_num << left_shift_bits) |
2104			    (lower_num >> (64 - left_shift_bits));
2105		lower_num = lower_num << left_shift_bits;
2106	}
2107
2108	if (right_shift_bits >= 64) {
2109		lower_num = upper_num >> (right_shift_bits - 64);
2110		upper_num = 0;
2111	} else {
2112		lower_num = (lower_num >> right_shift_bits) |
2113			    (upper_num << (64 - right_shift_bits));
2114		upper_num = upper_num >> right_shift_bits;
2115	}
2116
2117#ifdef __BIG_ENDIAN_BITFIELD
2118	print_num[0] = upper_num;
2119	print_num[1] = lower_num;
2120#else
2121	print_num[0] = lower_num;
2122	print_num[1] = upper_num;
2123#endif
2124}
2125
2126static void btf_bitfield_show(void *data, u8 bits_offset,
2127			      u8 nr_bits, struct btf_show *show)
2128{
2129	u16 left_shift_bits, right_shift_bits;
2130	u8 nr_copy_bytes;
2131	u8 nr_copy_bits;
2132	u64 print_num[2] = {};
2133
2134	nr_copy_bits = nr_bits + bits_offset;
2135	nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
2136
2137	memcpy(print_num, data, nr_copy_bytes);
2138
2139#ifdef __BIG_ENDIAN_BITFIELD
2140	left_shift_bits = bits_offset;
2141#else
2142	left_shift_bits = BITS_PER_U128 - nr_copy_bits;
2143#endif
2144	right_shift_bits = BITS_PER_U128 - nr_bits;
2145
2146	btf_int128_shift(print_num, left_shift_bits, right_shift_bits);
2147	btf_int128_print(show, print_num);
2148}
2149
2150
2151static void btf_int_bits_show(const struct btf *btf,
2152			      const struct btf_type *t,
2153			      void *data, u8 bits_offset,
2154			      struct btf_show *show)
2155{
2156	u32 int_data = btf_type_int(t);
2157	u8 nr_bits = BTF_INT_BITS(int_data);
2158	u8 total_bits_offset;
2159
2160	/*
2161	 * bits_offset is at most 7.
2162	 * BTF_INT_OFFSET() cannot exceed 128 bits.
2163	 */
2164	total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
2165	data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
2166	bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
2167	btf_bitfield_show(data, bits_offset, nr_bits, show);
2168}
2169
2170static void btf_int_show(const struct btf *btf, const struct btf_type *t,
2171			 u32 type_id, void *data, u8 bits_offset,
2172			 struct btf_show *show)
2173{
2174	u32 int_data = btf_type_int(t);
2175	u8 encoding = BTF_INT_ENCODING(int_data);
2176	bool sign = encoding & BTF_INT_SIGNED;
2177	u8 nr_bits = BTF_INT_BITS(int_data);
2178	void *safe_data;
2179
2180	safe_data = btf_show_start_type(show, t, type_id, data);
2181	if (!safe_data)
2182		return;
2183
2184	if (bits_offset || BTF_INT_OFFSET(int_data) ||
2185	    BITS_PER_BYTE_MASKED(nr_bits)) {
2186		btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2187		goto out;
2188	}
2189
2190	switch (nr_bits) {
2191	case 128:
2192		btf_int128_print(show, safe_data);
2193		break;
2194	case 64:
2195		if (sign)
2196			btf_show_type_value(show, "%lld", *(s64 *)safe_data);
2197		else
2198			btf_show_type_value(show, "%llu", *(u64 *)safe_data);
2199		break;
2200	case 32:
2201		if (sign)
2202			btf_show_type_value(show, "%d", *(s32 *)safe_data);
2203		else
2204			btf_show_type_value(show, "%u", *(u32 *)safe_data);
2205		break;
2206	case 16:
2207		if (sign)
2208			btf_show_type_value(show, "%d", *(s16 *)safe_data);
2209		else
2210			btf_show_type_value(show, "%u", *(u16 *)safe_data);
2211		break;
2212	case 8:
2213		if (show->state.array_encoding == BTF_INT_CHAR) {
2214			/* check for null terminator */
2215			if (show->state.array_terminated)
2216				break;
2217			if (*(char *)data == '\0') {
2218				show->state.array_terminated = 1;
2219				break;
2220			}
2221			if (isprint(*(char *)data)) {
2222				btf_show_type_value(show, "'%c'",
2223						    *(char *)safe_data);
2224				break;
2225			}
2226		}
2227		if (sign)
2228			btf_show_type_value(show, "%d", *(s8 *)safe_data);
2229		else
2230			btf_show_type_value(show, "%u", *(u8 *)safe_data);
2231		break;
2232	default:
2233		btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2234		break;
2235	}
2236out:
2237	btf_show_end_type(show);
2238}
2239
2240static const struct btf_kind_operations int_ops = {
2241	.check_meta = btf_int_check_meta,
2242	.resolve = btf_df_resolve,
2243	.check_member = btf_int_check_member,
2244	.check_kflag_member = btf_int_check_kflag_member,
2245	.log_details = btf_int_log,
2246	.show = btf_int_show,
2247};
2248
2249static int btf_modifier_check_member(struct btf_verifier_env *env,
2250				     const struct btf_type *struct_type,
2251				     const struct btf_member *member,
2252				     const struct btf_type *member_type)
2253{
2254	const struct btf_type *resolved_type;
2255	u32 resolved_type_id = member->type;
2256	struct btf_member resolved_member;
2257	struct btf *btf = env->btf;
2258
2259	resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2260	if (!resolved_type) {
2261		btf_verifier_log_member(env, struct_type, member,
2262					"Invalid member");
2263		return -EINVAL;
2264	}
2265
2266	resolved_member = *member;
2267	resolved_member.type = resolved_type_id;
2268
2269	return btf_type_ops(resolved_type)->check_member(env, struct_type,
2270							 &resolved_member,
2271							 resolved_type);
2272}
2273
2274static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,
2275					   const struct btf_type *struct_type,
2276					   const struct btf_member *member,
2277					   const struct btf_type *member_type)
2278{
2279	const struct btf_type *resolved_type;
2280	u32 resolved_type_id = member->type;
2281	struct btf_member resolved_member;
2282	struct btf *btf = env->btf;
2283
2284	resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2285	if (!resolved_type) {
2286		btf_verifier_log_member(env, struct_type, member,
2287					"Invalid member");
2288		return -EINVAL;
2289	}
2290
2291	resolved_member = *member;
2292	resolved_member.type = resolved_type_id;
2293
2294	return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,
2295							       &resolved_member,
2296							       resolved_type);
2297}
2298
2299static int btf_ptr_check_member(struct btf_verifier_env *env,
2300				const struct btf_type *struct_type,
2301				const struct btf_member *member,
2302				const struct btf_type *member_type)
2303{
2304	u32 struct_size, struct_bits_off, bytes_offset;
2305
2306	struct_size = struct_type->size;
2307	struct_bits_off = member->offset;
2308	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2309
2310	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2311		btf_verifier_log_member(env, struct_type, member,
2312					"Member is not byte aligned");
2313		return -EINVAL;
2314	}
2315
2316	if (struct_size - bytes_offset < sizeof(void *)) {
2317		btf_verifier_log_member(env, struct_type, member,
2318					"Member exceeds struct_size");
2319		return -EINVAL;
2320	}
2321
2322	return 0;
2323}
2324
2325static int btf_ref_type_check_meta(struct btf_verifier_env *env,
2326				   const struct btf_type *t,
2327				   u32 meta_left)
2328{
 
 
2329	if (btf_type_vlen(t)) {
2330		btf_verifier_log_type(env, t, "vlen != 0");
2331		return -EINVAL;
2332	}
2333
2334	if (btf_type_kflag(t)) {
2335		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2336		return -EINVAL;
2337	}
2338
2339	if (!BTF_TYPE_ID_VALID(t->type)) {
2340		btf_verifier_log_type(env, t, "Invalid type_id");
2341		return -EINVAL;
2342	}
2343
2344	/* typedef type must have a valid name, and other ref types,
2345	 * volatile, const, restrict, should have a null name.
2346	 */
2347	if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
2348		if (!t->name_off ||
2349		    !btf_name_valid_identifier(env->btf, t->name_off)) {
2350			btf_verifier_log_type(env, t, "Invalid name");
2351			return -EINVAL;
2352		}
 
 
 
 
 
 
2353	} else {
2354		if (t->name_off) {
2355			btf_verifier_log_type(env, t, "Invalid name");
2356			return -EINVAL;
2357		}
2358	}
2359
2360	btf_verifier_log_type(env, t, NULL);
2361
2362	return 0;
2363}
2364
2365static int btf_modifier_resolve(struct btf_verifier_env *env,
2366				const struct resolve_vertex *v)
2367{
2368	const struct btf_type *t = v->t;
2369	const struct btf_type *next_type;
2370	u32 next_type_id = t->type;
2371	struct btf *btf = env->btf;
2372
2373	next_type = btf_type_by_id(btf, next_type_id);
2374	if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2375		btf_verifier_log_type(env, v->t, "Invalid type_id");
2376		return -EINVAL;
2377	}
2378
2379	if (!env_type_is_resolve_sink(env, next_type) &&
2380	    !env_type_is_resolved(env, next_type_id))
2381		return env_stack_push(env, next_type, next_type_id);
2382
2383	/* Figure out the resolved next_type_id with size.
2384	 * They will be stored in the current modifier's
2385	 * resolved_ids and resolved_sizes such that it can
2386	 * save us a few type-following when we use it later (e.g. in
2387	 * pretty print).
2388	 */
2389	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2390		if (env_type_is_resolved(env, next_type_id))
2391			next_type = btf_type_id_resolve(btf, &next_type_id);
2392
2393		/* "typedef void new_void", "const void"...etc */
2394		if (!btf_type_is_void(next_type) &&
2395		    !btf_type_is_fwd(next_type) &&
2396		    !btf_type_is_func_proto(next_type)) {
2397			btf_verifier_log_type(env, v->t, "Invalid type_id");
2398			return -EINVAL;
2399		}
2400	}
2401
2402	env_stack_pop_resolved(env, next_type_id, 0);
2403
2404	return 0;
2405}
2406
2407static int btf_var_resolve(struct btf_verifier_env *env,
2408			   const struct resolve_vertex *v)
2409{
2410	const struct btf_type *next_type;
2411	const struct btf_type *t = v->t;
2412	u32 next_type_id = t->type;
2413	struct btf *btf = env->btf;
2414
2415	next_type = btf_type_by_id(btf, next_type_id);
2416	if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2417		btf_verifier_log_type(env, v->t, "Invalid type_id");
2418		return -EINVAL;
2419	}
2420
2421	if (!env_type_is_resolve_sink(env, next_type) &&
2422	    !env_type_is_resolved(env, next_type_id))
2423		return env_stack_push(env, next_type, next_type_id);
2424
2425	if (btf_type_is_modifier(next_type)) {
2426		const struct btf_type *resolved_type;
2427		u32 resolved_type_id;
2428
2429		resolved_type_id = next_type_id;
2430		resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2431
2432		if (btf_type_is_ptr(resolved_type) &&
2433		    !env_type_is_resolve_sink(env, resolved_type) &&
2434		    !env_type_is_resolved(env, resolved_type_id))
2435			return env_stack_push(env, resolved_type,
2436					      resolved_type_id);
2437	}
2438
2439	/* We must resolve to something concrete at this point, no
2440	 * forward types or similar that would resolve to size of
2441	 * zero is allowed.
2442	 */
2443	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2444		btf_verifier_log_type(env, v->t, "Invalid type_id");
2445		return -EINVAL;
2446	}
2447
2448	env_stack_pop_resolved(env, next_type_id, 0);
2449
2450	return 0;
2451}
2452
2453static int btf_ptr_resolve(struct btf_verifier_env *env,
2454			   const struct resolve_vertex *v)
2455{
2456	const struct btf_type *next_type;
2457	const struct btf_type *t = v->t;
2458	u32 next_type_id = t->type;
2459	struct btf *btf = env->btf;
2460
2461	next_type = btf_type_by_id(btf, next_type_id);
2462	if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2463		btf_verifier_log_type(env, v->t, "Invalid type_id");
2464		return -EINVAL;
2465	}
2466
2467	if (!env_type_is_resolve_sink(env, next_type) &&
2468	    !env_type_is_resolved(env, next_type_id))
2469		return env_stack_push(env, next_type, next_type_id);
2470
2471	/* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
2472	 * the modifier may have stopped resolving when it was resolved
2473	 * to a ptr (last-resolved-ptr).
2474	 *
2475	 * We now need to continue from the last-resolved-ptr to
2476	 * ensure the last-resolved-ptr will not referring back to
2477	 * the currenct ptr (t).
2478	 */
2479	if (btf_type_is_modifier(next_type)) {
2480		const struct btf_type *resolved_type;
2481		u32 resolved_type_id;
2482
2483		resolved_type_id = next_type_id;
2484		resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2485
2486		if (btf_type_is_ptr(resolved_type) &&
2487		    !env_type_is_resolve_sink(env, resolved_type) &&
2488		    !env_type_is_resolved(env, resolved_type_id))
2489			return env_stack_push(env, resolved_type,
2490					      resolved_type_id);
2491	}
2492
2493	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2494		if (env_type_is_resolved(env, next_type_id))
2495			next_type = btf_type_id_resolve(btf, &next_type_id);
2496
2497		if (!btf_type_is_void(next_type) &&
2498		    !btf_type_is_fwd(next_type) &&
2499		    !btf_type_is_func_proto(next_type)) {
2500			btf_verifier_log_type(env, v->t, "Invalid type_id");
2501			return -EINVAL;
2502		}
2503	}
2504
2505	env_stack_pop_resolved(env, next_type_id, 0);
2506
2507	return 0;
2508}
2509
2510static void btf_modifier_show(const struct btf *btf,
2511			      const struct btf_type *t,
2512			      u32 type_id, void *data,
2513			      u8 bits_offset, struct btf_show *show)
2514{
2515	if (btf->resolved_ids)
2516		t = btf_type_id_resolve(btf, &type_id);
2517	else
2518		t = btf_type_skip_modifiers(btf, type_id, NULL);
2519
2520	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2521}
2522
2523static void btf_var_show(const struct btf *btf, const struct btf_type *t,
2524			 u32 type_id, void *data, u8 bits_offset,
2525			 struct btf_show *show)
2526{
2527	t = btf_type_id_resolve(btf, &type_id);
2528
2529	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2530}
2531
2532static void btf_ptr_show(const struct btf *btf, const struct btf_type *t,
2533			 u32 type_id, void *data, u8 bits_offset,
2534			 struct btf_show *show)
2535{
2536	void *safe_data;
2537
2538	safe_data = btf_show_start_type(show, t, type_id, data);
2539	if (!safe_data)
2540		return;
2541
2542	/* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */
2543	if (show->flags & BTF_SHOW_PTR_RAW)
2544		btf_show_type_value(show, "0x%px", *(void **)safe_data);
2545	else
2546		btf_show_type_value(show, "0x%p", *(void **)safe_data);
2547	btf_show_end_type(show);
2548}
2549
2550static void btf_ref_type_log(struct btf_verifier_env *env,
2551			     const struct btf_type *t)
2552{
2553	btf_verifier_log(env, "type_id=%u", t->type);
2554}
2555
2556static struct btf_kind_operations modifier_ops = {
2557	.check_meta = btf_ref_type_check_meta,
2558	.resolve = btf_modifier_resolve,
2559	.check_member = btf_modifier_check_member,
2560	.check_kflag_member = btf_modifier_check_kflag_member,
2561	.log_details = btf_ref_type_log,
2562	.show = btf_modifier_show,
2563};
2564
2565static struct btf_kind_operations ptr_ops = {
2566	.check_meta = btf_ref_type_check_meta,
2567	.resolve = btf_ptr_resolve,
2568	.check_member = btf_ptr_check_member,
2569	.check_kflag_member = btf_generic_check_kflag_member,
2570	.log_details = btf_ref_type_log,
2571	.show = btf_ptr_show,
2572};
2573
2574static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
2575			      const struct btf_type *t,
2576			      u32 meta_left)
2577{
2578	if (btf_type_vlen(t)) {
2579		btf_verifier_log_type(env, t, "vlen != 0");
2580		return -EINVAL;
2581	}
2582
2583	if (t->type) {
2584		btf_verifier_log_type(env, t, "type != 0");
2585		return -EINVAL;
2586	}
2587
2588	/* fwd type must have a valid name */
2589	if (!t->name_off ||
2590	    !btf_name_valid_identifier(env->btf, t->name_off)) {
2591		btf_verifier_log_type(env, t, "Invalid name");
2592		return -EINVAL;
2593	}
2594
2595	btf_verifier_log_type(env, t, NULL);
2596
2597	return 0;
2598}
2599
2600static void btf_fwd_type_log(struct btf_verifier_env *env,
2601			     const struct btf_type *t)
2602{
2603	btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
2604}
2605
2606static struct btf_kind_operations fwd_ops = {
2607	.check_meta = btf_fwd_check_meta,
2608	.resolve = btf_df_resolve,
2609	.check_member = btf_df_check_member,
2610	.check_kflag_member = btf_df_check_kflag_member,
2611	.log_details = btf_fwd_type_log,
2612	.show = btf_df_show,
2613};
2614
2615static int btf_array_check_member(struct btf_verifier_env *env,
2616				  const struct btf_type *struct_type,
2617				  const struct btf_member *member,
2618				  const struct btf_type *member_type)
2619{
2620	u32 struct_bits_off = member->offset;
2621	u32 struct_size, bytes_offset;
2622	u32 array_type_id, array_size;
2623	struct btf *btf = env->btf;
2624
2625	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2626		btf_verifier_log_member(env, struct_type, member,
2627					"Member is not byte aligned");
2628		return -EINVAL;
2629	}
2630
2631	array_type_id = member->type;
2632	btf_type_id_size(btf, &array_type_id, &array_size);
2633	struct_size = struct_type->size;
2634	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2635	if (struct_size - bytes_offset < array_size) {
2636		btf_verifier_log_member(env, struct_type, member,
2637					"Member exceeds struct_size");
2638		return -EINVAL;
2639	}
2640
2641	return 0;
2642}
2643
2644static s32 btf_array_check_meta(struct btf_verifier_env *env,
2645				const struct btf_type *t,
2646				u32 meta_left)
2647{
2648	const struct btf_array *array = btf_type_array(t);
2649	u32 meta_needed = sizeof(*array);
2650
2651	if (meta_left < meta_needed) {
2652		btf_verifier_log_basic(env, t,
2653				       "meta_left:%u meta_needed:%u",
2654				       meta_left, meta_needed);
2655		return -EINVAL;
2656	}
2657
2658	/* array type should not have a name */
2659	if (t->name_off) {
2660		btf_verifier_log_type(env, t, "Invalid name");
2661		return -EINVAL;
2662	}
2663
2664	if (btf_type_vlen(t)) {
2665		btf_verifier_log_type(env, t, "vlen != 0");
2666		return -EINVAL;
2667	}
2668
2669	if (btf_type_kflag(t)) {
2670		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2671		return -EINVAL;
2672	}
2673
2674	if (t->size) {
2675		btf_verifier_log_type(env, t, "size != 0");
2676		return -EINVAL;
2677	}
2678
2679	/* Array elem type and index type cannot be in type void,
2680	 * so !array->type and !array->index_type are not allowed.
2681	 */
2682	if (!array->type || !BTF_TYPE_ID_VALID(array->type)) {
2683		btf_verifier_log_type(env, t, "Invalid elem");
2684		return -EINVAL;
2685	}
2686
2687	if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) {
2688		btf_verifier_log_type(env, t, "Invalid index");
2689		return -EINVAL;
2690	}
2691
2692	btf_verifier_log_type(env, t, NULL);
2693
2694	return meta_needed;
2695}
2696
2697static int btf_array_resolve(struct btf_verifier_env *env,
2698			     const struct resolve_vertex *v)
2699{
2700	const struct btf_array *array = btf_type_array(v->t);
2701	const struct btf_type *elem_type, *index_type;
2702	u32 elem_type_id, index_type_id;
2703	struct btf *btf = env->btf;
2704	u32 elem_size;
2705
2706	/* Check array->index_type */
2707	index_type_id = array->index_type;
2708	index_type = btf_type_by_id(btf, index_type_id);
2709	if (btf_type_nosize_or_null(index_type) ||
2710	    btf_type_is_resolve_source_only(index_type)) {
2711		btf_verifier_log_type(env, v->t, "Invalid index");
2712		return -EINVAL;
2713	}
2714
2715	if (!env_type_is_resolve_sink(env, index_type) &&
2716	    !env_type_is_resolved(env, index_type_id))
2717		return env_stack_push(env, index_type, index_type_id);
2718
2719	index_type = btf_type_id_size(btf, &index_type_id, NULL);
2720	if (!index_type || !btf_type_is_int(index_type) ||
2721	    !btf_type_int_is_regular(index_type)) {
2722		btf_verifier_log_type(env, v->t, "Invalid index");
2723		return -EINVAL;
2724	}
2725
2726	/* Check array->type */
2727	elem_type_id = array->type;
2728	elem_type = btf_type_by_id(btf, elem_type_id);
2729	if (btf_type_nosize_or_null(elem_type) ||
2730	    btf_type_is_resolve_source_only(elem_type)) {
2731		btf_verifier_log_type(env, v->t,
2732				      "Invalid elem");
2733		return -EINVAL;
2734	}
2735
2736	if (!env_type_is_resolve_sink(env, elem_type) &&
2737	    !env_type_is_resolved(env, elem_type_id))
2738		return env_stack_push(env, elem_type, elem_type_id);
2739
2740	elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
2741	if (!elem_type) {
2742		btf_verifier_log_type(env, v->t, "Invalid elem");
2743		return -EINVAL;
2744	}
2745
2746	if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
2747		btf_verifier_log_type(env, v->t, "Invalid array of int");
2748		return -EINVAL;
2749	}
2750
2751	if (array->nelems && elem_size > U32_MAX / array->nelems) {
2752		btf_verifier_log_type(env, v->t,
2753				      "Array size overflows U32_MAX");
2754		return -EINVAL;
2755	}
2756
2757	env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
2758
2759	return 0;
2760}
2761
2762static void btf_array_log(struct btf_verifier_env *env,
2763			  const struct btf_type *t)
2764{
2765	const struct btf_array *array = btf_type_array(t);
2766
2767	btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
2768			 array->type, array->index_type, array->nelems);
2769}
2770
2771static void __btf_array_show(const struct btf *btf, const struct btf_type *t,
2772			     u32 type_id, void *data, u8 bits_offset,
2773			     struct btf_show *show)
2774{
2775	const struct btf_array *array = btf_type_array(t);
2776	const struct btf_kind_operations *elem_ops;
2777	const struct btf_type *elem_type;
2778	u32 i, elem_size = 0, elem_type_id;
2779	u16 encoding = 0;
2780
2781	elem_type_id = array->type;
2782	elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL);
2783	if (elem_type && btf_type_has_size(elem_type))
2784		elem_size = elem_type->size;
2785
2786	if (elem_type && btf_type_is_int(elem_type)) {
2787		u32 int_type = btf_type_int(elem_type);
2788
2789		encoding = BTF_INT_ENCODING(int_type);
2790
2791		/*
2792		 * BTF_INT_CHAR encoding never seems to be set for
2793		 * char arrays, so if size is 1 and element is
2794		 * printable as a char, we'll do that.
2795		 */
2796		if (elem_size == 1)
2797			encoding = BTF_INT_CHAR;
2798	}
2799
2800	if (!btf_show_start_array_type(show, t, type_id, encoding, data))
2801		return;
2802
2803	if (!elem_type)
2804		goto out;
2805	elem_ops = btf_type_ops(elem_type);
2806
2807	for (i = 0; i < array->nelems; i++) {
2808
2809		btf_show_start_array_member(show);
2810
2811		elem_ops->show(btf, elem_type, elem_type_id, data,
2812			       bits_offset, show);
2813		data += elem_size;
2814
2815		btf_show_end_array_member(show);
2816
2817		if (show->state.array_terminated)
2818			break;
2819	}
2820out:
2821	btf_show_end_array_type(show);
2822}
2823
2824static void btf_array_show(const struct btf *btf, const struct btf_type *t,
2825			   u32 type_id, void *data, u8 bits_offset,
2826			   struct btf_show *show)
2827{
2828	const struct btf_member *m = show->state.member;
2829
2830	/*
2831	 * First check if any members would be shown (are non-zero).
2832	 * See comments above "struct btf_show" definition for more
2833	 * details on how this works at a high-level.
2834	 */
2835	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
2836		if (!show->state.depth_check) {
2837			show->state.depth_check = show->state.depth + 1;
2838			show->state.depth_to_show = 0;
2839		}
2840		__btf_array_show(btf, t, type_id, data, bits_offset, show);
2841		show->state.member = m;
2842
2843		if (show->state.depth_check != show->state.depth + 1)
2844			return;
2845		show->state.depth_check = 0;
2846
2847		if (show->state.depth_to_show <= show->state.depth)
2848			return;
2849		/*
2850		 * Reaching here indicates we have recursed and found
2851		 * non-zero array member(s).
2852		 */
2853	}
2854	__btf_array_show(btf, t, type_id, data, bits_offset, show);
2855}
2856
2857static struct btf_kind_operations array_ops = {
2858	.check_meta = btf_array_check_meta,
2859	.resolve = btf_array_resolve,
2860	.check_member = btf_array_check_member,
2861	.check_kflag_member = btf_generic_check_kflag_member,
2862	.log_details = btf_array_log,
2863	.show = btf_array_show,
2864};
2865
2866static int btf_struct_check_member(struct btf_verifier_env *env,
2867				   const struct btf_type *struct_type,
2868				   const struct btf_member *member,
2869				   const struct btf_type *member_type)
2870{
2871	u32 struct_bits_off = member->offset;
2872	u32 struct_size, bytes_offset;
2873
2874	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2875		btf_verifier_log_member(env, struct_type, member,
2876					"Member is not byte aligned");
2877		return -EINVAL;
2878	}
2879
2880	struct_size = struct_type->size;
2881	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2882	if (struct_size - bytes_offset < member_type->size) {
2883		btf_verifier_log_member(env, struct_type, member,
2884					"Member exceeds struct_size");
2885		return -EINVAL;
2886	}
2887
2888	return 0;
2889}
2890
2891static s32 btf_struct_check_meta(struct btf_verifier_env *env,
2892				 const struct btf_type *t,
2893				 u32 meta_left)
2894{
2895	bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
2896	const struct btf_member *member;
2897	u32 meta_needed, last_offset;
2898	struct btf *btf = env->btf;
2899	u32 struct_size = t->size;
2900	u32 offset;
2901	u16 i;
2902
2903	meta_needed = btf_type_vlen(t) * sizeof(*member);
2904	if (meta_left < meta_needed) {
2905		btf_verifier_log_basic(env, t,
2906				       "meta_left:%u meta_needed:%u",
2907				       meta_left, meta_needed);
2908		return -EINVAL;
2909	}
2910
2911	/* struct type either no name or a valid one */
2912	if (t->name_off &&
2913	    !btf_name_valid_identifier(env->btf, t->name_off)) {
2914		btf_verifier_log_type(env, t, "Invalid name");
2915		return -EINVAL;
2916	}
2917
2918	btf_verifier_log_type(env, t, NULL);
2919
2920	last_offset = 0;
2921	for_each_member(i, t, member) {
2922		if (!btf_name_offset_valid(btf, member->name_off)) {
2923			btf_verifier_log_member(env, t, member,
2924						"Invalid member name_offset:%u",
2925						member->name_off);
2926			return -EINVAL;
2927		}
2928
2929		/* struct member either no name or a valid one */
2930		if (member->name_off &&
2931		    !btf_name_valid_identifier(btf, member->name_off)) {
2932			btf_verifier_log_member(env, t, member, "Invalid name");
2933			return -EINVAL;
2934		}
2935		/* A member cannot be in type void */
2936		if (!member->type || !BTF_TYPE_ID_VALID(member->type)) {
2937			btf_verifier_log_member(env, t, member,
2938						"Invalid type_id");
2939			return -EINVAL;
2940		}
2941
2942		offset = btf_member_bit_offset(t, member);
2943		if (is_union && offset) {
2944			btf_verifier_log_member(env, t, member,
2945						"Invalid member bits_offset");
2946			return -EINVAL;
2947		}
2948
2949		/*
2950		 * ">" instead of ">=" because the last member could be
2951		 * "char a[0];"
2952		 */
2953		if (last_offset > offset) {
2954			btf_verifier_log_member(env, t, member,
2955						"Invalid member bits_offset");
2956			return -EINVAL;
2957		}
2958
2959		if (BITS_ROUNDUP_BYTES(offset) > struct_size) {
2960			btf_verifier_log_member(env, t, member,
2961						"Member bits_offset exceeds its struct size");
2962			return -EINVAL;
2963		}
2964
2965		btf_verifier_log_member(env, t, member, NULL);
2966		last_offset = offset;
2967	}
2968
2969	return meta_needed;
2970}
2971
2972static int btf_struct_resolve(struct btf_verifier_env *env,
2973			      const struct resolve_vertex *v)
2974{
2975	const struct btf_member *member;
2976	int err;
2977	u16 i;
2978
2979	/* Before continue resolving the next_member,
2980	 * ensure the last member is indeed resolved to a
2981	 * type with size info.
2982	 */
2983	if (v->next_member) {
2984		const struct btf_type *last_member_type;
2985		const struct btf_member *last_member;
2986		u16 last_member_type_id;
2987
2988		last_member = btf_type_member(v->t) + v->next_member - 1;
2989		last_member_type_id = last_member->type;
2990		if (WARN_ON_ONCE(!env_type_is_resolved(env,
2991						       last_member_type_id)))
2992			return -EINVAL;
2993
2994		last_member_type = btf_type_by_id(env->btf,
2995						  last_member_type_id);
2996		if (btf_type_kflag(v->t))
2997			err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,
2998								last_member,
2999								last_member_type);
3000		else
3001			err = btf_type_ops(last_member_type)->check_member(env, v->t,
3002								last_member,
3003								last_member_type);
3004		if (err)
3005			return err;
3006	}
3007
3008	for_each_member_from(i, v->next_member, v->t, member) {
3009		u32 member_type_id = member->type;
3010		const struct btf_type *member_type = btf_type_by_id(env->btf,
3011								member_type_id);
3012
3013		if (btf_type_nosize_or_null(member_type) ||
3014		    btf_type_is_resolve_source_only(member_type)) {
3015			btf_verifier_log_member(env, v->t, member,
3016						"Invalid member");
3017			return -EINVAL;
3018		}
3019
3020		if (!env_type_is_resolve_sink(env, member_type) &&
3021		    !env_type_is_resolved(env, member_type_id)) {
3022			env_stack_set_next_member(env, i + 1);
3023			return env_stack_push(env, member_type, member_type_id);
3024		}
3025
3026		if (btf_type_kflag(v->t))
3027			err = btf_type_ops(member_type)->check_kflag_member(env, v->t,
3028									    member,
3029									    member_type);
3030		else
3031			err = btf_type_ops(member_type)->check_member(env, v->t,
3032								      member,
3033								      member_type);
3034		if (err)
3035			return err;
3036	}
3037
3038	env_stack_pop_resolved(env, 0, 0);
3039
3040	return 0;
3041}
3042
3043static void btf_struct_log(struct btf_verifier_env *env,
3044			   const struct btf_type *t)
3045{
3046	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3047}
3048
3049/* find 'struct bpf_spin_lock' in map value.
3050 * return >= 0 offset if found
3051 * and < 0 in case of error
3052 */
3053int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3054{
3055	const struct btf_member *member;
3056	u32 i, off = -ENOENT;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3057
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3058	if (!__btf_type_is_struct(t))
3059		return -EINVAL;
3060
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3061	for_each_member(i, t, member) {
3062		const struct btf_type *member_type = btf_type_by_id(btf,
3063								    member->type);
3064		if (!__btf_type_is_struct(member_type))
3065			continue;
3066		if (member_type->size != sizeof(struct bpf_spin_lock))
3067			continue;
3068		if (strcmp(__btf_name_by_offset(btf, member_type->name_off),
3069			   "bpf_spin_lock"))
3070			continue;
3071		if (off != -ENOENT)
3072			/* only one 'struct bpf_spin_lock' is allowed */
3073			return -E2BIG;
3074		off = btf_member_bit_offset(t, member);
3075		if (off % 8)
3076			/* valid C code cannot generate such BTF */
3077			return -EINVAL;
3078		off /= 8;
3079		if (off % __alignof__(struct bpf_spin_lock))
3080			/* valid struct bpf_spin_lock will be 4 byte aligned */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3081			return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
3082	}
3083	return off;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3084}
3085
3086static void __btf_struct_show(const struct btf *btf, const struct btf_type *t,
3087			      u32 type_id, void *data, u8 bits_offset,
3088			      struct btf_show *show)
3089{
3090	const struct btf_member *member;
3091	void *safe_data;
3092	u32 i;
3093
3094	safe_data = btf_show_start_struct_type(show, t, type_id, data);
3095	if (!safe_data)
3096		return;
3097
3098	for_each_member(i, t, member) {
3099		const struct btf_type *member_type = btf_type_by_id(btf,
3100								member->type);
3101		const struct btf_kind_operations *ops;
3102		u32 member_offset, bitfield_size;
3103		u32 bytes_offset;
3104		u8 bits8_offset;
3105
3106		btf_show_start_member(show, member);
3107
3108		member_offset = btf_member_bit_offset(t, member);
3109		bitfield_size = btf_member_bitfield_size(t, member);
3110		bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
3111		bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
3112		if (bitfield_size) {
3113			safe_data = btf_show_start_type(show, member_type,
3114							member->type,
3115							data + bytes_offset);
3116			if (safe_data)
3117				btf_bitfield_show(safe_data,
3118						  bits8_offset,
3119						  bitfield_size, show);
3120			btf_show_end_type(show);
3121		} else {
3122			ops = btf_type_ops(member_type);
3123			ops->show(btf, member_type, member->type,
3124				  data + bytes_offset, bits8_offset, show);
3125		}
3126
3127		btf_show_end_member(show);
3128	}
3129
3130	btf_show_end_struct_type(show);
3131}
3132
3133static void btf_struct_show(const struct btf *btf, const struct btf_type *t,
3134			    u32 type_id, void *data, u8 bits_offset,
3135			    struct btf_show *show)
3136{
3137	const struct btf_member *m = show->state.member;
3138
3139	/*
3140	 * First check if any members would be shown (are non-zero).
3141	 * See comments above "struct btf_show" definition for more
3142	 * details on how this works at a high-level.
3143	 */
3144	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
3145		if (!show->state.depth_check) {
3146			show->state.depth_check = show->state.depth + 1;
3147			show->state.depth_to_show = 0;
3148		}
3149		__btf_struct_show(btf, t, type_id, data, bits_offset, show);
3150		/* Restore saved member data here */
3151		show->state.member = m;
3152		if (show->state.depth_check != show->state.depth + 1)
3153			return;
3154		show->state.depth_check = 0;
3155
3156		if (show->state.depth_to_show <= show->state.depth)
3157			return;
3158		/*
3159		 * Reaching here indicates we have recursed and found
3160		 * non-zero child values.
3161		 */
3162	}
3163
3164	__btf_struct_show(btf, t, type_id, data, bits_offset, show);
3165}
3166
3167static struct btf_kind_operations struct_ops = {
3168	.check_meta = btf_struct_check_meta,
3169	.resolve = btf_struct_resolve,
3170	.check_member = btf_struct_check_member,
3171	.check_kflag_member = btf_generic_check_kflag_member,
3172	.log_details = btf_struct_log,
3173	.show = btf_struct_show,
3174};
3175
3176static int btf_enum_check_member(struct btf_verifier_env *env,
3177				 const struct btf_type *struct_type,
3178				 const struct btf_member *member,
3179				 const struct btf_type *member_type)
3180{
3181	u32 struct_bits_off = member->offset;
3182	u32 struct_size, bytes_offset;
3183
3184	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3185		btf_verifier_log_member(env, struct_type, member,
3186					"Member is not byte aligned");
3187		return -EINVAL;
3188	}
3189
3190	struct_size = struct_type->size;
3191	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
3192	if (struct_size - bytes_offset < member_type->size) {
3193		btf_verifier_log_member(env, struct_type, member,
3194					"Member exceeds struct_size");
3195		return -EINVAL;
3196	}
3197
3198	return 0;
3199}
3200
3201static int btf_enum_check_kflag_member(struct btf_verifier_env *env,
3202				       const struct btf_type *struct_type,
3203				       const struct btf_member *member,
3204				       const struct btf_type *member_type)
3205{
3206	u32 struct_bits_off, nr_bits, bytes_end, struct_size;
3207	u32 int_bitsize = sizeof(int) * BITS_PER_BYTE;
3208
3209	struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
3210	nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
3211	if (!nr_bits) {
3212		if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3213			btf_verifier_log_member(env, struct_type, member,
3214						"Member is not byte aligned");
3215			return -EINVAL;
3216		}
3217
3218		nr_bits = int_bitsize;
3219	} else if (nr_bits > int_bitsize) {
3220		btf_verifier_log_member(env, struct_type, member,
3221					"Invalid member bitfield_size");
3222		return -EINVAL;
3223	}
3224
3225	struct_size = struct_type->size;
3226	bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits);
3227	if (struct_size < bytes_end) {
3228		btf_verifier_log_member(env, struct_type, member,
3229					"Member exceeds struct_size");
3230		return -EINVAL;
3231	}
3232
3233	return 0;
3234}
3235
3236static s32 btf_enum_check_meta(struct btf_verifier_env *env,
3237			       const struct btf_type *t,
3238			       u32 meta_left)
3239{
3240	const struct btf_enum *enums = btf_type_enum(t);
3241	struct btf *btf = env->btf;
 
3242	u16 i, nr_enums;
3243	u32 meta_needed;
3244
3245	nr_enums = btf_type_vlen(t);
3246	meta_needed = nr_enums * sizeof(*enums);
3247
3248	if (meta_left < meta_needed) {
3249		btf_verifier_log_basic(env, t,
3250				       "meta_left:%u meta_needed:%u",
3251				       meta_left, meta_needed);
3252		return -EINVAL;
3253	}
3254
3255	if (btf_type_kflag(t)) {
3256		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3257		return -EINVAL;
3258	}
3259
3260	if (t->size > 8 || !is_power_of_2(t->size)) {
3261		btf_verifier_log_type(env, t, "Unexpected size");
3262		return -EINVAL;
3263	}
3264
3265	/* enum type either no name or a valid one */
3266	if (t->name_off &&
3267	    !btf_name_valid_identifier(env->btf, t->name_off)) {
3268		btf_verifier_log_type(env, t, "Invalid name");
3269		return -EINVAL;
3270	}
3271
3272	btf_verifier_log_type(env, t, NULL);
3273
3274	for (i = 0; i < nr_enums; i++) {
3275		if (!btf_name_offset_valid(btf, enums[i].name_off)) {
3276			btf_verifier_log(env, "\tInvalid name_offset:%u",
3277					 enums[i].name_off);
3278			return -EINVAL;
3279		}
3280
3281		/* enum member must have a valid name */
3282		if (!enums[i].name_off ||
3283		    !btf_name_valid_identifier(btf, enums[i].name_off)) {
3284			btf_verifier_log_type(env, t, "Invalid name");
3285			return -EINVAL;
3286		}
3287
3288		if (env->log.level == BPF_LOG_KERNEL)
3289			continue;
3290		btf_verifier_log(env, "\t%s val=%d\n",
 
3291				 __btf_name_by_offset(btf, enums[i].name_off),
3292				 enums[i].val);
3293	}
3294
3295	return meta_needed;
3296}
3297
3298static void btf_enum_log(struct btf_verifier_env *env,
3299			 const struct btf_type *t)
3300{
3301	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3302}
3303
3304static void btf_enum_show(const struct btf *btf, const struct btf_type *t,
3305			  u32 type_id, void *data, u8 bits_offset,
3306			  struct btf_show *show)
3307{
3308	const struct btf_enum *enums = btf_type_enum(t);
3309	u32 i, nr_enums = btf_type_vlen(t);
3310	void *safe_data;
3311	int v;
3312
3313	safe_data = btf_show_start_type(show, t, type_id, data);
3314	if (!safe_data)
3315		return;
3316
3317	v = *(int *)safe_data;
3318
3319	for (i = 0; i < nr_enums; i++) {
3320		if (v != enums[i].val)
3321			continue;
3322
3323		btf_show_type_value(show, "%s",
3324				    __btf_name_by_offset(btf,
3325							 enums[i].name_off));
3326
3327		btf_show_end_type(show);
3328		return;
3329	}
3330
3331	btf_show_type_value(show, "%d", v);
 
 
 
3332	btf_show_end_type(show);
3333}
3334
3335static struct btf_kind_operations enum_ops = {
3336	.check_meta = btf_enum_check_meta,
3337	.resolve = btf_df_resolve,
3338	.check_member = btf_enum_check_member,
3339	.check_kflag_member = btf_enum_check_kflag_member,
3340	.log_details = btf_enum_log,
3341	.show = btf_enum_show,
3342};
3343
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3344static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
3345				     const struct btf_type *t,
3346				     u32 meta_left)
3347{
3348	u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param);
3349
3350	if (meta_left < meta_needed) {
3351		btf_verifier_log_basic(env, t,
3352				       "meta_left:%u meta_needed:%u",
3353				       meta_left, meta_needed);
3354		return -EINVAL;
3355	}
3356
3357	if (t->name_off) {
3358		btf_verifier_log_type(env, t, "Invalid name");
3359		return -EINVAL;
3360	}
3361
3362	if (btf_type_kflag(t)) {
3363		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3364		return -EINVAL;
3365	}
3366
3367	btf_verifier_log_type(env, t, NULL);
3368
3369	return meta_needed;
3370}
3371
3372static void btf_func_proto_log(struct btf_verifier_env *env,
3373			       const struct btf_type *t)
3374{
3375	const struct btf_param *args = (const struct btf_param *)(t + 1);
3376	u16 nr_args = btf_type_vlen(t), i;
3377
3378	btf_verifier_log(env, "return=%u args=(", t->type);
3379	if (!nr_args) {
3380		btf_verifier_log(env, "void");
3381		goto done;
3382	}
3383
3384	if (nr_args == 1 && !args[0].type) {
3385		/* Only one vararg */
3386		btf_verifier_log(env, "vararg");
3387		goto done;
3388	}
3389
3390	btf_verifier_log(env, "%u %s", args[0].type,
3391			 __btf_name_by_offset(env->btf,
3392					      args[0].name_off));
3393	for (i = 1; i < nr_args - 1; i++)
3394		btf_verifier_log(env, ", %u %s", args[i].type,
3395				 __btf_name_by_offset(env->btf,
3396						      args[i].name_off));
3397
3398	if (nr_args > 1) {
3399		const struct btf_param *last_arg = &args[nr_args - 1];
3400
3401		if (last_arg->type)
3402			btf_verifier_log(env, ", %u %s", last_arg->type,
3403					 __btf_name_by_offset(env->btf,
3404							      last_arg->name_off));
3405		else
3406			btf_verifier_log(env, ", vararg");
3407	}
3408
3409done:
3410	btf_verifier_log(env, ")");
3411}
3412
3413static struct btf_kind_operations func_proto_ops = {
3414	.check_meta = btf_func_proto_check_meta,
3415	.resolve = btf_df_resolve,
3416	/*
3417	 * BTF_KIND_FUNC_PROTO cannot be directly referred by
3418	 * a struct's member.
3419	 *
3420	 * It should be a function pointer instead.
3421	 * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO)
3422	 *
3423	 * Hence, there is no btf_func_check_member().
3424	 */
3425	.check_member = btf_df_check_member,
3426	.check_kflag_member = btf_df_check_kflag_member,
3427	.log_details = btf_func_proto_log,
3428	.show = btf_df_show,
3429};
3430
3431static s32 btf_func_check_meta(struct btf_verifier_env *env,
3432			       const struct btf_type *t,
3433			       u32 meta_left)
3434{
3435	if (!t->name_off ||
3436	    !btf_name_valid_identifier(env->btf, t->name_off)) {
3437		btf_verifier_log_type(env, t, "Invalid name");
3438		return -EINVAL;
3439	}
3440
3441	if (btf_type_vlen(t) > BTF_FUNC_GLOBAL) {
3442		btf_verifier_log_type(env, t, "Invalid func linkage");
3443		return -EINVAL;
3444	}
3445
3446	if (btf_type_kflag(t)) {
3447		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3448		return -EINVAL;
3449	}
3450
3451	btf_verifier_log_type(env, t, NULL);
3452
3453	return 0;
3454}
3455
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3456static struct btf_kind_operations func_ops = {
3457	.check_meta = btf_func_check_meta,
3458	.resolve = btf_df_resolve,
3459	.check_member = btf_df_check_member,
3460	.check_kflag_member = btf_df_check_kflag_member,
3461	.log_details = btf_ref_type_log,
3462	.show = btf_df_show,
3463};
3464
3465static s32 btf_var_check_meta(struct btf_verifier_env *env,
3466			      const struct btf_type *t,
3467			      u32 meta_left)
3468{
3469	const struct btf_var *var;
3470	u32 meta_needed = sizeof(*var);
3471
3472	if (meta_left < meta_needed) {
3473		btf_verifier_log_basic(env, t,
3474				       "meta_left:%u meta_needed:%u",
3475				       meta_left, meta_needed);
3476		return -EINVAL;
3477	}
3478
3479	if (btf_type_vlen(t)) {
3480		btf_verifier_log_type(env, t, "vlen != 0");
3481		return -EINVAL;
3482	}
3483
3484	if (btf_type_kflag(t)) {
3485		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3486		return -EINVAL;
3487	}
3488
3489	if (!t->name_off ||
3490	    !__btf_name_valid(env->btf, t->name_off, true)) {
3491		btf_verifier_log_type(env, t, "Invalid name");
3492		return -EINVAL;
3493	}
3494
3495	/* A var cannot be in type void */
3496	if (!t->type || !BTF_TYPE_ID_VALID(t->type)) {
3497		btf_verifier_log_type(env, t, "Invalid type_id");
3498		return -EINVAL;
3499	}
3500
3501	var = btf_type_var(t);
3502	if (var->linkage != BTF_VAR_STATIC &&
3503	    var->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
3504		btf_verifier_log_type(env, t, "Linkage not supported");
3505		return -EINVAL;
3506	}
3507
3508	btf_verifier_log_type(env, t, NULL);
3509
3510	return meta_needed;
3511}
3512
3513static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t)
3514{
3515	const struct btf_var *var = btf_type_var(t);
3516
3517	btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);
3518}
3519
3520static const struct btf_kind_operations var_ops = {
3521	.check_meta		= btf_var_check_meta,
3522	.resolve		= btf_var_resolve,
3523	.check_member		= btf_df_check_member,
3524	.check_kflag_member	= btf_df_check_kflag_member,
3525	.log_details		= btf_var_log,
3526	.show			= btf_var_show,
3527};
3528
3529static s32 btf_datasec_check_meta(struct btf_verifier_env *env,
3530				  const struct btf_type *t,
3531				  u32 meta_left)
3532{
3533	const struct btf_var_secinfo *vsi;
3534	u64 last_vsi_end_off = 0, sum = 0;
3535	u32 i, meta_needed;
3536
3537	meta_needed = btf_type_vlen(t) * sizeof(*vsi);
3538	if (meta_left < meta_needed) {
3539		btf_verifier_log_basic(env, t,
3540				       "meta_left:%u meta_needed:%u",
3541				       meta_left, meta_needed);
3542		return -EINVAL;
3543	}
3544
3545	if (!t->size) {
3546		btf_verifier_log_type(env, t, "size == 0");
3547		return -EINVAL;
3548	}
3549
3550	if (btf_type_kflag(t)) {
3551		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3552		return -EINVAL;
3553	}
3554
3555	if (!t->name_off ||
3556	    !btf_name_valid_section(env->btf, t->name_off)) {
3557		btf_verifier_log_type(env, t, "Invalid name");
3558		return -EINVAL;
3559	}
3560
3561	btf_verifier_log_type(env, t, NULL);
3562
3563	for_each_vsi(i, t, vsi) {
3564		/* A var cannot be in type void */
3565		if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) {
3566			btf_verifier_log_vsi(env, t, vsi,
3567					     "Invalid type_id");
3568			return -EINVAL;
3569		}
3570
3571		if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) {
3572			btf_verifier_log_vsi(env, t, vsi,
3573					     "Invalid offset");
3574			return -EINVAL;
3575		}
3576
3577		if (!vsi->size || vsi->size > t->size) {
3578			btf_verifier_log_vsi(env, t, vsi,
3579					     "Invalid size");
3580			return -EINVAL;
3581		}
3582
3583		last_vsi_end_off = vsi->offset + vsi->size;
3584		if (last_vsi_end_off > t->size) {
3585			btf_verifier_log_vsi(env, t, vsi,
3586					     "Invalid offset+size");
3587			return -EINVAL;
3588		}
3589
3590		btf_verifier_log_vsi(env, t, vsi, NULL);
3591		sum += vsi->size;
3592	}
3593
3594	if (t->size < sum) {
3595		btf_verifier_log_type(env, t, "Invalid btf_info size");
3596		return -EINVAL;
3597	}
3598
3599	return meta_needed;
3600}
3601
3602static int btf_datasec_resolve(struct btf_verifier_env *env,
3603			       const struct resolve_vertex *v)
3604{
3605	const struct btf_var_secinfo *vsi;
3606	struct btf *btf = env->btf;
3607	u16 i;
3608
 
3609	for_each_vsi_from(i, v->next_member, v->t, vsi) {
3610		u32 var_type_id = vsi->type, type_id, type_size = 0;
3611		const struct btf_type *var_type = btf_type_by_id(env->btf,
3612								 var_type_id);
3613		if (!var_type || !btf_type_is_var(var_type)) {
3614			btf_verifier_log_vsi(env, v->t, vsi,
3615					     "Not a VAR kind member");
3616			return -EINVAL;
3617		}
3618
3619		if (!env_type_is_resolve_sink(env, var_type) &&
3620		    !env_type_is_resolved(env, var_type_id)) {
3621			env_stack_set_next_member(env, i + 1);
3622			return env_stack_push(env, var_type, var_type_id);
3623		}
3624
3625		type_id = var_type->type;
3626		if (!btf_type_id_size(btf, &type_id, &type_size)) {
3627			btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");
3628			return -EINVAL;
3629		}
3630
3631		if (vsi->size < type_size) {
3632			btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");
3633			return -EINVAL;
3634		}
3635	}
3636
3637	env_stack_pop_resolved(env, 0, 0);
3638	return 0;
3639}
3640
3641static void btf_datasec_log(struct btf_verifier_env *env,
3642			    const struct btf_type *t)
3643{
3644	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3645}
3646
3647static void btf_datasec_show(const struct btf *btf,
3648			     const struct btf_type *t, u32 type_id,
3649			     void *data, u8 bits_offset,
3650			     struct btf_show *show)
3651{
3652	const struct btf_var_secinfo *vsi;
3653	const struct btf_type *var;
3654	u32 i;
3655
3656	if (!btf_show_start_type(show, t, type_id, data))
3657		return;
3658
3659	btf_show_type_value(show, "section (\"%s\") = {",
3660			    __btf_name_by_offset(btf, t->name_off));
3661	for_each_vsi(i, t, vsi) {
3662		var = btf_type_by_id(btf, vsi->type);
3663		if (i)
3664			btf_show(show, ",");
3665		btf_type_ops(var)->show(btf, var, vsi->type,
3666					data + vsi->offset, bits_offset, show);
3667	}
3668	btf_show_end_type(show);
3669}
3670
3671static const struct btf_kind_operations datasec_ops = {
3672	.check_meta		= btf_datasec_check_meta,
3673	.resolve		= btf_datasec_resolve,
3674	.check_member		= btf_df_check_member,
3675	.check_kflag_member	= btf_df_check_kflag_member,
3676	.log_details		= btf_datasec_log,
3677	.show			= btf_datasec_show,
3678};
3679
3680static s32 btf_float_check_meta(struct btf_verifier_env *env,
3681				const struct btf_type *t,
3682				u32 meta_left)
3683{
3684	if (btf_type_vlen(t)) {
3685		btf_verifier_log_type(env, t, "vlen != 0");
3686		return -EINVAL;
3687	}
3688
3689	if (btf_type_kflag(t)) {
3690		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3691		return -EINVAL;
3692	}
3693
3694	if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 &&
3695	    t->size != 16) {
3696		btf_verifier_log_type(env, t, "Invalid type_size");
3697		return -EINVAL;
3698	}
3699
3700	btf_verifier_log_type(env, t, NULL);
3701
3702	return 0;
3703}
3704
3705static int btf_float_check_member(struct btf_verifier_env *env,
3706				  const struct btf_type *struct_type,
3707				  const struct btf_member *member,
3708				  const struct btf_type *member_type)
3709{
3710	u64 start_offset_bytes;
3711	u64 end_offset_bytes;
3712	u64 misalign_bits;
3713	u64 align_bytes;
3714	u64 align_bits;
3715
3716	/* Different architectures have different alignment requirements, so
3717	 * here we check only for the reasonable minimum. This way we ensure
3718	 * that types after CO-RE can pass the kernel BTF verifier.
3719	 */
3720	align_bytes = min_t(u64, sizeof(void *), member_type->size);
3721	align_bits = align_bytes * BITS_PER_BYTE;
3722	div64_u64_rem(member->offset, align_bits, &misalign_bits);
3723	if (misalign_bits) {
3724		btf_verifier_log_member(env, struct_type, member,
3725					"Member is not properly aligned");
3726		return -EINVAL;
3727	}
3728
3729	start_offset_bytes = member->offset / BITS_PER_BYTE;
3730	end_offset_bytes = start_offset_bytes + member_type->size;
3731	if (end_offset_bytes > struct_type->size) {
3732		btf_verifier_log_member(env, struct_type, member,
3733					"Member exceeds struct_size");
3734		return -EINVAL;
3735	}
3736
3737	return 0;
3738}
3739
3740static void btf_float_log(struct btf_verifier_env *env,
3741			  const struct btf_type *t)
3742{
3743	btf_verifier_log(env, "size=%u", t->size);
3744}
3745
3746static const struct btf_kind_operations float_ops = {
3747	.check_meta = btf_float_check_meta,
3748	.resolve = btf_df_resolve,
3749	.check_member = btf_float_check_member,
3750	.check_kflag_member = btf_generic_check_kflag_member,
3751	.log_details = btf_float_log,
3752	.show = btf_df_show,
3753};
3754
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3755static int btf_func_proto_check(struct btf_verifier_env *env,
3756				const struct btf_type *t)
3757{
3758	const struct btf_type *ret_type;
3759	const struct btf_param *args;
3760	const struct btf *btf;
3761	u16 nr_args, i;
3762	int err;
3763
3764	btf = env->btf;
3765	args = (const struct btf_param *)(t + 1);
3766	nr_args = btf_type_vlen(t);
3767
3768	/* Check func return type which could be "void" (t->type == 0) */
3769	if (t->type) {
3770		u32 ret_type_id = t->type;
3771
3772		ret_type = btf_type_by_id(btf, ret_type_id);
3773		if (!ret_type) {
3774			btf_verifier_log_type(env, t, "Invalid return type");
3775			return -EINVAL;
3776		}
3777
 
 
 
 
 
3778		if (btf_type_needs_resolve(ret_type) &&
3779		    !env_type_is_resolved(env, ret_type_id)) {
3780			err = btf_resolve(env, ret_type, ret_type_id);
3781			if (err)
3782				return err;
3783		}
3784
3785		/* Ensure the return type is a type that has a size */
3786		if (!btf_type_id_size(btf, &ret_type_id, NULL)) {
3787			btf_verifier_log_type(env, t, "Invalid return type");
3788			return -EINVAL;
3789		}
3790	}
3791
3792	if (!nr_args)
3793		return 0;
3794
3795	/* Last func arg type_id could be 0 if it is a vararg */
3796	if (!args[nr_args - 1].type) {
3797		if (args[nr_args - 1].name_off) {
3798			btf_verifier_log_type(env, t, "Invalid arg#%u",
3799					      nr_args);
3800			return -EINVAL;
3801		}
3802		nr_args--;
3803	}
3804
3805	err = 0;
3806	for (i = 0; i < nr_args; i++) {
3807		const struct btf_type *arg_type;
3808		u32 arg_type_id;
3809
3810		arg_type_id = args[i].type;
3811		arg_type = btf_type_by_id(btf, arg_type_id);
3812		if (!arg_type) {
3813			btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3814			err = -EINVAL;
3815			break;
 
 
 
 
3816		}
3817
3818		if (args[i].name_off &&
3819		    (!btf_name_offset_valid(btf, args[i].name_off) ||
3820		     !btf_name_valid_identifier(btf, args[i].name_off))) {
3821			btf_verifier_log_type(env, t,
3822					      "Invalid arg#%u", i + 1);
3823			err = -EINVAL;
3824			break;
3825		}
3826
3827		if (btf_type_needs_resolve(arg_type) &&
3828		    !env_type_is_resolved(env, arg_type_id)) {
3829			err = btf_resolve(env, arg_type, arg_type_id);
3830			if (err)
3831				break;
3832		}
3833
3834		if (!btf_type_id_size(btf, &arg_type_id, NULL)) {
3835			btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3836			err = -EINVAL;
3837			break;
3838		}
3839	}
3840
3841	return err;
3842}
3843
3844static int btf_func_check(struct btf_verifier_env *env,
3845			  const struct btf_type *t)
3846{
3847	const struct btf_type *proto_type;
3848	const struct btf_param *args;
3849	const struct btf *btf;
3850	u16 nr_args, i;
3851
3852	btf = env->btf;
3853	proto_type = btf_type_by_id(btf, t->type);
3854
3855	if (!proto_type || !btf_type_is_func_proto(proto_type)) {
3856		btf_verifier_log_type(env, t, "Invalid type_id");
3857		return -EINVAL;
3858	}
3859
3860	args = (const struct btf_param *)(proto_type + 1);
3861	nr_args = btf_type_vlen(proto_type);
3862	for (i = 0; i < nr_args; i++) {
3863		if (!args[i].name_off && args[i].type) {
3864			btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3865			return -EINVAL;
3866		}
3867	}
3868
3869	return 0;
3870}
3871
3872static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
3873	[BTF_KIND_INT] = &int_ops,
3874	[BTF_KIND_PTR] = &ptr_ops,
3875	[BTF_KIND_ARRAY] = &array_ops,
3876	[BTF_KIND_STRUCT] = &struct_ops,
3877	[BTF_KIND_UNION] = &struct_ops,
3878	[BTF_KIND_ENUM] = &enum_ops,
3879	[BTF_KIND_FWD] = &fwd_ops,
3880	[BTF_KIND_TYPEDEF] = &modifier_ops,
3881	[BTF_KIND_VOLATILE] = &modifier_ops,
3882	[BTF_KIND_CONST] = &modifier_ops,
3883	[BTF_KIND_RESTRICT] = &modifier_ops,
3884	[BTF_KIND_FUNC] = &func_ops,
3885	[BTF_KIND_FUNC_PROTO] = &func_proto_ops,
3886	[BTF_KIND_VAR] = &var_ops,
3887	[BTF_KIND_DATASEC] = &datasec_ops,
3888	[BTF_KIND_FLOAT] = &float_ops,
 
 
 
3889};
3890
3891static s32 btf_check_meta(struct btf_verifier_env *env,
3892			  const struct btf_type *t,
3893			  u32 meta_left)
3894{
3895	u32 saved_meta_left = meta_left;
3896	s32 var_meta_size;
3897
3898	if (meta_left < sizeof(*t)) {
3899		btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
3900				 env->log_type_id, meta_left, sizeof(*t));
3901		return -EINVAL;
3902	}
3903	meta_left -= sizeof(*t);
3904
3905	if (t->info & ~BTF_INFO_MASK) {
3906		btf_verifier_log(env, "[%u] Invalid btf_info:%x",
3907				 env->log_type_id, t->info);
3908		return -EINVAL;
3909	}
3910
3911	if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
3912	    BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
3913		btf_verifier_log(env, "[%u] Invalid kind:%u",
3914				 env->log_type_id, BTF_INFO_KIND(t->info));
3915		return -EINVAL;
3916	}
3917
3918	if (!btf_name_offset_valid(env->btf, t->name_off)) {
3919		btf_verifier_log(env, "[%u] Invalid name_offset:%u",
3920				 env->log_type_id, t->name_off);
3921		return -EINVAL;
3922	}
3923
3924	var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
3925	if (var_meta_size < 0)
3926		return var_meta_size;
3927
3928	meta_left -= var_meta_size;
3929
3930	return saved_meta_left - meta_left;
3931}
3932
3933static int btf_check_all_metas(struct btf_verifier_env *env)
3934{
3935	struct btf *btf = env->btf;
3936	struct btf_header *hdr;
3937	void *cur, *end;
3938
3939	hdr = &btf->hdr;
3940	cur = btf->nohdr_data + hdr->type_off;
3941	end = cur + hdr->type_len;
3942
3943	env->log_type_id = btf->base_btf ? btf->start_id : 1;
3944	while (cur < end) {
3945		struct btf_type *t = cur;
3946		s32 meta_size;
3947
3948		meta_size = btf_check_meta(env, t, end - cur);
3949		if (meta_size < 0)
3950			return meta_size;
3951
3952		btf_add_type(env, t);
3953		cur += meta_size;
3954		env->log_type_id++;
3955	}
3956
3957	return 0;
3958}
3959
3960static bool btf_resolve_valid(struct btf_verifier_env *env,
3961			      const struct btf_type *t,
3962			      u32 type_id)
3963{
3964	struct btf *btf = env->btf;
3965
3966	if (!env_type_is_resolved(env, type_id))
3967		return false;
3968
3969	if (btf_type_is_struct(t) || btf_type_is_datasec(t))
3970		return !btf_resolved_type_id(btf, type_id) &&
3971		       !btf_resolved_type_size(btf, type_id);
3972
 
 
 
 
3973	if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||
3974	    btf_type_is_var(t)) {
3975		t = btf_type_id_resolve(btf, &type_id);
3976		return t &&
3977		       !btf_type_is_modifier(t) &&
3978		       !btf_type_is_var(t) &&
3979		       !btf_type_is_datasec(t);
3980	}
3981
3982	if (btf_type_is_array(t)) {
3983		const struct btf_array *array = btf_type_array(t);
3984		const struct btf_type *elem_type;
3985		u32 elem_type_id = array->type;
3986		u32 elem_size;
3987
3988		elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
3989		return elem_type && !btf_type_is_modifier(elem_type) &&
3990			(array->nelems * elem_size ==
3991			 btf_resolved_type_size(btf, type_id));
3992	}
3993
3994	return false;
3995}
3996
3997static int btf_resolve(struct btf_verifier_env *env,
3998		       const struct btf_type *t, u32 type_id)
3999{
4000	u32 save_log_type_id = env->log_type_id;
4001	const struct resolve_vertex *v;
4002	int err = 0;
4003
4004	env->resolve_mode = RESOLVE_TBD;
4005	env_stack_push(env, t, type_id);
4006	while (!err && (v = env_stack_peak(env))) {
4007		env->log_type_id = v->type_id;
4008		err = btf_type_ops(v->t)->resolve(env, v);
4009	}
4010
4011	env->log_type_id = type_id;
4012	if (err == -E2BIG) {
4013		btf_verifier_log_type(env, t,
4014				      "Exceeded max resolving depth:%u",
4015				      MAX_RESOLVE_DEPTH);
4016	} else if (err == -EEXIST) {
4017		btf_verifier_log_type(env, t, "Loop detected");
4018	}
4019
4020	/* Final sanity check */
4021	if (!err && !btf_resolve_valid(env, t, type_id)) {
4022		btf_verifier_log_type(env, t, "Invalid resolve state");
4023		err = -EINVAL;
4024	}
4025
4026	env->log_type_id = save_log_type_id;
4027	return err;
4028}
4029
4030static int btf_check_all_types(struct btf_verifier_env *env)
4031{
4032	struct btf *btf = env->btf;
4033	const struct btf_type *t;
4034	u32 type_id, i;
4035	int err;
4036
4037	err = env_resolve_init(env);
4038	if (err)
4039		return err;
4040
4041	env->phase++;
4042	for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) {
4043		type_id = btf->start_id + i;
4044		t = btf_type_by_id(btf, type_id);
4045
4046		env->log_type_id = type_id;
4047		if (btf_type_needs_resolve(t) &&
4048		    !env_type_is_resolved(env, type_id)) {
4049			err = btf_resolve(env, t, type_id);
4050			if (err)
4051				return err;
4052		}
4053
4054		if (btf_type_is_func_proto(t)) {
4055			err = btf_func_proto_check(env, t);
4056			if (err)
4057				return err;
4058		}
4059
4060		if (btf_type_is_func(t)) {
4061			err = btf_func_check(env, t);
4062			if (err)
4063				return err;
4064		}
4065	}
4066
4067	return 0;
4068}
4069
4070static int btf_parse_type_sec(struct btf_verifier_env *env)
4071{
4072	const struct btf_header *hdr = &env->btf->hdr;
4073	int err;
4074
4075	/* Type section must align to 4 bytes */
4076	if (hdr->type_off & (sizeof(u32) - 1)) {
4077		btf_verifier_log(env, "Unaligned type_off");
4078		return -EINVAL;
4079	}
4080
4081	if (!env->btf->base_btf && !hdr->type_len) {
4082		btf_verifier_log(env, "No type found");
4083		return -EINVAL;
4084	}
4085
4086	err = btf_check_all_metas(env);
4087	if (err)
4088		return err;
4089
4090	return btf_check_all_types(env);
4091}
4092
4093static int btf_parse_str_sec(struct btf_verifier_env *env)
4094{
4095	const struct btf_header *hdr;
4096	struct btf *btf = env->btf;
4097	const char *start, *end;
4098
4099	hdr = &btf->hdr;
4100	start = btf->nohdr_data + hdr->str_off;
4101	end = start + hdr->str_len;
4102
4103	if (end != btf->data + btf->data_size) {
4104		btf_verifier_log(env, "String section is not at the end");
4105		return -EINVAL;
4106	}
4107
4108	btf->strings = start;
4109
4110	if (btf->base_btf && !hdr->str_len)
4111		return 0;
4112	if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) {
4113		btf_verifier_log(env, "Invalid string section");
4114		return -EINVAL;
4115	}
4116	if (!btf->base_btf && start[0]) {
4117		btf_verifier_log(env, "Invalid string section");
4118		return -EINVAL;
4119	}
4120
4121	return 0;
4122}
4123
4124static const size_t btf_sec_info_offset[] = {
4125	offsetof(struct btf_header, type_off),
4126	offsetof(struct btf_header, str_off),
4127};
4128
4129static int btf_sec_info_cmp(const void *a, const void *b)
4130{
4131	const struct btf_sec_info *x = a;
4132	const struct btf_sec_info *y = b;
4133
4134	return (int)(x->off - y->off) ? : (int)(x->len - y->len);
4135}
4136
4137static int btf_check_sec_info(struct btf_verifier_env *env,
4138			      u32 btf_data_size)
4139{
4140	struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
4141	u32 total, expected_total, i;
4142	const struct btf_header *hdr;
4143	const struct btf *btf;
4144
4145	btf = env->btf;
4146	hdr = &btf->hdr;
4147
4148	/* Populate the secs from hdr */
4149	for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
4150		secs[i] = *(struct btf_sec_info *)((void *)hdr +
4151						   btf_sec_info_offset[i]);
4152
4153	sort(secs, ARRAY_SIZE(btf_sec_info_offset),
4154	     sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
4155
4156	/* Check for gaps and overlap among sections */
4157	total = 0;
4158	expected_total = btf_data_size - hdr->hdr_len;
4159	for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
4160		if (expected_total < secs[i].off) {
4161			btf_verifier_log(env, "Invalid section offset");
4162			return -EINVAL;
4163		}
4164		if (total < secs[i].off) {
4165			/* gap */
4166			btf_verifier_log(env, "Unsupported section found");
4167			return -EINVAL;
4168		}
4169		if (total > secs[i].off) {
4170			btf_verifier_log(env, "Section overlap found");
4171			return -EINVAL;
4172		}
4173		if (expected_total - total < secs[i].len) {
4174			btf_verifier_log(env,
4175					 "Total section length too long");
4176			return -EINVAL;
4177		}
4178		total += secs[i].len;
4179	}
4180
4181	/* There is data other than hdr and known sections */
4182	if (expected_total != total) {
4183		btf_verifier_log(env, "Unsupported section found");
4184		return -EINVAL;
4185	}
4186
4187	return 0;
4188}
4189
4190static int btf_parse_hdr(struct btf_verifier_env *env)
4191{
4192	u32 hdr_len, hdr_copy, btf_data_size;
4193	const struct btf_header *hdr;
4194	struct btf *btf;
4195	int err;
4196
4197	btf = env->btf;
4198	btf_data_size = btf->data_size;
4199
4200	if (btf_data_size <
4201	    offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
4202		btf_verifier_log(env, "hdr_len not found");
4203		return -EINVAL;
4204	}
4205
4206	hdr = btf->data;
4207	hdr_len = hdr->hdr_len;
4208	if (btf_data_size < hdr_len) {
4209		btf_verifier_log(env, "btf_header not found");
4210		return -EINVAL;
4211	}
4212
4213	/* Ensure the unsupported header fields are zero */
4214	if (hdr_len > sizeof(btf->hdr)) {
4215		u8 *expected_zero = btf->data + sizeof(btf->hdr);
4216		u8 *end = btf->data + hdr_len;
4217
4218		for (; expected_zero < end; expected_zero++) {
4219			if (*expected_zero) {
4220				btf_verifier_log(env, "Unsupported btf_header");
4221				return -E2BIG;
4222			}
4223		}
4224	}
4225
4226	hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
4227	memcpy(&btf->hdr, btf->data, hdr_copy);
4228
4229	hdr = &btf->hdr;
4230
4231	btf_verifier_log_hdr(env, btf_data_size);
4232
4233	if (hdr->magic != BTF_MAGIC) {
4234		btf_verifier_log(env, "Invalid magic");
4235		return -EINVAL;
4236	}
4237
4238	if (hdr->version != BTF_VERSION) {
4239		btf_verifier_log(env, "Unsupported version");
4240		return -ENOTSUPP;
4241	}
4242
4243	if (hdr->flags) {
4244		btf_verifier_log(env, "Unsupported flags");
4245		return -ENOTSUPP;
4246	}
4247
4248	if (!btf->base_btf && btf_data_size == hdr->hdr_len) {
4249		btf_verifier_log(env, "No data");
4250		return -EINVAL;
4251	}
4252
4253	err = btf_check_sec_info(env, btf_data_size);
4254	if (err)
4255		return err;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4256
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4257	return 0;
4258}
4259
4260static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
4261			     u32 log_level, char __user *log_ubuf, u32 log_size)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4262{
 
 
 
4263	struct btf_verifier_env *env = NULL;
4264	struct bpf_verifier_log *log;
4265	struct btf *btf = NULL;
4266	u8 *data;
4267	int err;
4268
4269	if (btf_data_size > BTF_MAX_SIZE)
4270		return ERR_PTR(-E2BIG);
4271
4272	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4273	if (!env)
4274		return ERR_PTR(-ENOMEM);
4275
4276	log = &env->log;
4277	if (log_level || log_ubuf || log_size) {
4278		/* user requested verbose verifier output
4279		 * and supplied buffer to store the verification trace
4280		 */
4281		log->level = log_level;
4282		log->ubuf = log_ubuf;
4283		log->len_total = log_size;
4284
4285		/* log attributes have to be sane */
4286		if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
4287		    !log->level || !log->ubuf) {
4288			err = -EINVAL;
4289			goto errout;
4290		}
4291	}
4292
4293	btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4294	if (!btf) {
4295		err = -ENOMEM;
4296		goto errout;
4297	}
4298	env->btf = btf;
4299
4300	data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
4301	if (!data) {
4302		err = -ENOMEM;
4303		goto errout;
4304	}
4305
4306	btf->data = data;
4307	btf->data_size = btf_data_size;
4308
4309	if (copy_from_bpfptr(data, btf_data, btf_data_size)) {
4310		err = -EFAULT;
4311		goto errout;
4312	}
4313
4314	err = btf_parse_hdr(env);
4315	if (err)
4316		goto errout;
4317
4318	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4319
4320	err = btf_parse_str_sec(env);
4321	if (err)
4322		goto errout;
4323
4324	err = btf_parse_type_sec(env);
4325	if (err)
4326		goto errout;
4327
4328	if (log->level && bpf_verifier_log_full(log)) {
4329		err = -ENOSPC;
 
 
 
 
 
4330		goto errout;
4331	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4332
4333	btf_verifier_env_free(env);
4334	refcount_set(&btf->refcnt, 1);
4335	return btf;
4336
 
 
4337errout:
 
 
 
 
 
4338	btf_verifier_env_free(env);
4339	if (btf)
4340		btf_free(btf);
4341	return ERR_PTR(err);
4342}
4343
4344extern char __weak __start_BTF[];
4345extern char __weak __stop_BTF[];
4346extern struct btf *btf_vmlinux;
4347
4348#define BPF_MAP_TYPE(_id, _ops)
4349#define BPF_LINK_TYPE(_id, _name)
4350static union {
4351	struct bpf_ctx_convert {
4352#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4353	prog_ctx_type _id##_prog; \
4354	kern_ctx_type _id##_kern;
4355#include <linux/bpf_types.h>
4356#undef BPF_PROG_TYPE
4357	} *__t;
4358	/* 't' is written once under lock. Read many times. */
4359	const struct btf_type *t;
4360} bpf_ctx_convert;
4361enum {
4362#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4363	__ctx_convert##_id,
4364#include <linux/bpf_types.h>
4365#undef BPF_PROG_TYPE
4366	__ctx_convert_unused, /* to avoid empty enum in extreme .config */
4367};
4368static u8 bpf_ctx_convert_map[] = {
4369#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4370	[_id] = __ctx_convert##_id,
4371#include <linux/bpf_types.h>
4372#undef BPF_PROG_TYPE
4373	0, /* avoid empty array */
4374};
4375#undef BPF_MAP_TYPE
4376#undef BPF_LINK_TYPE
4377
4378static const struct btf_member *
4379btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
4380		      const struct btf_type *t, enum bpf_prog_type prog_type,
4381		      int arg)
4382{
4383	const struct btf_type *conv_struct;
4384	const struct btf_type *ctx_struct;
4385	const struct btf_member *ctx_type;
4386	const char *tname, *ctx_tname;
4387
4388	conv_struct = bpf_ctx_convert.t;
4389	if (!conv_struct) {
4390		bpf_log(log, "btf_vmlinux is malformed\n");
4391		return NULL;
4392	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4393	t = btf_type_by_id(btf, t->type);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4394	while (btf_type_is_modifier(t))
4395		t = btf_type_by_id(btf, t->type);
4396	if (!btf_type_is_struct(t)) {
4397		/* Only pointer to struct is supported for now.
4398		 * That means that BPF_PROG_TYPE_TRACEPOINT with BTF
4399		 * is not supported yet.
4400		 * BPF_PROG_TYPE_RAW_TRACEPOINT is fine.
4401		 */
4402		return NULL;
4403	}
4404	tname = btf_name_by_offset(btf, t->name_off);
4405	if (!tname) {
4406		bpf_log(log, "arg#%d struct doesn't have a name\n", arg);
4407		return NULL;
4408	}
4409	/* prog_type is valid bpf program type. No need for bounds check. */
4410	ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2;
4411	/* ctx_struct is a pointer to prog_ctx_type in vmlinux.
4412	 * Like 'struct __sk_buff'
4413	 */
4414	ctx_struct = btf_type_by_id(btf_vmlinux, ctx_type->type);
4415	if (!ctx_struct)
4416		/* should not happen */
4417		return NULL;
4418	ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_struct->name_off);
 
 
4419	if (!ctx_tname) {
4420		/* should not happen */
4421		bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n");
4422		return NULL;
4423	}
 
 
 
4424	/* only compare that prog's ctx type name is the same as
4425	 * kernel expects. No need to compare field by field.
4426	 * It's ok for bpf prog to do:
4427	 * struct __sk_buff {};
4428	 * int socket_filter_bpf_prog(struct __sk_buff *skb)
4429	 * { // no fields of skb are ever used }
4430	 */
4431	if (strcmp(ctx_tname, tname))
4432		return NULL;
4433	return ctx_type;
 
 
 
 
 
 
 
 
 
 
 
 
4434}
4435
4436static const struct bpf_map_ops * const btf_vmlinux_map_ops[] = {
4437#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
4438#define BPF_LINK_TYPE(_id, _name)
4439#define BPF_MAP_TYPE(_id, _ops) \
4440	[_id] = &_ops,
4441#include <linux/bpf_types.h>
4442#undef BPF_PROG_TYPE
4443#undef BPF_LINK_TYPE
4444#undef BPF_MAP_TYPE
4445};
4446
4447static int btf_vmlinux_map_ids_init(const struct btf *btf,
4448				    struct bpf_verifier_log *log)
 
 
4449{
4450	const struct bpf_map_ops *ops;
4451	int i, btf_id;
4452
4453	for (i = 0; i < ARRAY_SIZE(btf_vmlinux_map_ops); ++i) {
4454		ops = btf_vmlinux_map_ops[i];
4455		if (!ops || (!ops->map_btf_name && !ops->map_btf_id))
4456			continue;
4457		if (!ops->map_btf_name || !ops->map_btf_id) {
4458			bpf_log(log, "map type %d is misconfigured\n", i);
4459			return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4460		}
4461		btf_id = btf_find_by_name_kind(btf, ops->map_btf_name,
4462					       BTF_KIND_STRUCT);
4463		if (btf_id < 0)
4464			return btf_id;
4465		*ops->map_btf_id = btf_id;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4466	}
4467
4468	return 0;
4469}
4470
4471static int btf_translate_to_vmlinux(struct bpf_verifier_log *log,
4472				     struct btf *btf,
4473				     const struct btf_type *t,
4474				     enum bpf_prog_type prog_type,
4475				     int arg)
4476{
4477	const struct btf_member *prog_ctx_type, *kern_ctx_type;
4478
4479	prog_ctx_type = btf_get_prog_ctx_type(log, btf, t, prog_type, arg);
4480	if (!prog_ctx_type)
4481		return -ENOENT;
4482	kern_ctx_type = prog_ctx_type + 1;
4483	return kern_ctx_type->type;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4484}
4485
4486BTF_ID_LIST(bpf_ctx_convert_btf_id)
4487BTF_ID(struct, bpf_ctx_convert)
4488
4489struct btf *btf_parse_vmlinux(void)
4490{
4491	struct btf_verifier_env *env = NULL;
4492	struct bpf_verifier_log *log;
4493	struct btf *btf = NULL;
4494	int err;
4495
4496	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4497	if (!env)
4498		return ERR_PTR(-ENOMEM);
4499
4500	log = &env->log;
4501	log->level = BPF_LOG_KERNEL;
4502
4503	btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4504	if (!btf) {
4505		err = -ENOMEM;
4506		goto errout;
4507	}
4508	env->btf = btf;
4509
4510	btf->data = __start_BTF;
4511	btf->data_size = __stop_BTF - __start_BTF;
4512	btf->kernel_btf = true;
4513	snprintf(btf->name, sizeof(btf->name), "vmlinux");
4514
4515	err = btf_parse_hdr(env);
4516	if (err)
4517		goto errout;
4518
4519	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4520
4521	err = btf_parse_str_sec(env);
4522	if (err)
4523		goto errout;
4524
4525	err = btf_check_all_metas(env);
4526	if (err)
4527		goto errout;
4528
4529	/* btf_parse_vmlinux() runs under bpf_verifier_lock */
4530	bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
4531
4532	/* find bpf map structs for map_ptr access checking */
4533	err = btf_vmlinux_map_ids_init(btf, log);
4534	if (err < 0)
4535		goto errout;
4536
4537	bpf_struct_ops_init(btf, log);
 
4538
4539	refcount_set(&btf->refcnt, 1);
4540
4541	err = btf_alloc_id(btf);
4542	if (err)
4543		goto errout;
4544
4545	btf_verifier_env_free(env);
4546	return btf;
4547
4548errout:
4549	btf_verifier_env_free(env);
4550	if (btf) {
4551		kvfree(btf->types);
4552		kfree(btf);
4553	}
4554	return ERR_PTR(err);
4555}
4556
4557#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
4558
4559static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size)
4560{
4561	struct btf_verifier_env *env = NULL;
4562	struct bpf_verifier_log *log;
4563	struct btf *btf = NULL, *base_btf;
4564	int err;
4565
4566	base_btf = bpf_get_btf_vmlinux();
4567	if (IS_ERR(base_btf))
4568		return base_btf;
4569	if (!base_btf)
4570		return ERR_PTR(-EINVAL);
4571
4572	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4573	if (!env)
4574		return ERR_PTR(-ENOMEM);
4575
4576	log = &env->log;
4577	log->level = BPF_LOG_KERNEL;
4578
4579	btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4580	if (!btf) {
4581		err = -ENOMEM;
4582		goto errout;
4583	}
4584	env->btf = btf;
4585
4586	btf->base_btf = base_btf;
4587	btf->start_id = base_btf->nr_types;
4588	btf->start_str_off = base_btf->hdr.str_len;
4589	btf->kernel_btf = true;
4590	snprintf(btf->name, sizeof(btf->name), "%s", module_name);
4591
4592	btf->data = kvmalloc(data_size, GFP_KERNEL | __GFP_NOWARN);
4593	if (!btf->data) {
4594		err = -ENOMEM;
4595		goto errout;
4596	}
4597	memcpy(btf->data, data, data_size);
4598	btf->data_size = data_size;
4599
4600	err = btf_parse_hdr(env);
4601	if (err)
4602		goto errout;
4603
4604	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4605
4606	err = btf_parse_str_sec(env);
4607	if (err)
4608		goto errout;
4609
4610	err = btf_check_all_metas(env);
4611	if (err)
4612		goto errout;
4613
 
 
 
 
4614	btf_verifier_env_free(env);
4615	refcount_set(&btf->refcnt, 1);
4616	return btf;
4617
4618errout:
4619	btf_verifier_env_free(env);
4620	if (btf) {
4621		kvfree(btf->data);
4622		kvfree(btf->types);
4623		kfree(btf);
4624	}
4625	return ERR_PTR(err);
4626}
4627
4628#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */
4629
4630struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
4631{
4632	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
4633
4634	if (tgt_prog)
4635		return tgt_prog->aux->btf;
4636	else
4637		return prog->aux->attach_btf;
4638}
4639
4640static bool is_string_ptr(struct btf *btf, const struct btf_type *t)
4641{
4642	/* t comes in already as a pointer */
4643	t = btf_type_by_id(btf, t->type);
4644
4645	/* allow const */
4646	if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)
4647		t = btf_type_by_id(btf, t->type);
4648
4649	/* char, signed char, unsigned char */
4650	return btf_type_is_int(t) && t->size == 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4651}
4652
4653bool btf_ctx_access(int off, int size, enum bpf_access_type type,
4654		    const struct bpf_prog *prog,
4655		    struct bpf_insn_access_aux *info)
4656{
4657	const struct btf_type *t = prog->aux->attach_func_proto;
4658	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
4659	struct btf *btf = bpf_prog_get_target_btf(prog);
4660	const char *tname = prog->aux->attach_func_name;
4661	struct bpf_verifier_log *log = info->log;
4662	const struct btf_param *args;
 
4663	u32 nr_args, arg;
4664	int i, ret;
4665
4666	if (off % 8) {
4667		bpf_log(log, "func '%s' offset %d is not multiple of 8\n",
4668			tname, off);
4669		return false;
4670	}
4671	arg = off / 8;
4672	args = (const struct btf_param *)(t + 1);
4673	/* if (t == NULL) Fall back to default BPF prog with
4674	 * MAX_BPF_FUNC_REG_ARGS u64 arguments.
4675	 */
4676	nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS;
4677	if (prog->aux->attach_btf_trace) {
4678		/* skip first 'void *__data' argument in btf_trace_##name typedef */
4679		args++;
4680		nr_args--;
4681	}
4682
4683	if (arg > nr_args) {
4684		bpf_log(log, "func '%s' doesn't have %d-th argument\n",
4685			tname, arg + 1);
4686		return false;
4687	}
4688
4689	if (arg == nr_args) {
4690		switch (prog->expected_attach_type) {
 
4691		case BPF_LSM_MAC:
4692		case BPF_TRACE_FEXIT:
4693			/* When LSM programs are attached to void LSM hooks
4694			 * they use FEXIT trampolines and when attached to
4695			 * int LSM hooks, they use MODIFY_RETURN trampolines.
4696			 *
4697			 * While the LSM programs are BPF_MODIFY_RETURN-like
4698			 * the check:
4699			 *
4700			 *	if (ret_type != 'int')
4701			 *		return -EINVAL;
4702			 *
4703			 * is _not_ done here. This is still safe as LSM hooks
4704			 * have only void and int return types.
4705			 */
4706			if (!t)
4707				return true;
4708			t = btf_type_by_id(btf, t->type);
4709			break;
4710		case BPF_MODIFY_RETURN:
4711			/* For now the BPF_MODIFY_RETURN can only be attached to
4712			 * functions that return an int.
4713			 */
4714			if (!t)
4715				return false;
4716
4717			t = btf_type_skip_modifiers(btf, t->type, NULL);
4718			if (!btf_type_is_small_int(t)) {
4719				bpf_log(log,
4720					"ret type %s not allowed for fmod_ret\n",
4721					btf_kind_str[BTF_INFO_KIND(t->info)]);
4722				return false;
4723			}
4724			break;
4725		default:
4726			bpf_log(log, "func '%s' doesn't have %d-th argument\n",
4727				tname, arg + 1);
4728			return false;
4729		}
4730	} else {
4731		if (!t)
4732			/* Default prog with MAX_BPF_FUNC_REG_ARGS args */
4733			return true;
4734		t = btf_type_by_id(btf, args[arg].type);
4735	}
4736
4737	/* skip modifiers */
4738	while (btf_type_is_modifier(t))
4739		t = btf_type_by_id(btf, t->type);
4740	if (btf_type_is_small_int(t) || btf_type_is_enum(t))
4741		/* accessing a scalar */
4742		return true;
4743	if (!btf_type_is_ptr(t)) {
4744		bpf_log(log,
4745			"func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n",
4746			tname, arg,
4747			__btf_name_by_offset(btf, t->name_off),
4748			btf_kind_str[BTF_INFO_KIND(t->info)]);
4749		return false;
4750	}
4751
4752	/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
4753	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4754		const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
 
4755
4756		if (ctx_arg_info->offset == off &&
4757		    (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL ||
4758		     ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
 
4759			info->reg_type = ctx_arg_info->reg_type;
4760			return true;
4761		}
4762	}
4763
4764	if (t->type == 0)
4765		/* This is a pointer to void.
4766		 * It is the same as scalar from the verifier safety pov.
4767		 * No further pointer walking is allowed.
4768		 */
4769		return true;
4770
4771	if (is_string_ptr(btf, t))
4772		return true;
4773
4774	/* this is a pointer to another type */
4775	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4776		const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
4777
4778		if (ctx_arg_info->offset == off) {
 
 
 
 
 
4779			info->reg_type = ctx_arg_info->reg_type;
4780			info->btf = btf_vmlinux;
4781			info->btf_id = ctx_arg_info->btf_id;
4782			return true;
4783		}
4784	}
4785
4786	info->reg_type = PTR_TO_BTF_ID;
 
 
 
4787	if (tgt_prog) {
4788		enum bpf_prog_type tgt_type;
4789
4790		if (tgt_prog->type == BPF_PROG_TYPE_EXT)
4791			tgt_type = tgt_prog->aux->saved_dst_prog_type;
4792		else
4793			tgt_type = tgt_prog->type;
4794
4795		ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
4796		if (ret > 0) {
4797			info->btf = btf_vmlinux;
4798			info->btf_id = ret;
4799			return true;
4800		} else {
4801			return false;
4802		}
4803	}
4804
4805	info->btf = btf;
4806	info->btf_id = t->type;
4807	t = btf_type_by_id(btf, t->type);
 
 
 
 
 
 
 
 
 
4808	/* skip modifiers */
4809	while (btf_type_is_modifier(t)) {
4810		info->btf_id = t->type;
4811		t = btf_type_by_id(btf, t->type);
4812	}
4813	if (!btf_type_is_struct(t)) {
4814		bpf_log(log,
4815			"func '%s' arg%d type %s is not a struct\n",
4816			tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]);
4817		return false;
4818	}
4819	bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
4820		tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)],
4821		__btf_name_by_offset(btf, t->name_off));
4822	return true;
4823}
 
4824
4825enum bpf_struct_walk_result {
4826	/* < 0 error */
4827	WALK_SCALAR = 0,
4828	WALK_PTR,
4829	WALK_STRUCT,
4830};
4831
4832static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
4833			   const struct btf_type *t, int off, int size,
4834			   u32 *next_btf_id)
 
4835{
4836	u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
4837	const struct btf_type *mtype, *elem_type = NULL;
4838	const struct btf_member *member;
4839	const char *tname, *mname;
4840	u32 vlen, elem_id, mid;
4841
4842again:
 
 
4843	tname = __btf_name_by_offset(btf, t->name_off);
4844	if (!btf_type_is_struct(t)) {
4845		bpf_log(log, "Type '%s' is not a struct\n", tname);
4846		return -EINVAL;
4847	}
4848
4849	vlen = btf_type_vlen(t);
 
 
 
 
 
 
 
 
4850	if (off + size > t->size) {
4851		/* If the last element is a variable size array, we may
4852		 * need to relax the rule.
4853		 */
4854		struct btf_array *array_elem;
4855
4856		if (vlen == 0)
4857			goto error;
4858
4859		member = btf_type_member(t) + vlen - 1;
4860		mtype = btf_type_skip_modifiers(btf, member->type,
4861						NULL);
4862		if (!btf_type_is_array(mtype))
4863			goto error;
4864
4865		array_elem = (struct btf_array *)(mtype + 1);
4866		if (array_elem->nelems != 0)
4867			goto error;
4868
4869		moff = btf_member_bit_offset(t, member) / 8;
4870		if (off < moff)
4871			goto error;
4872
4873		/* Only allow structure for now, can be relaxed for
4874		 * other types later.
4875		 */
4876		t = btf_type_skip_modifiers(btf, array_elem->type,
4877					    NULL);
 
 
 
 
4878		if (!btf_type_is_struct(t))
4879			goto error;
4880
4881		off = (off - moff) % t->size;
4882		goto again;
4883
4884error:
4885		bpf_log(log, "access beyond struct %s at off %u size %u\n",
4886			tname, off, size);
4887		return -EACCES;
4888	}
4889
4890	for_each_member(i, t, member) {
4891		/* offset of the field in bytes */
4892		moff = btf_member_bit_offset(t, member) / 8;
4893		if (off + size <= moff)
4894			/* won't find anything, field is already too far */
4895			break;
4896
4897		if (btf_member_bitfield_size(t, member)) {
4898			u32 end_bit = btf_member_bit_offset(t, member) +
4899				btf_member_bitfield_size(t, member);
4900
4901			/* off <= moff instead of off == moff because clang
4902			 * does not generate a BTF member for anonymous
4903			 * bitfield like the ":16" here:
4904			 * struct {
4905			 *	int :16;
4906			 *	int x:8;
4907			 * };
4908			 */
4909			if (off <= moff &&
4910			    BITS_ROUNDUP_BYTES(end_bit) <= off + size)
4911				return WALK_SCALAR;
4912
4913			/* off may be accessing a following member
4914			 *
4915			 * or
4916			 *
4917			 * Doing partial access at either end of this
4918			 * bitfield.  Continue on this case also to
4919			 * treat it as not accessing this bitfield
4920			 * and eventually error out as field not
4921			 * found to keep it simple.
4922			 * It could be relaxed if there was a legit
4923			 * partial access case later.
4924			 */
4925			continue;
4926		}
4927
4928		/* In case of "off" is pointing to holes of a struct */
4929		if (off < moff)
4930			break;
4931
4932		/* type of the field */
4933		mid = member->type;
4934		mtype = btf_type_by_id(btf, member->type);
4935		mname = __btf_name_by_offset(btf, member->name_off);
4936
4937		mtype = __btf_resolve_size(btf, mtype, &msize,
4938					   &elem_type, &elem_id, &total_nelems,
4939					   &mid);
4940		if (IS_ERR(mtype)) {
4941			bpf_log(log, "field %s doesn't have size\n", mname);
4942			return -EFAULT;
4943		}
4944
4945		mtrue_end = moff + msize;
4946		if (off >= mtrue_end)
4947			/* no overlap with member, keep iterating */
4948			continue;
4949
4950		if (btf_type_is_array(mtype)) {
4951			u32 elem_idx;
4952
4953			/* __btf_resolve_size() above helps to
4954			 * linearize a multi-dimensional array.
4955			 *
4956			 * The logic here is treating an array
4957			 * in a struct as the following way:
4958			 *
4959			 * struct outer {
4960			 *	struct inner array[2][2];
4961			 * };
4962			 *
4963			 * looks like:
4964			 *
4965			 * struct outer {
4966			 *	struct inner array_elem0;
4967			 *	struct inner array_elem1;
4968			 *	struct inner array_elem2;
4969			 *	struct inner array_elem3;
4970			 * };
4971			 *
4972			 * When accessing outer->array[1][0], it moves
4973			 * moff to "array_elem2", set mtype to
4974			 * "struct inner", and msize also becomes
4975			 * sizeof(struct inner).  Then most of the
4976			 * remaining logic will fall through without
4977			 * caring the current member is an array or
4978			 * not.
4979			 *
4980			 * Unlike mtype/msize/moff, mtrue_end does not
4981			 * change.  The naming difference ("_true") tells
4982			 * that it is not always corresponding to
4983			 * the current mtype/msize/moff.
4984			 * It is the true end of the current
4985			 * member (i.e. array in this case).  That
4986			 * will allow an int array to be accessed like
4987			 * a scratch space,
4988			 * i.e. allow access beyond the size of
4989			 *      the array's element as long as it is
4990			 *      within the mtrue_end boundary.
4991			 */
4992
4993			/* skip empty array */
4994			if (moff == mtrue_end)
4995				continue;
4996
4997			msize /= total_nelems;
4998			elem_idx = (off - moff) / msize;
4999			moff += elem_idx * msize;
5000			mtype = elem_type;
5001			mid = elem_id;
5002		}
5003
5004		/* the 'off' we're looking for is either equal to start
5005		 * of this field or inside of this struct
5006		 */
5007		if (btf_type_is_struct(mtype)) {
5008			/* our field must be inside that union or struct */
5009			t = mtype;
5010
5011			/* return if the offset matches the member offset */
5012			if (off == moff) {
5013				*next_btf_id = mid;
5014				return WALK_STRUCT;
5015			}
5016
5017			/* adjust offset we're looking for */
5018			off -= moff;
5019			goto again;
5020		}
5021
5022		if (btf_type_is_ptr(mtype)) {
5023			const struct btf_type *stype;
 
5024			u32 id;
5025
5026			if (msize != size || off != moff) {
5027				bpf_log(log,
5028					"cannot access ptr member %s with moff %u in struct %s with off %u size %u\n",
5029					mname, moff, tname, off, size);
5030				return -EACCES;
5031			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5032			stype = btf_type_skip_modifiers(btf, mtype->type, &id);
5033			if (btf_type_is_struct(stype)) {
5034				*next_btf_id = id;
 
 
 
5035				return WALK_PTR;
5036			}
5037		}
5038
5039		/* Allow more flexible access within an int as long as
5040		 * it is within mtrue_end.
5041		 * Since mtrue_end could be the end of an array,
5042		 * that also allows using an array of int as a scratch
5043		 * space. e.g. skb->cb[].
5044		 */
5045		if (off + size > mtrue_end) {
5046			bpf_log(log,
5047				"access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n",
5048				mname, mtrue_end, tname, off, size);
5049			return -EACCES;
5050		}
5051
5052		return WALK_SCALAR;
5053	}
5054	bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
5055	return -EINVAL;
5056}
5057
5058int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
5059		      const struct btf_type *t, int off, int size,
5060		      enum bpf_access_type atype __maybe_unused,
5061		      u32 *next_btf_id)
 
5062{
 
 
 
 
5063	int err;
5064	u32 id;
5065
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5066	do {
5067		err = btf_struct_walk(log, btf, t, off, size, &id);
5068
5069		switch (err) {
5070		case WALK_PTR:
 
 
 
 
 
5071			/* If we found the pointer or scalar on t+off,
5072			 * we're done.
5073			 */
5074			*next_btf_id = id;
 
5075			return PTR_TO_BTF_ID;
5076		case WALK_SCALAR:
5077			return SCALAR_VALUE;
5078		case WALK_STRUCT:
5079			/* We found nested struct, so continue the search
5080			 * by diving in it. At this point the offset is
5081			 * aligned with the new type, so set it to 0.
5082			 */
5083			t = btf_type_by_id(btf, id);
5084			off = 0;
5085			break;
5086		default:
5087			/* It's either error or unknown return value..
5088			 * scream and leave.
5089			 */
5090			if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value"))
5091				return -EINVAL;
5092			return err;
5093		}
5094	} while (t);
5095
5096	return -EINVAL;
5097}
5098
5099/* Check that two BTF types, each specified as an BTF object + id, are exactly
5100 * the same. Trivial ID check is not enough due to module BTFs, because we can
5101 * end up with two different module BTFs, but IDs point to the common type in
5102 * vmlinux BTF.
5103 */
5104static bool btf_types_are_same(const struct btf *btf1, u32 id1,
5105			       const struct btf *btf2, u32 id2)
5106{
5107	if (id1 != id2)
5108		return false;
5109	if (btf1 == btf2)
5110		return true;
5111	return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2);
5112}
5113
5114bool btf_struct_ids_match(struct bpf_verifier_log *log,
5115			  const struct btf *btf, u32 id, int off,
5116			  const struct btf *need_btf, u32 need_type_id)
 
5117{
5118	const struct btf_type *type;
 
5119	int err;
5120
5121	/* Are we already done? */
5122	if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id))
5123		return true;
5124
 
 
 
 
 
5125again:
5126	type = btf_type_by_id(btf, id);
5127	if (!type)
5128		return false;
5129	err = btf_struct_walk(log, btf, type, off, 1, &id);
5130	if (err != WALK_STRUCT)
5131		return false;
5132
5133	/* We found nested struct object. If it matches
5134	 * the requested ID, we're done. Otherwise let's
5135	 * continue the search with offset 0 in the new
5136	 * type.
5137	 */
5138	if (!btf_types_are_same(btf, id, need_btf, need_type_id)) {
5139		off = 0;
5140		goto again;
5141	}
5142
5143	return true;
5144}
5145
5146static int __get_type_size(struct btf *btf, u32 btf_id,
5147			   const struct btf_type **bad_type)
5148{
5149	const struct btf_type *t;
5150
 
5151	if (!btf_id)
5152		/* void */
5153		return 0;
5154	t = btf_type_by_id(btf, btf_id);
5155	while (t && btf_type_is_modifier(t))
5156		t = btf_type_by_id(btf, t->type);
5157	if (!t) {
5158		*bad_type = btf_type_by_id(btf, 0);
5159		return -EINVAL;
5160	}
5161	if (btf_type_is_ptr(t))
5162		/* kernel size of pointer. Not BPF's size of pointer*/
5163		return sizeof(void *);
5164	if (btf_type_is_int(t) || btf_type_is_enum(t))
5165		return t->size;
5166	*bad_type = t;
5167	return -EINVAL;
5168}
5169
 
 
 
 
 
 
 
 
 
 
 
 
5170int btf_distill_func_proto(struct bpf_verifier_log *log,
5171			   struct btf *btf,
5172			   const struct btf_type *func,
5173			   const char *tname,
5174			   struct btf_func_model *m)
5175{
5176	const struct btf_param *args;
5177	const struct btf_type *t;
5178	u32 i, nargs;
5179	int ret;
5180
5181	if (!func) {
5182		/* BTF function prototype doesn't match the verifier types.
5183		 * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args.
5184		 */
5185		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
5186			m->arg_size[i] = 8;
 
 
5187		m->ret_size = 8;
 
5188		m->nr_args = MAX_BPF_FUNC_REG_ARGS;
5189		return 0;
5190	}
5191	args = (const struct btf_param *)(func + 1);
5192	nargs = btf_type_vlen(func);
5193	if (nargs >= MAX_BPF_FUNC_ARGS) {
5194		bpf_log(log,
5195			"The function %s has %d arguments. Too many.\n",
5196			tname, nargs);
5197		return -EINVAL;
5198	}
5199	ret = __get_type_size(btf, func->type, &t);
5200	if (ret < 0) {
5201		bpf_log(log,
5202			"The function %s return type %s is unsupported.\n",
5203			tname, btf_kind_str[BTF_INFO_KIND(t->info)]);
5204		return -EINVAL;
5205	}
5206	m->ret_size = ret;
 
5207
5208	for (i = 0; i < nargs; i++) {
5209		if (i == nargs - 1 && args[i].type == 0) {
5210			bpf_log(log,
5211				"The function %s with variable args is unsupported.\n",
5212				tname);
5213			return -EINVAL;
5214		}
5215		ret = __get_type_size(btf, args[i].type, &t);
5216		if (ret < 0) {
 
 
5217			bpf_log(log,
5218				"The function %s arg%d type %s is unsupported.\n",
5219				tname, i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5220			return -EINVAL;
5221		}
5222		if (ret == 0) {
5223			bpf_log(log,
5224				"The function %s has malformed void argument.\n",
5225				tname);
5226			return -EINVAL;
5227		}
5228		m->arg_size[i] = ret;
 
5229	}
5230	m->nr_args = nargs;
5231	return 0;
5232}
5233
5234/* Compare BTFs of two functions assuming only scalars and pointers to context.
5235 * t1 points to BTF_KIND_FUNC in btf1
5236 * t2 points to BTF_KIND_FUNC in btf2
5237 * Returns:
5238 * EINVAL - function prototype mismatch
5239 * EFAULT - verifier bug
5240 * 0 - 99% match. The last 1% is validated by the verifier.
5241 */
5242static int btf_check_func_type_match(struct bpf_verifier_log *log,
5243				     struct btf *btf1, const struct btf_type *t1,
5244				     struct btf *btf2, const struct btf_type *t2)
5245{
5246	const struct btf_param *args1, *args2;
5247	const char *fn1, *fn2, *s1, *s2;
5248	u32 nargs1, nargs2, i;
5249
5250	fn1 = btf_name_by_offset(btf1, t1->name_off);
5251	fn2 = btf_name_by_offset(btf2, t2->name_off);
5252
5253	if (btf_func_linkage(t1) != BTF_FUNC_GLOBAL) {
5254		bpf_log(log, "%s() is not a global function\n", fn1);
5255		return -EINVAL;
5256	}
5257	if (btf_func_linkage(t2) != BTF_FUNC_GLOBAL) {
5258		bpf_log(log, "%s() is not a global function\n", fn2);
5259		return -EINVAL;
5260	}
5261
5262	t1 = btf_type_by_id(btf1, t1->type);
5263	if (!t1 || !btf_type_is_func_proto(t1))
5264		return -EFAULT;
5265	t2 = btf_type_by_id(btf2, t2->type);
5266	if (!t2 || !btf_type_is_func_proto(t2))
5267		return -EFAULT;
5268
5269	args1 = (const struct btf_param *)(t1 + 1);
5270	nargs1 = btf_type_vlen(t1);
5271	args2 = (const struct btf_param *)(t2 + 1);
5272	nargs2 = btf_type_vlen(t2);
5273
5274	if (nargs1 != nargs2) {
5275		bpf_log(log, "%s() has %d args while %s() has %d args\n",
5276			fn1, nargs1, fn2, nargs2);
5277		return -EINVAL;
5278	}
5279
5280	t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
5281	t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
5282	if (t1->info != t2->info) {
5283		bpf_log(log,
5284			"Return type %s of %s() doesn't match type %s of %s()\n",
5285			btf_type_str(t1), fn1,
5286			btf_type_str(t2), fn2);
5287		return -EINVAL;
5288	}
5289
5290	for (i = 0; i < nargs1; i++) {
5291		t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL);
5292		t2 = btf_type_skip_modifiers(btf2, args2[i].type, NULL);
5293
5294		if (t1->info != t2->info) {
5295			bpf_log(log, "arg%d in %s() is %s while %s() has %s\n",
5296				i, fn1, btf_type_str(t1),
5297				fn2, btf_type_str(t2));
5298			return -EINVAL;
5299		}
5300		if (btf_type_has_size(t1) && t1->size != t2->size) {
5301			bpf_log(log,
5302				"arg%d in %s() has size %d while %s() has %d\n",
5303				i, fn1, t1->size,
5304				fn2, t2->size);
5305			return -EINVAL;
5306		}
5307
5308		/* global functions are validated with scalars and pointers
5309		 * to context only. And only global functions can be replaced.
5310		 * Hence type check only those types.
5311		 */
5312		if (btf_type_is_int(t1) || btf_type_is_enum(t1))
5313			continue;
5314		if (!btf_type_is_ptr(t1)) {
5315			bpf_log(log,
5316				"arg%d in %s() has unrecognized type\n",
5317				i, fn1);
5318			return -EINVAL;
5319		}
5320		t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
5321		t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
5322		if (!btf_type_is_struct(t1)) {
5323			bpf_log(log,
5324				"arg%d in %s() is not a pointer to context\n",
5325				i, fn1);
5326			return -EINVAL;
5327		}
5328		if (!btf_type_is_struct(t2)) {
5329			bpf_log(log,
5330				"arg%d in %s() is not a pointer to context\n",
5331				i, fn2);
5332			return -EINVAL;
5333		}
5334		/* This is an optional check to make program writing easier.
5335		 * Compare names of structs and report an error to the user.
5336		 * btf_prepare_func_args() already checked that t2 struct
5337		 * is a context type. btf_prepare_func_args() will check
5338		 * later that t1 struct is a context type as well.
5339		 */
5340		s1 = btf_name_by_offset(btf1, t1->name_off);
5341		s2 = btf_name_by_offset(btf2, t2->name_off);
5342		if (strcmp(s1, s2)) {
5343			bpf_log(log,
5344				"arg%d %s(struct %s *) doesn't match %s(struct %s *)\n",
5345				i, fn1, s1, fn2, s2);
5346			return -EINVAL;
5347		}
5348	}
5349	return 0;
5350}
5351
5352/* Compare BTFs of given program with BTF of target program */
5353int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
5354			 struct btf *btf2, const struct btf_type *t2)
5355{
5356	struct btf *btf1 = prog->aux->btf;
5357	const struct btf_type *t1;
5358	u32 btf_id = 0;
5359
5360	if (!prog->aux->func_info) {
5361		bpf_log(log, "Program extension requires BTF\n");
5362		return -EINVAL;
5363	}
5364
5365	btf_id = prog->aux->func_info[0].type_id;
5366	if (!btf_id)
5367		return -EFAULT;
5368
5369	t1 = btf_type_by_id(btf1, btf_id);
5370	if (!t1 || !btf_type_is_func(t1))
5371		return -EFAULT;
5372
5373	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5374}
5375
5376static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5377#ifdef CONFIG_NET
5378	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5379	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5380	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5381#endif
5382};
5383
5384static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5385				    const struct btf *btf, u32 func_id,
5386				    struct bpf_reg_state *regs,
5387				    bool ptr_to_mem_ok)
5388{
5389	struct bpf_verifier_log *log = &env->log;
5390	const char *func_name, *ref_tname;
5391	const struct btf_type *t, *ref_t;
5392	const struct btf_param *args;
5393	u32 i, nargs, ref_id;
5394
5395	t = btf_type_by_id(btf, func_id);
5396	if (!t || !btf_type_is_func(t)) {
5397		/* These checks were already done by the verifier while loading
5398		 * struct bpf_func_info or in add_kfunc_call().
5399		 */
5400		bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5401			func_id);
5402		return -EFAULT;
5403	}
5404	func_name = btf_name_by_offset(btf, t->name_off);
5405
5406	t = btf_type_by_id(btf, t->type);
5407	if (!t || !btf_type_is_func_proto(t)) {
5408		bpf_log(log, "Invalid BTF of func %s\n", func_name);
5409		return -EFAULT;
5410	}
5411	args = (const struct btf_param *)(t + 1);
5412	nargs = btf_type_vlen(t);
5413	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5414		bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
5415			MAX_BPF_FUNC_REG_ARGS);
5416		return -EINVAL;
5417	}
5418
5419	/* check that BTF function arguments match actual types that the
5420	 * verifier sees.
5421	 */
5422	for (i = 0; i < nargs; i++) {
5423		u32 regno = i + 1;
5424		struct bpf_reg_state *reg = &regs[regno];
5425
5426		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
5427		if (btf_type_is_scalar(t)) {
5428			if (reg->type == SCALAR_VALUE)
5429				continue;
5430			bpf_log(log, "R%d is not a scalar\n", regno);
5431			return -EINVAL;
5432		}
5433
5434		if (!btf_type_is_ptr(t)) {
5435			bpf_log(log, "Unrecognized arg#%d type %s\n",
5436				i, btf_type_str(t));
5437			return -EINVAL;
5438		}
5439
5440		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5441		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5442		if (btf_is_kernel(btf)) {
5443			const struct btf_type *reg_ref_t;
5444			const struct btf *reg_btf;
5445			const char *reg_ref_tname;
5446			u32 reg_ref_id;
5447
5448			if (!btf_type_is_struct(ref_t)) {
5449				bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
5450					func_name, i, btf_type_str(ref_t),
5451					ref_tname);
5452				return -EINVAL;
5453			}
5454
5455			if (reg->type == PTR_TO_BTF_ID) {
5456				reg_btf = reg->btf;
5457				reg_ref_id = reg->btf_id;
5458			} else if (reg2btf_ids[reg->type]) {
5459				reg_btf = btf_vmlinux;
5460				reg_ref_id = *reg2btf_ids[reg->type];
5461			} else {
5462				bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5463					func_name, i,
5464					btf_type_str(ref_t), ref_tname, regno);
5465				return -EINVAL;
5466			}
5467
5468			reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
5469							    &reg_ref_id);
5470			reg_ref_tname = btf_name_by_offset(reg_btf,
5471							   reg_ref_t->name_off);
5472			if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
5473						  reg->off, btf, ref_id)) {
5474				bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
5475					func_name, i,
5476					btf_type_str(ref_t), ref_tname,
5477					regno, btf_type_str(reg_ref_t),
5478					reg_ref_tname);
5479				return -EINVAL;
5480			}
5481		} else if (btf_get_prog_ctx_type(log, btf, t,
5482						 env->prog->type, i)) {
5483			/* If function expects ctx type in BTF check that caller
5484			 * is passing PTR_TO_CTX.
5485			 */
5486			if (reg->type != PTR_TO_CTX) {
5487				bpf_log(log,
5488					"arg#%d expected pointer to ctx, but got %s\n",
5489					i, btf_type_str(t));
5490				return -EINVAL;
5491			}
5492			if (check_ctx_reg(env, reg, regno))
5493				return -EINVAL;
5494		} else if (ptr_to_mem_ok) {
5495			const struct btf_type *resolve_ret;
5496			u32 type_size;
5497
5498			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5499			if (IS_ERR(resolve_ret)) {
5500				bpf_log(log,
5501					"arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5502					i, btf_type_str(ref_t), ref_tname,
5503					PTR_ERR(resolve_ret));
5504				return -EINVAL;
5505			}
5506
5507			if (check_mem_reg(env, reg, regno, type_size))
5508				return -EINVAL;
5509		} else {
5510			return -EINVAL;
5511		}
5512	}
5513
5514	return 0;
5515}
5516
5517/* Compare BTF of a function with given bpf_reg_state.
5518 * Returns:
5519 * EFAULT - there is a verifier bug. Abort verification.
5520 * EINVAL - there is a type mismatch or BTF is not available.
5521 * 0 - BTF matches with what bpf_reg_state expects.
5522 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
5523 */
5524int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
5525				struct bpf_reg_state *regs)
5526{
5527	struct bpf_prog *prog = env->prog;
5528	struct btf *btf = prog->aux->btf;
5529	bool is_global;
5530	u32 btf_id;
5531	int err;
5532
5533	if (!prog->aux->func_info)
5534		return -EINVAL;
5535
5536	btf_id = prog->aux->func_info[subprog].type_id;
5537	if (!btf_id)
5538		return -EFAULT;
5539
5540	if (prog->aux->func_info_aux[subprog].unreliable)
5541		return -EINVAL;
 
 
 
 
 
 
 
 
5542
5543	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5544	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
 
 
 
 
 
5545
5546	/* Compiler optimizations can remove arguments from static functions
5547	 * or mismatched type can be passed into a global function.
5548	 * In such cases mark the function as unreliable from BTF point of view.
5549	 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5550	if (err)
5551		prog->aux->func_info_aux[subprog].unreliable = true;
5552	return err;
5553}
5554
5555int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
5556			      const struct btf *btf, u32 func_id,
5557			      struct bpf_reg_state *regs)
5558{
5559	return btf_check_func_arg_match(env, btf, func_id, regs, false);
5560}
5561
5562/* Convert BTF of a function into bpf_reg_state if possible
 
 
 
 
 
 
 
 
 
 
5563 * Returns:
5564 * EFAULT - there is a verifier bug. Abort verification.
5565 * EINVAL - cannot convert BTF.
5566 * 0 - Successfully converted BTF into bpf_reg_state
5567 * (either PTR_TO_CTX or SCALAR_VALUE).
5568 */
5569int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
5570			  struct bpf_reg_state *regs)
5571{
 
 
5572	struct bpf_verifier_log *log = &env->log;
5573	struct bpf_prog *prog = env->prog;
5574	enum bpf_prog_type prog_type = prog->type;
5575	struct btf *btf = prog->aux->btf;
5576	const struct btf_param *args;
5577	const struct btf_type *t, *ref_t;
5578	u32 i, nargs, btf_id;
5579	const char *tname;
5580
5581	if (!prog->aux->func_info ||
5582	    prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) {
 
 
5583		bpf_log(log, "Verifier bug\n");
5584		return -EFAULT;
5585	}
5586
5587	btf_id = prog->aux->func_info[subprog].type_id;
5588	if (!btf_id) {
 
 
5589		bpf_log(log, "Global functions need valid BTF\n");
5590		return -EFAULT;
5591	}
5592
5593	t = btf_type_by_id(btf, btf_id);
5594	if (!t || !btf_type_is_func(t)) {
5595		/* These checks were already done by the verifier while loading
5596		 * struct bpf_func_info
5597		 */
5598		bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",
5599			subprog);
5600		return -EFAULT;
5601	}
5602	tname = btf_name_by_offset(btf, t->name_off);
5603
5604	if (log->level & BPF_LOG_LEVEL)
5605		bpf_log(log, "Validating %s() func#%d...\n",
5606			tname, subprog);
5607
5608	if (prog->aux->func_info_aux[subprog].unreliable) {
5609		bpf_log(log, "Verifier bug in function %s()\n", tname);
5610		return -EFAULT;
5611	}
5612	if (prog_type == BPF_PROG_TYPE_EXT)
5613		prog_type = prog->aux->dst_prog->type;
5614
5615	t = btf_type_by_id(btf, t->type);
5616	if (!t || !btf_type_is_func_proto(t)) {
5617		bpf_log(log, "Invalid type of function %s()\n", tname);
5618		return -EFAULT;
5619	}
5620	args = (const struct btf_param *)(t + 1);
5621	nargs = btf_type_vlen(t);
5622	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
 
 
5623		bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n",
5624			tname, nargs, MAX_BPF_FUNC_REG_ARGS);
5625		return -EINVAL;
5626	}
5627	/* check that function returns int */
5628	t = btf_type_by_id(btf, t->type);
5629	while (btf_type_is_modifier(t))
5630		t = btf_type_by_id(btf, t->type);
5631	if (!btf_type_is_int(t) && !btf_type_is_enum(t)) {
 
 
5632		bpf_log(log,
5633			"Global function %s() doesn't return scalar. Only those are supported.\n",
5634			tname);
5635		return -EINVAL;
5636	}
5637	/* Convert BTF function arguments into verifier types.
5638	 * Only PTR_TO_CTX and SCALAR are supported atm.
5639	 */
5640	for (i = 0; i < nargs; i++) {
5641		struct bpf_reg_state *reg = &regs[i + 1];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5642
5643		t = btf_type_by_id(btf, args[i].type);
5644		while (btf_type_is_modifier(t))
5645			t = btf_type_by_id(btf, t->type);
5646		if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5647			reg->type = SCALAR_VALUE;
 
 
 
 
 
 
 
 
 
 
 
5648			continue;
5649		}
5650		if (btf_type_is_ptr(t)) {
5651			if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5652				reg->type = PTR_TO_CTX;
5653				continue;
5654			}
 
 
 
 
 
5655
5656			t = btf_type_skip_modifiers(btf, t->type, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5657
5658			ref_t = btf_resolve_size(btf, t, &reg->mem_size);
 
 
 
 
 
 
5659			if (IS_ERR(ref_t)) {
5660				bpf_log(log,
5661				    "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5662				    i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5663					PTR_ERR(ref_t));
5664				return -EINVAL;
5665			}
5666
5667			reg->type = PTR_TO_MEM_OR_NULL;
5668			reg->id = ++env->id_gen;
 
 
 
 
5669
 
 
 
 
 
 
 
5670			continue;
5671		}
 
 
5672		bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
5673			i, btf_kind_str[BTF_INFO_KIND(t->info)], tname);
5674		return -EINVAL;
5675	}
 
 
 
 
5676	return 0;
5677}
5678
5679static void btf_type_show(const struct btf *btf, u32 type_id, void *obj,
5680			  struct btf_show *show)
5681{
5682	const struct btf_type *t = btf_type_by_id(btf, type_id);
5683
5684	show->btf = btf;
5685	memset(&show->state, 0, sizeof(show->state));
5686	memset(&show->obj, 0, sizeof(show->obj));
5687
5688	btf_type_ops(t)->show(btf, t, type_id, obj, 0, show);
5689}
5690
5691static void btf_seq_show(struct btf_show *show, const char *fmt,
5692			 va_list args)
5693{
5694	seq_vprintf((struct seq_file *)show->target, fmt, args);
5695}
5696
5697int btf_type_seq_show_flags(const struct btf *btf, u32 type_id,
5698			    void *obj, struct seq_file *m, u64 flags)
5699{
5700	struct btf_show sseq;
5701
5702	sseq.target = m;
5703	sseq.showfn = btf_seq_show;
5704	sseq.flags = flags;
5705
5706	btf_type_show(btf, type_id, obj, &sseq);
5707
5708	return sseq.state.status;
5709}
5710
5711void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
5712		       struct seq_file *m)
5713{
5714	(void) btf_type_seq_show_flags(btf, type_id, obj, m,
5715				       BTF_SHOW_NONAME | BTF_SHOW_COMPACT |
5716				       BTF_SHOW_ZERO | BTF_SHOW_UNSAFE);
5717}
5718
5719struct btf_show_snprintf {
5720	struct btf_show show;
5721	int len_left;		/* space left in string */
5722	int len;		/* length we would have written */
5723};
5724
5725static void btf_snprintf_show(struct btf_show *show, const char *fmt,
5726			      va_list args)
5727{
5728	struct btf_show_snprintf *ssnprintf = (struct btf_show_snprintf *)show;
5729	int len;
5730
5731	len = vsnprintf(show->target, ssnprintf->len_left, fmt, args);
5732
5733	if (len < 0) {
5734		ssnprintf->len_left = 0;
5735		ssnprintf->len = len;
5736	} else if (len > ssnprintf->len_left) {
5737		/* no space, drive on to get length we would have written */
5738		ssnprintf->len_left = 0;
5739		ssnprintf->len += len;
5740	} else {
5741		ssnprintf->len_left -= len;
5742		ssnprintf->len += len;
5743		show->target += len;
5744	}
5745}
5746
5747int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj,
5748			   char *buf, int len, u64 flags)
5749{
5750	struct btf_show_snprintf ssnprintf;
5751
5752	ssnprintf.show.target = buf;
5753	ssnprintf.show.flags = flags;
5754	ssnprintf.show.showfn = btf_snprintf_show;
5755	ssnprintf.len_left = len;
5756	ssnprintf.len = 0;
5757
5758	btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf);
5759
5760	/* If we encontered an error, return it. */
5761	if (ssnprintf.show.state.status)
5762		return ssnprintf.show.state.status;
5763
5764	/* Otherwise return length we would have written */
5765	return ssnprintf.len;
5766}
5767
5768#ifdef CONFIG_PROC_FS
5769static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp)
5770{
5771	const struct btf *btf = filp->private_data;
5772
5773	seq_printf(m, "btf_id:\t%u\n", btf->id);
5774}
5775#endif
5776
5777static int btf_release(struct inode *inode, struct file *filp)
5778{
5779	btf_put(filp->private_data);
5780	return 0;
5781}
5782
5783const struct file_operations btf_fops = {
5784#ifdef CONFIG_PROC_FS
5785	.show_fdinfo	= bpf_btf_show_fdinfo,
5786#endif
5787	.release	= btf_release,
5788};
5789
5790static int __btf_new_fd(struct btf *btf)
5791{
5792	return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
5793}
5794
5795int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr)
5796{
5797	struct btf *btf;
5798	int ret;
5799
5800	btf = btf_parse(make_bpfptr(attr->btf, uattr.is_kernel),
5801			attr->btf_size, attr->btf_log_level,
5802			u64_to_user_ptr(attr->btf_log_buf),
5803			attr->btf_log_size);
5804	if (IS_ERR(btf))
5805		return PTR_ERR(btf);
5806
5807	ret = btf_alloc_id(btf);
5808	if (ret) {
5809		btf_free(btf);
5810		return ret;
5811	}
5812
5813	/*
5814	 * The BTF ID is published to the userspace.
5815	 * All BTF free must go through call_rcu() from
5816	 * now on (i.e. free by calling btf_put()).
5817	 */
5818
5819	ret = __btf_new_fd(btf);
5820	if (ret < 0)
5821		btf_put(btf);
5822
5823	return ret;
5824}
5825
5826struct btf *btf_get_by_fd(int fd)
5827{
5828	struct btf *btf;
5829	struct fd f;
5830
5831	f = fdget(fd);
5832
5833	if (!f.file)
5834		return ERR_PTR(-EBADF);
5835
5836	if (f.file->f_op != &btf_fops) {
5837		fdput(f);
5838		return ERR_PTR(-EINVAL);
5839	}
5840
5841	btf = f.file->private_data;
5842	refcount_inc(&btf->refcnt);
5843	fdput(f);
5844
5845	return btf;
5846}
5847
5848int btf_get_info_by_fd(const struct btf *btf,
5849		       const union bpf_attr *attr,
5850		       union bpf_attr __user *uattr)
5851{
5852	struct bpf_btf_info __user *uinfo;
5853	struct bpf_btf_info info;
5854	u32 info_copy, btf_copy;
5855	void __user *ubtf;
5856	char __user *uname;
5857	u32 uinfo_len, uname_len, name_len;
5858	int ret = 0;
5859
5860	uinfo = u64_to_user_ptr(attr->info.info);
5861	uinfo_len = attr->info.info_len;
5862
5863	info_copy = min_t(u32, uinfo_len, sizeof(info));
5864	memset(&info, 0, sizeof(info));
5865	if (copy_from_user(&info, uinfo, info_copy))
5866		return -EFAULT;
5867
5868	info.id = btf->id;
5869	ubtf = u64_to_user_ptr(info.btf);
5870	btf_copy = min_t(u32, btf->data_size, info.btf_size);
5871	if (copy_to_user(ubtf, btf->data, btf_copy))
5872		return -EFAULT;
5873	info.btf_size = btf->data_size;
5874
5875	info.kernel_btf = btf->kernel_btf;
5876
5877	uname = u64_to_user_ptr(info.name);
5878	uname_len = info.name_len;
5879	if (!uname ^ !uname_len)
5880		return -EINVAL;
5881
5882	name_len = strlen(btf->name);
5883	info.name_len = name_len;
5884
5885	if (uname) {
5886		if (uname_len >= name_len + 1) {
5887			if (copy_to_user(uname, btf->name, name_len + 1))
5888				return -EFAULT;
5889		} else {
5890			char zero = '\0';
5891
5892			if (copy_to_user(uname, btf->name, uname_len - 1))
5893				return -EFAULT;
5894			if (put_user(zero, uname + uname_len - 1))
5895				return -EFAULT;
5896			/* let user-space know about too short buffer */
5897			ret = -ENOSPC;
5898		}
5899	}
5900
5901	if (copy_to_user(uinfo, &info, info_copy) ||
5902	    put_user(info_copy, &uattr->info.info_len))
5903		return -EFAULT;
5904
5905	return ret;
5906}
5907
5908int btf_get_fd_by_id(u32 id)
5909{
5910	struct btf *btf;
5911	int fd;
5912
5913	rcu_read_lock();
5914	btf = idr_find(&btf_idr, id);
5915	if (!btf || !refcount_inc_not_zero(&btf->refcnt))
5916		btf = ERR_PTR(-ENOENT);
5917	rcu_read_unlock();
5918
5919	if (IS_ERR(btf))
5920		return PTR_ERR(btf);
5921
5922	fd = __btf_new_fd(btf);
5923	if (fd < 0)
5924		btf_put(btf);
5925
5926	return fd;
5927}
5928
5929u32 btf_obj_id(const struct btf *btf)
5930{
5931	return btf->id;
5932}
5933
5934bool btf_is_kernel(const struct btf *btf)
5935{
5936	return btf->kernel_btf;
5937}
5938
5939bool btf_is_module(const struct btf *btf)
5940{
5941	return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0;
5942}
5943
5944static int btf_id_cmp_func(const void *a, const void *b)
5945{
5946	const int *pa = a, *pb = b;
5947
5948	return *pa - *pb;
5949}
5950
5951bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
5952{
5953	return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
5954}
5955
5956#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
5957struct btf_module {
5958	struct list_head list;
5959	struct module *module;
5960	struct btf *btf;
5961	struct bin_attribute *sysfs_attr;
 
5962};
5963
5964static LIST_HEAD(btf_modules);
5965static DEFINE_MUTEX(btf_module_mutex);
5966
5967static ssize_t
5968btf_module_read(struct file *file, struct kobject *kobj,
5969		struct bin_attribute *bin_attr,
5970		char *buf, loff_t off, size_t len)
5971{
5972	const struct btf *btf = bin_attr->private;
5973
5974	memcpy(buf, btf->data + off, len);
5975	return len;
5976}
5977
 
 
5978static int btf_module_notify(struct notifier_block *nb, unsigned long op,
5979			     void *module)
5980{
5981	struct btf_module *btf_mod, *tmp;
5982	struct module *mod = module;
5983	struct btf *btf;
5984	int err = 0;
5985
5986	if (mod->btf_data_size == 0 ||
5987	    (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
 
5988		goto out;
5989
5990	switch (op) {
5991	case MODULE_STATE_COMING:
5992		btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
5993		if (!btf_mod) {
5994			err = -ENOMEM;
5995			goto out;
5996		}
5997		btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size);
5998		if (IS_ERR(btf)) {
5999			pr_warn("failed to validate module [%s] BTF: %ld\n",
6000				mod->name, PTR_ERR(btf));
6001			kfree(btf_mod);
6002			err = PTR_ERR(btf);
 
 
 
 
 
 
6003			goto out;
6004		}
6005		err = btf_alloc_id(btf);
6006		if (err) {
6007			btf_free(btf);
6008			kfree(btf_mod);
6009			goto out;
6010		}
6011
 
6012		mutex_lock(&btf_module_mutex);
6013		btf_mod->module = module;
6014		btf_mod->btf = btf;
6015		list_add(&btf_mod->list, &btf_modules);
6016		mutex_unlock(&btf_module_mutex);
6017
6018		if (IS_ENABLED(CONFIG_SYSFS)) {
6019			struct bin_attribute *attr;
6020
6021			attr = kzalloc(sizeof(*attr), GFP_KERNEL);
6022			if (!attr)
6023				goto out;
6024
6025			sysfs_bin_attr_init(attr);
6026			attr->attr.name = btf->name;
6027			attr->attr.mode = 0444;
6028			attr->size = btf->data_size;
6029			attr->private = btf;
6030			attr->read = btf_module_read;
6031
6032			err = sysfs_create_bin_file(btf_kobj, attr);
6033			if (err) {
6034				pr_warn("failed to register module [%s] BTF in sysfs: %d\n",
6035					mod->name, err);
6036				kfree(attr);
6037				err = 0;
6038				goto out;
6039			}
6040
6041			btf_mod->sysfs_attr = attr;
6042		}
6043
6044		break;
 
 
 
 
 
 
 
 
 
 
 
6045	case MODULE_STATE_GOING:
6046		mutex_lock(&btf_module_mutex);
6047		list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
6048			if (btf_mod->module != module)
6049				continue;
6050
6051			list_del(&btf_mod->list);
6052			if (btf_mod->sysfs_attr)
6053				sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
 
6054			btf_put(btf_mod->btf);
6055			kfree(btf_mod->sysfs_attr);
6056			kfree(btf_mod);
6057			break;
6058		}
6059		mutex_unlock(&btf_module_mutex);
6060		break;
6061	}
6062out:
6063	return notifier_from_errno(err);
6064}
6065
6066static struct notifier_block btf_module_nb = {
6067	.notifier_call = btf_module_notify,
6068};
6069
6070static int __init btf_module_init(void)
6071{
6072	register_module_notifier(&btf_module_nb);
6073	return 0;
6074}
6075
6076fs_initcall(btf_module_init);
6077#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */
6078
6079struct module *btf_try_get_module(const struct btf *btf)
6080{
6081	struct module *res = NULL;
6082#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6083	struct btf_module *btf_mod, *tmp;
6084
6085	mutex_lock(&btf_module_mutex);
6086	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
6087		if (btf_mod->btf != btf)
6088			continue;
6089
6090		if (try_module_get(btf_mod->module))
 
 
 
 
 
6091			res = btf_mod->module;
6092
6093		break;
6094	}
6095	mutex_unlock(&btf_module_mutex);
6096#endif
6097
6098	return res;
6099}
6100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6101BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags)
6102{
6103	struct btf *btf;
 
6104	long ret;
6105
6106	if (flags)
6107		return -EINVAL;
6108
6109	if (name_sz <= 1 || name[name_sz - 1])
6110		return -EINVAL;
6111
6112	btf = bpf_get_btf_vmlinux();
6113	if (IS_ERR(btf))
6114		return PTR_ERR(btf);
6115
6116	ret = btf_find_by_name_kind(btf, name, kind);
6117	/* ret is never zero, since btf_find_by_name_kind returns
6118	 * positive btf_id or negative error.
6119	 */
6120	if (ret < 0) {
6121		struct btf *mod_btf;
6122		int id;
6123
6124		/* If name is not found in vmlinux's BTF then search in module's BTFs */
6125		spin_lock_bh(&btf_idr_lock);
6126		idr_for_each_entry(&btf_idr, mod_btf, id) {
6127			if (!btf_is_module(mod_btf))
6128				continue;
6129			/* linear search could be slow hence unlock/lock
6130			 * the IDR to avoiding holding it for too long
6131			 */
6132			btf_get(mod_btf);
6133			spin_unlock_bh(&btf_idr_lock);
6134			ret = btf_find_by_name_kind(mod_btf, name, kind);
6135			if (ret > 0) {
6136				int btf_obj_fd;
6137
6138				btf_obj_fd = __btf_new_fd(mod_btf);
6139				if (btf_obj_fd < 0) {
6140					btf_put(mod_btf);
6141					return btf_obj_fd;
6142				}
6143				return ret | (((u64)btf_obj_fd) << 32);
6144			}
6145			spin_lock_bh(&btf_idr_lock);
6146			btf_put(mod_btf);
6147		}
6148		spin_unlock_bh(&btf_idr_lock);
6149	}
 
 
6150	return ret;
6151}
6152
6153const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
6154	.func		= bpf_btf_find_by_name_kind,
6155	.gpl_only	= false,
6156	.ret_type	= RET_INTEGER,
6157	.arg1_type	= ARG_PTR_TO_MEM,
6158	.arg2_type	= ARG_CONST_SIZE,
6159	.arg3_type	= ARG_ANYTHING,
6160	.arg4_type	= ARG_ANYTHING,
6161};