1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4 */
   5
   6#include <string.h>
   7#include <stdlib.h>
   8#include <inttypes.h>
   9#include <sys/mman.h>
  10
  11#include <arch/elf.h>
  12#include <objtool/builtin.h>
  13#include <objtool/cfi.h>
  14#include <objtool/arch.h>
  15#include <objtool/check.h>
  16#include <objtool/special.h>
  17#include <objtool/warn.h>
  18#include <objtool/endianness.h>
  19
  20#include <linux/objtool.h>
  21#include <linux/hashtable.h>
  22#include <linux/kernel.h>
  23#include <linux/static_call_types.h>
  24
  25struct alternative {
  26	struct list_head list;
  27	struct instruction *insn;
  28	bool skip_orig;
  29};
  30
  31static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33static struct cfi_init_state initial_func_cfi;
  34static struct cfi_state init_cfi;
  35static struct cfi_state func_cfi;
  36
  37struct instruction *find_insn(struct objtool_file *file,
  38			      struct section *sec, unsigned long offset)
  39{
  40	struct instruction *insn;
  41
  42	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43		if (insn->sec == sec && insn->offset == offset)
  44			return insn;
  45	}
  46
  47	return NULL;
  48}
  49
  50static struct instruction *next_insn_same_sec(struct objtool_file *file,
  51					      struct instruction *insn)
  52{
  53	struct instruction *next = list_next_entry(insn, list);
  54
  55	if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  56		return NULL;
  57
  58	return next;
  59}
  60
  61static struct instruction *next_insn_same_func(struct objtool_file *file,
  62					       struct instruction *insn)
  63{
  64	struct instruction *next = list_next_entry(insn, list);
  65	struct symbol *func = insn_func(insn);
  66
  67	if (!func)
  68		return NULL;
  69
  70	if (&next->list != &file->insn_list && insn_func(next) == func)
  71		return next;
  72
  73	/* Check if we're already in the subfunction: */
  74	if (func == func->cfunc)
  75		return NULL;
  76
  77	/* Move to the subfunction: */
  78	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  79}
  80
  81static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  82					       struct instruction *insn)
  83{
  84	struct instruction *prev = list_prev_entry(insn, list);
  85
  86	if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
  87		return prev;
  88
  89	return NULL;
  90}
  91
  92#define func_for_each_insn(file, func, insn)				\
  93	for (insn = find_insn(file, func->sec, func->offset);		\
  94	     insn;							\
  95	     insn = next_insn_same_func(file, insn))
  96
  97#define sym_for_each_insn(file, sym, insn)				\
  98	for (insn = find_insn(file, sym->sec, sym->offset);		\
  99	     insn && &insn->list != &file->insn_list &&			\
 100		insn->sec == sym->sec &&				\
 101		insn->offset < sym->offset + sym->len;			\
 102	     insn = list_next_entry(insn, list))
 103
 104#define sym_for_each_insn_continue_reverse(file, sym, insn)		\
 105	for (insn = list_prev_entry(insn, list);			\
 106	     &insn->list != &file->insn_list &&				\
 107		insn->sec == sym->sec && insn->offset >= sym->offset;	\
 108	     insn = list_prev_entry(insn, list))
 109
 110#define sec_for_each_insn_from(file, insn)				\
 111	for (; insn; insn = next_insn_same_sec(file, insn))
 112
 113#define sec_for_each_insn_continue(file, insn)				\
 114	for (insn = next_insn_same_sec(file, insn); insn;		\
 115	     insn = next_insn_same_sec(file, insn))
 116
 117static bool is_jump_table_jump(struct instruction *insn)
 118{
 119	struct alt_group *alt_group = insn->alt_group;
 120
 121	if (insn->jump_table)
 122		return true;
 123
 124	/* Retpoline alternative for a jump table? */
 125	return alt_group && alt_group->orig_group &&
 126	       alt_group->orig_group->first_insn->jump_table;
 127}
 128
 129static bool is_sibling_call(struct instruction *insn)
 130{
 131	/*
 132	 * Assume only STT_FUNC calls have jump-tables.
 133	 */
 134	if (insn_func(insn)) {
 135		/* An indirect jump is either a sibling call or a jump to a table. */
 136		if (insn->type == INSN_JUMP_DYNAMIC)
 137			return !is_jump_table_jump(insn);
 138	}
 139
 140	/* add_jump_destinations() sets insn->call_dest for sibling calls. */
 141	return (is_static_jump(insn) && insn->call_dest);
 142}
 143
 144/*
 145 * This checks to see if the given function is a "noreturn" function.
 146 *
 147 * For global functions which are outside the scope of this object file, we
 148 * have to keep a manual list of them.
 149 *
 150 * For local functions, we have to detect them manually by simply looking for
 151 * the lack of a return instruction.
 152 */
 153static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 154				int recursion)
 155{
 156	int i;
 157	struct instruction *insn;
 158	bool empty = true;
 159
 160	/*
 161	 * Unfortunately these have to be hard coded because the noreturn
 162	 * attribute isn't provided in ELF data. Keep 'em sorted.
 163	 */
 164	static const char * const global_noreturns[] = {
 165		"__invalid_creds",
 166		"__module_put_and_kthread_exit",
 167		"__reiserfs_panic",
 168		"__stack_chk_fail",
 169		"__ubsan_handle_builtin_unreachable",
 170		"cpu_bringup_and_idle",
 171		"cpu_startup_entry",
 172		"do_exit",
 173		"do_group_exit",
 174		"do_task_dead",
 175		"ex_handler_msr_mce",
 176		"fortify_panic",
 177		"kthread_complete_and_exit",
 178		"kthread_exit",
 179		"kunit_try_catch_throw",
 180		"lbug_with_loc",
 181		"machine_real_restart",
 182		"make_task_dead",
 183		"panic",
 184		"rewind_stack_and_make_dead",
 185		"sev_es_terminate",
 186		"snp_abort",
 187		"stop_this_cpu",
 188		"usercopy_abort",
 189		"xen_start_kernel",
 190	};
 191
 192	if (!func)
 193		return false;
 194
 195	if (func->bind == STB_WEAK)
 196		return false;
 197
 198	if (func->bind == STB_GLOBAL)
 199		for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 200			if (!strcmp(func->name, global_noreturns[i]))
 201				return true;
 202
 203	if (!func->len)
 204		return false;
 205
 206	insn = find_insn(file, func->sec, func->offset);
 207	if (!insn || !insn_func(insn))
 208		return false;
 209
 210	func_for_each_insn(file, func, insn) {
 211		empty = false;
 212
 213		if (insn->type == INSN_RETURN)
 214			return false;
 215	}
 216
 217	if (empty)
 218		return false;
 219
 220	/*
 221	 * A function can have a sibling call instead of a return.  In that
 222	 * case, the function's dead-end status depends on whether the target
 223	 * of the sibling call returns.
 224	 */
 225	func_for_each_insn(file, func, insn) {
 226		if (is_sibling_call(insn)) {
 227			struct instruction *dest = insn->jump_dest;
 228
 229			if (!dest)
 230				/* sibling call to another file */
 231				return false;
 232
 233			/* local sibling call */
 234			if (recursion == 5) {
 235				/*
 236				 * Infinite recursion: two functions have
 237				 * sibling calls to each other.  This is a very
 238				 * rare case.  It means they aren't dead ends.
 239				 */
 240				return false;
 241			}
 242
 243			return __dead_end_function(file, insn_func(dest), recursion+1);
 244		}
 245	}
 246
 247	return true;
 248}
 249
 250static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 251{
 252	return __dead_end_function(file, func, 0);
 253}
 254
 255static void init_cfi_state(struct cfi_state *cfi)
 256{
 257	int i;
 258
 259	for (i = 0; i < CFI_NUM_REGS; i++) {
 260		cfi->regs[i].base = CFI_UNDEFINED;
 261		cfi->vals[i].base = CFI_UNDEFINED;
 262	}
 263	cfi->cfa.base = CFI_UNDEFINED;
 264	cfi->drap_reg = CFI_UNDEFINED;
 265	cfi->drap_offset = -1;
 266}
 267
 268static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 269			    struct section *sec)
 270{
 271	memset(state, 0, sizeof(*state));
 272	init_cfi_state(&state->cfi);
 273
 274	/*
 275	 * We need the full vmlinux for noinstr validation, otherwise we can
 276	 * not correctly determine insn->call_dest->sec (external symbols do
 277	 * not have a section).
 278	 */
 279	if (opts.link && opts.noinstr && sec)
 280		state->noinstr = sec->noinstr;
 281}
 282
 283static struct cfi_state *cfi_alloc(void)
 284{
 285	struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
 286	if (!cfi) {
 287		WARN("calloc failed");
 288		exit(1);
 289	}
 290	nr_cfi++;
 291	return cfi;
 292}
 293
 294static int cfi_bits;
 295static struct hlist_head *cfi_hash;
 296
 297static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 298{
 299	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 300		      (void *)cfi2 + sizeof(cfi2->hash),
 301		      sizeof(struct cfi_state) - sizeof(struct hlist_node));
 302}
 303
 304static inline u32 cfi_key(struct cfi_state *cfi)
 305{
 306	return jhash((void *)cfi + sizeof(cfi->hash),
 307		     sizeof(*cfi) - sizeof(cfi->hash), 0);
 308}
 309
 310static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 311{
 312	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 313	struct cfi_state *obj;
 314
 315	hlist_for_each_entry(obj, head, hash) {
 316		if (!cficmp(cfi, obj)) {
 317			nr_cfi_cache++;
 318			return obj;
 319		}
 320	}
 321
 322	obj = cfi_alloc();
 323	*obj = *cfi;
 324	hlist_add_head(&obj->hash, head);
 325
 326	return obj;
 327}
 328
 329static void cfi_hash_add(struct cfi_state *cfi)
 330{
 331	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 332
 333	hlist_add_head(&cfi->hash, head);
 334}
 335
 336static void *cfi_hash_alloc(unsigned long size)
 337{
 338	cfi_bits = max(10, ilog2(size));
 339	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 340			PROT_READ|PROT_WRITE,
 341			MAP_PRIVATE|MAP_ANON, -1, 0);
 342	if (cfi_hash == (void *)-1L) {
 343		WARN("mmap fail cfi_hash");
 344		cfi_hash = NULL;
 345	}  else if (opts.stats) {
 346		printf("cfi_bits: %d\n", cfi_bits);
 347	}
 348
 349	return cfi_hash;
 350}
 351
 352static unsigned long nr_insns;
 353static unsigned long nr_insns_visited;
 354
 355/*
 356 * Call the arch-specific instruction decoder for all the instructions and add
 357 * them to the global instruction list.
 358 */
 359static int decode_instructions(struct objtool_file *file)
 360{
 361	struct section *sec;
 362	struct symbol *func;
 363	unsigned long offset;
 364	struct instruction *insn;
 365	int ret;
 366
 367	for_each_sec(file, sec) {
 368
 369		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 370			continue;
 371
 372		if (strcmp(sec->name, ".altinstr_replacement") &&
 373		    strcmp(sec->name, ".altinstr_aux") &&
 374		    strncmp(sec->name, ".discard.", 9))
 375			sec->text = true;
 376
 377		if (!strcmp(sec->name, ".noinstr.text") ||
 378		    !strcmp(sec->name, ".entry.text") ||
 379		    !strncmp(sec->name, ".text.__x86.", 12))
 380			sec->noinstr = true;
 381
 382		/*
 383		 * .init.text code is ran before userspace and thus doesn't
 384		 * strictly need retpolines, except for modules which are
 385		 * loaded late, they very much do need retpoline in their
 386		 * .init.text
 387		 */
 388		if (!strcmp(sec->name, ".init.text") && !opts.module)
 389			sec->init = true;
 390
 391		for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 392			insn = malloc(sizeof(*insn));
 393			if (!insn) {
 394				WARN("malloc failed");
 395				return -1;
 396			}
 397			memset(insn, 0, sizeof(*insn));
 398			INIT_LIST_HEAD(&insn->alts);
 399			INIT_LIST_HEAD(&insn->stack_ops);
 400			INIT_LIST_HEAD(&insn->call_node);
 401
 402			insn->sec = sec;
 403			insn->offset = offset;
 404
 405			ret = arch_decode_instruction(file, sec, offset,
 406						      sec->sh.sh_size - offset,
 407						      &insn->len, &insn->type,
 408						      &insn->immediate,
 409						      &insn->stack_ops);
 410			if (ret)
 411				goto err;
 412
 413			/*
 414			 * By default, "ud2" is a dead end unless otherwise
 415			 * annotated, because GCC 7 inserts it for certain
 416			 * divide-by-zero cases.
 417			 */
 418			if (insn->type == INSN_BUG)
 419				insn->dead_end = true;
 420
 421			hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 422			list_add_tail(&insn->list, &file->insn_list);
 423			nr_insns++;
 424		}
 425
 426		list_for_each_entry(func, &sec->symbol_list, list) {
 427			if (func->type != STT_NOTYPE && func->type != STT_FUNC)
 428				continue;
 429
 430			if (func->offset == sec->sh.sh_size) {
 431				/* Heuristic: likely an "end" symbol */
 432				if (func->type == STT_NOTYPE)
 433					continue;
 434				WARN("%s(): STT_FUNC at end of section",
 435				     func->name);
 436				return -1;
 437			}
 438
 439			if (func->return_thunk || func->alias != func)
 440				continue;
 441
 442			if (!find_insn(file, sec, func->offset)) {
 443				WARN("%s(): can't find starting instruction",
 444				     func->name);
 445				return -1;
 446			}
 447
 448			sym_for_each_insn(file, func, insn) {
 449				insn->sym = func;
 450				if (func->type == STT_FUNC &&
 451				    insn->type == INSN_ENDBR &&
 452				    list_empty(&insn->call_node)) {
 453					if (insn->offset == func->offset) {
 454						list_add_tail(&insn->call_node, &file->endbr_list);
 455						file->nr_endbr++;
 456					} else {
 457						file->nr_endbr_int++;
 458					}
 459				}
 460			}
 461		}
 462	}
 463
 464	if (opts.stats)
 465		printf("nr_insns: %lu\n", nr_insns);
 466
 467	return 0;
 468
 469err:
 470	free(insn);
 471	return ret;
 472}
 473
 474/*
 475 * Read the pv_ops[] .data table to find the static initialized values.
 476 */
 477static int add_pv_ops(struct objtool_file *file, const char *symname)
 478{
 479	struct symbol *sym, *func;
 480	unsigned long off, end;
 481	struct reloc *rel;
 482	int idx;
 483
 484	sym = find_symbol_by_name(file->elf, symname);
 485	if (!sym)
 486		return 0;
 487
 488	off = sym->offset;
 489	end = off + sym->len;
 490	for (;;) {
 491		rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 492		if (!rel)
 493			break;
 494
 495		func = rel->sym;
 496		if (func->type == STT_SECTION)
 497			func = find_symbol_by_offset(rel->sym->sec, rel->addend);
 498
 499		idx = (rel->offset - sym->offset) / sizeof(unsigned long);
 500
 501		objtool_pv_add(file, idx, func);
 502
 503		off = rel->offset + 1;
 504		if (off > end)
 505			break;
 506	}
 507
 508	return 0;
 509}
 510
 511/*
 512 * Allocate and initialize file->pv_ops[].
 513 */
 514static int init_pv_ops(struct objtool_file *file)
 515{
 516	static const char *pv_ops_tables[] = {
 517		"pv_ops",
 518		"xen_cpu_ops",
 519		"xen_irq_ops",
 520		"xen_mmu_ops",
 521		NULL,
 522	};
 523	const char *pv_ops;
 524	struct symbol *sym;
 525	int idx, nr;
 526
 527	if (!opts.noinstr)
 528		return 0;
 529
 530	file->pv_ops = NULL;
 531
 532	sym = find_symbol_by_name(file->elf, "pv_ops");
 533	if (!sym)
 534		return 0;
 535
 536	nr = sym->len / sizeof(unsigned long);
 537	file->pv_ops = calloc(sizeof(struct pv_state), nr);
 538	if (!file->pv_ops)
 539		return -1;
 540
 541	for (idx = 0; idx < nr; idx++)
 542		INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 543
 544	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 545		add_pv_ops(file, pv_ops);
 546
 547	return 0;
 548}
 549
 550static struct instruction *find_last_insn(struct objtool_file *file,
 551					  struct section *sec)
 552{
 553	struct instruction *insn = NULL;
 554	unsigned int offset;
 555	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 556
 557	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 558		insn = find_insn(file, sec, offset);
 559
 560	return insn;
 561}
 562
 563/*
 564 * Mark "ud2" instructions and manually annotated dead ends.
 565 */
 566static int add_dead_ends(struct objtool_file *file)
 567{
 568	struct section *sec;
 569	struct reloc *reloc;
 570	struct instruction *insn;
 571
 572	/*
 573	 * Check for manually annotated dead ends.
 574	 */
 575	sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 576	if (!sec)
 577		goto reachable;
 578
 579	list_for_each_entry(reloc, &sec->reloc_list, list) {
 580		if (reloc->sym->type != STT_SECTION) {
 581			WARN("unexpected relocation symbol type in %s", sec->name);
 582			return -1;
 583		}
 584		insn = find_insn(file, reloc->sym->sec, reloc->addend);
 585		if (insn)
 586			insn = list_prev_entry(insn, list);
 587		else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 588			insn = find_last_insn(file, reloc->sym->sec);
 589			if (!insn) {
 590				WARN("can't find unreachable insn at %s+0x%" PRIx64,
 591				     reloc->sym->sec->name, reloc->addend);
 592				return -1;
 593			}
 594		} else {
 595			WARN("can't find unreachable insn at %s+0x%" PRIx64,
 596			     reloc->sym->sec->name, reloc->addend);
 597			return -1;
 598		}
 599
 600		insn->dead_end = true;
 601	}
 602
 603reachable:
 604	/*
 605	 * These manually annotated reachable checks are needed for GCC 4.4,
 606	 * where the Linux unreachable() macro isn't supported.  In that case
 607	 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 608	 * not a dead end.
 609	 */
 610	sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 611	if (!sec)
 612		return 0;
 613
 614	list_for_each_entry(reloc, &sec->reloc_list, list) {
 615		if (reloc->sym->type != STT_SECTION) {
 616			WARN("unexpected relocation symbol type in %s", sec->name);
 617			return -1;
 618		}
 619		insn = find_insn(file, reloc->sym->sec, reloc->addend);
 620		if (insn)
 621			insn = list_prev_entry(insn, list);
 622		else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 623			insn = find_last_insn(file, reloc->sym->sec);
 624			if (!insn) {
 625				WARN("can't find reachable insn at %s+0x%" PRIx64,
 626				     reloc->sym->sec->name, reloc->addend);
 627				return -1;
 628			}
 629		} else {
 630			WARN("can't find reachable insn at %s+0x%" PRIx64,
 631			     reloc->sym->sec->name, reloc->addend);
 632			return -1;
 633		}
 634
 635		insn->dead_end = false;
 636	}
 637
 638	return 0;
 639}
 640
 641static int create_static_call_sections(struct objtool_file *file)
 642{
 643	struct section *sec;
 644	struct static_call_site *site;
 645	struct instruction *insn;
 646	struct symbol *key_sym;
 647	char *key_name, *tmp;
 648	int idx;
 649
 650	sec = find_section_by_name(file->elf, ".static_call_sites");
 651	if (sec) {
 652		INIT_LIST_HEAD(&file->static_call_list);
 653		WARN("file already has .static_call_sites section, skipping");
 654		return 0;
 655	}
 656
 657	if (list_empty(&file->static_call_list))
 658		return 0;
 659
 660	idx = 0;
 661	list_for_each_entry(insn, &file->static_call_list, call_node)
 662		idx++;
 663
 664	sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 665				 sizeof(struct static_call_site), idx);
 666	if (!sec)
 667		return -1;
 668
 669	idx = 0;
 670	list_for_each_entry(insn, &file->static_call_list, call_node) {
 671
 672		site = (struct static_call_site *)sec->data->d_buf + idx;
 673		memset(site, 0, sizeof(struct static_call_site));
 674
 675		/* populate reloc for 'addr' */
 676		if (elf_add_reloc_to_insn(file->elf, sec,
 677					  idx * sizeof(struct static_call_site),
 678					  R_X86_64_PC32,
 679					  insn->sec, insn->offset))
 680			return -1;
 681
 682		/* find key symbol */
 683		key_name = strdup(insn->call_dest->name);
 684		if (!key_name) {
 685			perror("strdup");
 686			return -1;
 687		}
 688		if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 689			    STATIC_CALL_TRAMP_PREFIX_LEN)) {
 690			WARN("static_call: trampoline name malformed: %s", key_name);
 691			return -1;
 692		}
 693		tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 694		memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 695
 696		key_sym = find_symbol_by_name(file->elf, tmp);
 697		if (!key_sym) {
 698			if (!opts.module) {
 699				WARN("static_call: can't find static_call_key symbol: %s", tmp);
 700				return -1;
 701			}
 702
 703			/*
 704			 * For modules(), the key might not be exported, which
 705			 * means the module can make static calls but isn't
 706			 * allowed to change them.
 707			 *
 708			 * In that case we temporarily set the key to be the
 709			 * trampoline address.  This is fixed up in
 710			 * static_call_add_module().
 711			 */
 712			key_sym = insn->call_dest;
 713		}
 714		free(key_name);
 715
 716		/* populate reloc for 'key' */
 717		if (elf_add_reloc(file->elf, sec,
 718				  idx * sizeof(struct static_call_site) + 4,
 719				  R_X86_64_PC32, key_sym,
 720				  is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 721			return -1;
 722
 723		idx++;
 724	}
 725
 726	return 0;
 727}
 728
 729static int create_retpoline_sites_sections(struct objtool_file *file)
 730{
 731	struct instruction *insn;
 732	struct section *sec;
 733	int idx;
 734
 735	sec = find_section_by_name(file->elf, ".retpoline_sites");
 736	if (sec) {
 737		WARN("file already has .retpoline_sites, skipping");
 738		return 0;
 739	}
 740
 741	idx = 0;
 742	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 743		idx++;
 744
 745	if (!idx)
 746		return 0;
 747
 748	sec = elf_create_section(file->elf, ".retpoline_sites", 0,
 749				 sizeof(int), idx);
 750	if (!sec) {
 751		WARN("elf_create_section: .retpoline_sites");
 752		return -1;
 753	}
 754
 755	idx = 0;
 756	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 757
 758		int *site = (int *)sec->data->d_buf + idx;
 759		*site = 0;
 760
 761		if (elf_add_reloc_to_insn(file->elf, sec,
 762					  idx * sizeof(int),
 763					  R_X86_64_PC32,
 764					  insn->sec, insn->offset)) {
 765			WARN("elf_add_reloc_to_insn: .retpoline_sites");
 766			return -1;
 767		}
 768
 769		idx++;
 770	}
 771
 772	return 0;
 773}
 774
 775static int create_return_sites_sections(struct objtool_file *file)
 776{
 777	struct instruction *insn;
 778	struct section *sec;
 779	int idx;
 780
 781	sec = find_section_by_name(file->elf, ".return_sites");
 782	if (sec) {
 783		WARN("file already has .return_sites, skipping");
 784		return 0;
 785	}
 786
 787	idx = 0;
 788	list_for_each_entry(insn, &file->return_thunk_list, call_node)
 789		idx++;
 790
 791	if (!idx)
 792		return 0;
 793
 794	sec = elf_create_section(file->elf, ".return_sites", 0,
 795				 sizeof(int), idx);
 796	if (!sec) {
 797		WARN("elf_create_section: .return_sites");
 798		return -1;
 799	}
 800
 801	idx = 0;
 802	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 803
 804		int *site = (int *)sec->data->d_buf + idx;
 805		*site = 0;
 806
 807		if (elf_add_reloc_to_insn(file->elf, sec,
 808					  idx * sizeof(int),
 809					  R_X86_64_PC32,
 810					  insn->sec, insn->offset)) {
 811			WARN("elf_add_reloc_to_insn: .return_sites");
 812			return -1;
 813		}
 814
 815		idx++;
 816	}
 817
 818	return 0;
 819}
 820
 821static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 822{
 823	struct instruction *insn;
 824	struct section *sec;
 825	int idx;
 826
 827	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 828	if (sec) {
 829		WARN("file already has .ibt_endbr_seal, skipping");
 830		return 0;
 831	}
 832
 833	idx = 0;
 834	list_for_each_entry(insn, &file->endbr_list, call_node)
 835		idx++;
 836
 837	if (opts.stats) {
 838		printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 839		printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 840		printf("ibt: superfluous ENDBR:       %d\n", idx);
 841	}
 842
 843	if (!idx)
 844		return 0;
 845
 846	sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
 847				 sizeof(int), idx);
 848	if (!sec) {
 849		WARN("elf_create_section: .ibt_endbr_seal");
 850		return -1;
 851	}
 852
 853	idx = 0;
 854	list_for_each_entry(insn, &file->endbr_list, call_node) {
 855
 856		int *site = (int *)sec->data->d_buf + idx;
 857		*site = 0;
 858
 859		if (elf_add_reloc_to_insn(file->elf, sec,
 860					  idx * sizeof(int),
 861					  R_X86_64_PC32,
 862					  insn->sec, insn->offset)) {
 863			WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
 864			return -1;
 865		}
 866
 867		idx++;
 868	}
 869
 870	return 0;
 871}
 872
 873static int create_cfi_sections(struct objtool_file *file)
 874{
 875	struct section *sec, *s;
 876	struct symbol *sym;
 877	unsigned int *loc;
 878	int idx;
 879
 880	sec = find_section_by_name(file->elf, ".cfi_sites");
 881	if (sec) {
 882		INIT_LIST_HEAD(&file->call_list);
 883		WARN("file already has .cfi_sites section, skipping");
 884		return 0;
 885	}
 886
 887	idx = 0;
 888	for_each_sec(file, s) {
 889		if (!s->text)
 890			continue;
 891
 892		list_for_each_entry(sym, &s->symbol_list, list) {
 893			if (sym->type != STT_FUNC)
 894				continue;
 895
 896			if (strncmp(sym->name, "__cfi_", 6))
 897				continue;
 898
 899			idx++;
 900		}
 901	}
 902
 903	sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
 904	if (!sec)
 905		return -1;
 906
 907	idx = 0;
 908	for_each_sec(file, s) {
 909		if (!s->text)
 910			continue;
 911
 912		list_for_each_entry(sym, &s->symbol_list, list) {
 913			if (sym->type != STT_FUNC)
 914				continue;
 915
 916			if (strncmp(sym->name, "__cfi_", 6))
 917				continue;
 918
 919			loc = (unsigned int *)sec->data->d_buf + idx;
 920			memset(loc, 0, sizeof(unsigned int));
 921
 922			if (elf_add_reloc_to_insn(file->elf, sec,
 923						  idx * sizeof(unsigned int),
 924						  R_X86_64_PC32,
 925						  s, sym->offset))
 926				return -1;
 927
 928			idx++;
 929		}
 930	}
 931
 932	return 0;
 933}
 934
 935static int create_mcount_loc_sections(struct objtool_file *file)
 936{
 937	int addrsize = elf_class_addrsize(file->elf);
 938	struct instruction *insn;
 939	struct section *sec;
 940	int idx;
 941
 942	sec = find_section_by_name(file->elf, "__mcount_loc");
 943	if (sec) {
 944		INIT_LIST_HEAD(&file->mcount_loc_list);
 945		WARN("file already has __mcount_loc section, skipping");
 946		return 0;
 947	}
 948
 949	if (list_empty(&file->mcount_loc_list))
 950		return 0;
 951
 952	idx = 0;
 953	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 954		idx++;
 955
 956	sec = elf_create_section(file->elf, "__mcount_loc", 0, addrsize, idx);
 957	if (!sec)
 958		return -1;
 959
 960	sec->sh.sh_addralign = addrsize;
 961
 962	idx = 0;
 963	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 964		void *loc;
 965
 966		loc = sec->data->d_buf + idx;
 967		memset(loc, 0, addrsize);
 968
 969		if (elf_add_reloc_to_insn(file->elf, sec, idx,
 970					  addrsize == sizeof(u64) ? R_ABS64 : R_ABS32,
 971					  insn->sec, insn->offset))
 972			return -1;
 973
 974		idx += addrsize;
 975	}
 976
 977	return 0;
 978}
 979
 980static int create_direct_call_sections(struct objtool_file *file)
 981{
 982	struct instruction *insn;
 983	struct section *sec;
 984	unsigned int *loc;
 985	int idx;
 986
 987	sec = find_section_by_name(file->elf, ".call_sites");
 988	if (sec) {
 989		INIT_LIST_HEAD(&file->call_list);
 990		WARN("file already has .call_sites section, skipping");
 991		return 0;
 992	}
 993
 994	if (list_empty(&file->call_list))
 995		return 0;
 996
 997	idx = 0;
 998	list_for_each_entry(insn, &file->call_list, call_node)
 999		idx++;
1000
1001	sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
1002	if (!sec)
1003		return -1;
1004
1005	idx = 0;
1006	list_for_each_entry(insn, &file->call_list, call_node) {
1007
1008		loc = (unsigned int *)sec->data->d_buf + idx;
1009		memset(loc, 0, sizeof(unsigned int));
1010
1011		if (elf_add_reloc_to_insn(file->elf, sec,
1012					  idx * sizeof(unsigned int),
1013					  R_X86_64_PC32,
1014					  insn->sec, insn->offset))
1015			return -1;
1016
1017		idx++;
1018	}
1019
1020	return 0;
1021}
1022
1023/*
1024 * Warnings shouldn't be reported for ignored functions.
1025 */
1026static void add_ignores(struct objtool_file *file)
1027{
1028	struct instruction *insn;
1029	struct section *sec;
1030	struct symbol *func;
1031	struct reloc *reloc;
1032
1033	sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1034	if (!sec)
1035		return;
1036
1037	list_for_each_entry(reloc, &sec->reloc_list, list) {
1038		switch (reloc->sym->type) {
1039		case STT_FUNC:
1040			func = reloc->sym;
1041			break;
1042
1043		case STT_SECTION:
1044			func = find_func_by_offset(reloc->sym->sec, reloc->addend);
1045			if (!func)
1046				continue;
1047			break;
1048
1049		default:
1050			WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
1051			continue;
1052		}
1053
1054		func_for_each_insn(file, func, insn)
1055			insn->ignore = true;
1056	}
1057}
1058
1059/*
1060 * This is a whitelist of functions that is allowed to be called with AC set.
1061 * The list is meant to be minimal and only contains compiler instrumentation
1062 * ABI and a few functions used to implement *_{to,from}_user() functions.
1063 *
1064 * These functions must not directly change AC, but may PUSHF/POPF.
1065 */
1066static const char *uaccess_safe_builtin[] = {
1067	/* KASAN */
1068	"kasan_report",
1069	"kasan_check_range",
1070	/* KASAN out-of-line */
1071	"__asan_loadN_noabort",
1072	"__asan_load1_noabort",
1073	"__asan_load2_noabort",
1074	"__asan_load4_noabort",
1075	"__asan_load8_noabort",
1076	"__asan_load16_noabort",
1077	"__asan_storeN_noabort",
1078	"__asan_store1_noabort",
1079	"__asan_store2_noabort",
1080	"__asan_store4_noabort",
1081	"__asan_store8_noabort",
1082	"__asan_store16_noabort",
1083	"__kasan_check_read",
1084	"__kasan_check_write",
1085	/* KASAN in-line */
1086	"__asan_report_load_n_noabort",
1087	"__asan_report_load1_noabort",
1088	"__asan_report_load2_noabort",
1089	"__asan_report_load4_noabort",
1090	"__asan_report_load8_noabort",
1091	"__asan_report_load16_noabort",
1092	"__asan_report_store_n_noabort",
1093	"__asan_report_store1_noabort",
1094	"__asan_report_store2_noabort",
1095	"__asan_report_store4_noabort",
1096	"__asan_report_store8_noabort",
1097	"__asan_report_store16_noabort",
1098	/* KCSAN */
1099	"__kcsan_check_access",
1100	"__kcsan_mb",
1101	"__kcsan_wmb",
1102	"__kcsan_rmb",
1103	"__kcsan_release",
1104	"kcsan_found_watchpoint",
1105	"kcsan_setup_watchpoint",
1106	"kcsan_check_scoped_accesses",
1107	"kcsan_disable_current",
1108	"kcsan_enable_current_nowarn",
1109	/* KCSAN/TSAN */
1110	"__tsan_func_entry",
1111	"__tsan_func_exit",
1112	"__tsan_read_range",
1113	"__tsan_write_range",
1114	"__tsan_read1",
1115	"__tsan_read2",
1116	"__tsan_read4",
1117	"__tsan_read8",
1118	"__tsan_read16",
1119	"__tsan_write1",
1120	"__tsan_write2",
1121	"__tsan_write4",
1122	"__tsan_write8",
1123	"__tsan_write16",
1124	"__tsan_read_write1",
1125	"__tsan_read_write2",
1126	"__tsan_read_write4",
1127	"__tsan_read_write8",
1128	"__tsan_read_write16",
1129	"__tsan_volatile_read1",
1130	"__tsan_volatile_read2",
1131	"__tsan_volatile_read4",
1132	"__tsan_volatile_read8",
1133	"__tsan_volatile_read16",
1134	"__tsan_volatile_write1",
1135	"__tsan_volatile_write2",
1136	"__tsan_volatile_write4",
1137	"__tsan_volatile_write8",
1138	"__tsan_volatile_write16",
1139	"__tsan_atomic8_load",
1140	"__tsan_atomic16_load",
1141	"__tsan_atomic32_load",
1142	"__tsan_atomic64_load",
1143	"__tsan_atomic8_store",
1144	"__tsan_atomic16_store",
1145	"__tsan_atomic32_store",
1146	"__tsan_atomic64_store",
1147	"__tsan_atomic8_exchange",
1148	"__tsan_atomic16_exchange",
1149	"__tsan_atomic32_exchange",
1150	"__tsan_atomic64_exchange",
1151	"__tsan_atomic8_fetch_add",
1152	"__tsan_atomic16_fetch_add",
1153	"__tsan_atomic32_fetch_add",
1154	"__tsan_atomic64_fetch_add",
1155	"__tsan_atomic8_fetch_sub",
1156	"__tsan_atomic16_fetch_sub",
1157	"__tsan_atomic32_fetch_sub",
1158	"__tsan_atomic64_fetch_sub",
1159	"__tsan_atomic8_fetch_and",
1160	"__tsan_atomic16_fetch_and",
1161	"__tsan_atomic32_fetch_and",
1162	"__tsan_atomic64_fetch_and",
1163	"__tsan_atomic8_fetch_or",
1164	"__tsan_atomic16_fetch_or",
1165	"__tsan_atomic32_fetch_or",
1166	"__tsan_atomic64_fetch_or",
1167	"__tsan_atomic8_fetch_xor",
1168	"__tsan_atomic16_fetch_xor",
1169	"__tsan_atomic32_fetch_xor",
1170	"__tsan_atomic64_fetch_xor",
1171	"__tsan_atomic8_fetch_nand",
1172	"__tsan_atomic16_fetch_nand",
1173	"__tsan_atomic32_fetch_nand",
1174	"__tsan_atomic64_fetch_nand",
1175	"__tsan_atomic8_compare_exchange_strong",
1176	"__tsan_atomic16_compare_exchange_strong",
1177	"__tsan_atomic32_compare_exchange_strong",
1178	"__tsan_atomic64_compare_exchange_strong",
1179	"__tsan_atomic8_compare_exchange_weak",
1180	"__tsan_atomic16_compare_exchange_weak",
1181	"__tsan_atomic32_compare_exchange_weak",
1182	"__tsan_atomic64_compare_exchange_weak",
1183	"__tsan_atomic8_compare_exchange_val",
1184	"__tsan_atomic16_compare_exchange_val",
1185	"__tsan_atomic32_compare_exchange_val",
1186	"__tsan_atomic64_compare_exchange_val",
1187	"__tsan_atomic_thread_fence",
1188	"__tsan_atomic_signal_fence",
1189	/* KCOV */
1190	"write_comp_data",
1191	"check_kcov_mode",
1192	"__sanitizer_cov_trace_pc",
1193	"__sanitizer_cov_trace_const_cmp1",
1194	"__sanitizer_cov_trace_const_cmp2",
1195	"__sanitizer_cov_trace_const_cmp4",
1196	"__sanitizer_cov_trace_const_cmp8",
1197	"__sanitizer_cov_trace_cmp1",
1198	"__sanitizer_cov_trace_cmp2",
1199	"__sanitizer_cov_trace_cmp4",
1200	"__sanitizer_cov_trace_cmp8",
1201	"__sanitizer_cov_trace_switch",
1202	/* KMSAN */
1203	"kmsan_copy_to_user",
1204	"kmsan_report",
1205	"kmsan_unpoison_entry_regs",
1206	"kmsan_unpoison_memory",
1207	"__msan_chain_origin",
1208	"__msan_get_context_state",
1209	"__msan_instrument_asm_store",
1210	"__msan_metadata_ptr_for_load_1",
1211	"__msan_metadata_ptr_for_load_2",
1212	"__msan_metadata_ptr_for_load_4",
1213	"__msan_metadata_ptr_for_load_8",
1214	"__msan_metadata_ptr_for_load_n",
1215	"__msan_metadata_ptr_for_store_1",
1216	"__msan_metadata_ptr_for_store_2",
1217	"__msan_metadata_ptr_for_store_4",
1218	"__msan_metadata_ptr_for_store_8",
1219	"__msan_metadata_ptr_for_store_n",
1220	"__msan_poison_alloca",
1221	"__msan_warning",
1222	/* UBSAN */
1223	"ubsan_type_mismatch_common",
1224	"__ubsan_handle_type_mismatch",
1225	"__ubsan_handle_type_mismatch_v1",
1226	"__ubsan_handle_shift_out_of_bounds",
1227	/* misc */
1228	"csum_partial_copy_generic",
1229	"copy_mc_fragile",
1230	"copy_mc_fragile_handle_tail",
1231	"copy_mc_enhanced_fast_string",
1232	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1233	"clear_user_erms",
1234	"clear_user_rep_good",
1235	"clear_user_original",
1236	NULL
1237};
1238
1239static void add_uaccess_safe(struct objtool_file *file)
1240{
1241	struct symbol *func;
1242	const char **name;
1243
1244	if (!opts.uaccess)
1245		return;
1246
1247	for (name = uaccess_safe_builtin; *name; name++) {
1248		func = find_symbol_by_name(file->elf, *name);
1249		if (!func)
1250			continue;
1251
1252		func->uaccess_safe = true;
1253	}
1254}
1255
1256/*
1257 * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1258 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1259 * But it at least allows objtool to understand the control flow *around* the
1260 * retpoline.
1261 */
1262static int add_ignore_alternatives(struct objtool_file *file)
1263{
1264	struct section *sec;
1265	struct reloc *reloc;
1266	struct instruction *insn;
1267
1268	sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1269	if (!sec)
1270		return 0;
1271
1272	list_for_each_entry(reloc, &sec->reloc_list, list) {
1273		if (reloc->sym->type != STT_SECTION) {
1274			WARN("unexpected relocation symbol type in %s", sec->name);
1275			return -1;
1276		}
1277
1278		insn = find_insn(file, reloc->sym->sec, reloc->addend);
1279		if (!insn) {
1280			WARN("bad .discard.ignore_alts entry");
1281			return -1;
1282		}
1283
1284		insn->ignore_alts = true;
1285	}
1286
1287	return 0;
1288}
1289
1290__weak bool arch_is_retpoline(struct symbol *sym)
1291{
1292	return false;
1293}
1294
1295__weak bool arch_is_rethunk(struct symbol *sym)
1296{
1297	return false;
1298}
1299
1300#define NEGATIVE_RELOC	((void *)-1L)
1301
1302static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1303{
1304	if (insn->reloc == NEGATIVE_RELOC)
1305		return NULL;
1306
1307	if (!insn->reloc) {
1308		if (!file)
1309			return NULL;
1310
1311		insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1312						       insn->offset, insn->len);
1313		if (!insn->reloc) {
1314			insn->reloc = NEGATIVE_RELOC;
1315			return NULL;
1316		}
1317	}
1318
1319	return insn->reloc;
1320}
1321
1322static void remove_insn_ops(struct instruction *insn)
1323{
1324	struct stack_op *op, *tmp;
1325
1326	list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1327		list_del(&op->list);
1328		free(op);
1329	}
1330}
1331
1332static void annotate_call_site(struct objtool_file *file,
1333			       struct instruction *insn, bool sibling)
1334{
1335	struct reloc *reloc = insn_reloc(file, insn);
1336	struct symbol *sym = insn->call_dest;
1337
1338	if (!sym)
1339		sym = reloc->sym;
1340
1341	/*
1342	 * Alternative replacement code is just template code which is
1343	 * sometimes copied to the original instruction. For now, don't
1344	 * annotate it. (In the future we might consider annotating the
1345	 * original instruction if/when it ever makes sense to do so.)
1346	 */
1347	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1348		return;
1349
1350	if (sym->static_call_tramp) {
1351		list_add_tail(&insn->call_node, &file->static_call_list);
1352		return;
1353	}
1354
1355	if (sym->retpoline_thunk) {
1356		list_add_tail(&insn->call_node, &file->retpoline_call_list);
1357		return;
1358	}
1359
1360	/*
1361	 * Many compilers cannot disable KCOV or sanitizer calls with a function
1362	 * attribute so they need a little help, NOP out any such calls from
1363	 * noinstr text.
1364	 */
1365	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1366		if (reloc) {
1367			reloc->type = R_NONE;
1368			elf_write_reloc(file->elf, reloc);
1369		}
1370
1371		elf_write_insn(file->elf, insn->sec,
1372			       insn->offset, insn->len,
1373			       sibling ? arch_ret_insn(insn->len)
1374			               : arch_nop_insn(insn->len));
1375
1376		insn->type = sibling ? INSN_RETURN : INSN_NOP;
1377
1378		if (sibling) {
1379			/*
1380			 * We've replaced the tail-call JMP insn by two new
1381			 * insn: RET; INT3, except we only have a single struct
1382			 * insn here. Mark it retpoline_safe to avoid the SLS
1383			 * warning, instead of adding another insn.
1384			 */
1385			insn->retpoline_safe = true;
1386		}
1387
1388		return;
1389	}
1390
1391	if (opts.mcount && sym->fentry) {
1392		if (sibling)
1393			WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1394		if (opts.mnop) {
1395			if (reloc) {
1396				reloc->type = R_NONE;
1397				elf_write_reloc(file->elf, reloc);
1398			}
1399
1400			elf_write_insn(file->elf, insn->sec,
1401				       insn->offset, insn->len,
1402				       arch_nop_insn(insn->len));
1403
1404			insn->type = INSN_NOP;
1405		}
1406
1407		list_add_tail(&insn->call_node, &file->mcount_loc_list);
1408		return;
1409	}
1410
1411	if (insn->type == INSN_CALL && !insn->sec->init)
1412		list_add_tail(&insn->call_node, &file->call_list);
1413
1414	if (!sibling && dead_end_function(file, sym))
1415		insn->dead_end = true;
1416}
1417
1418static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1419			  struct symbol *dest, bool sibling)
1420{
1421	insn->call_dest = dest;
1422	if (!dest)
1423		return;
1424
1425	/*
1426	 * Whatever stack impact regular CALLs have, should be undone
1427	 * by the RETURN of the called function.
1428	 *
1429	 * Annotated intra-function calls retain the stack_ops but
1430	 * are converted to JUMP, see read_intra_function_calls().
1431	 */
1432	remove_insn_ops(insn);
1433
1434	annotate_call_site(file, insn, sibling);
1435}
1436
1437static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1438{
1439	/*
1440	 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1441	 * so convert them accordingly.
1442	 */
1443	switch (insn->type) {
1444	case INSN_CALL:
1445		insn->type = INSN_CALL_DYNAMIC;
1446		break;
1447	case INSN_JUMP_UNCONDITIONAL:
1448		insn->type = INSN_JUMP_DYNAMIC;
1449		break;
1450	case INSN_JUMP_CONDITIONAL:
1451		insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1452		break;
1453	default:
1454		return;
1455	}
1456
1457	insn->retpoline_safe = true;
1458
1459	/*
1460	 * Whatever stack impact regular CALLs have, should be undone
1461	 * by the RETURN of the called function.
1462	 *
1463	 * Annotated intra-function calls retain the stack_ops but
1464	 * are converted to JUMP, see read_intra_function_calls().
1465	 */
1466	remove_insn_ops(insn);
1467
1468	annotate_call_site(file, insn, false);
1469}
1470
1471static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1472{
1473	/*
1474	 * Return thunk tail calls are really just returns in disguise,
1475	 * so convert them accordingly.
1476	 */
1477	insn->type = INSN_RETURN;
1478	insn->retpoline_safe = true;
1479
1480	if (add)
1481		list_add_tail(&insn->call_node, &file->return_thunk_list);
1482}
1483
1484static bool is_first_func_insn(struct objtool_file *file,
1485			       struct instruction *insn, struct symbol *sym)
1486{
1487	if (insn->offset == sym->offset)
1488		return true;
1489
1490	/* Allow direct CALL/JMP past ENDBR */
1491	if (opts.ibt) {
1492		struct instruction *prev = prev_insn_same_sym(file, insn);
1493
1494		if (prev && prev->type == INSN_ENDBR &&
1495		    insn->offset == sym->offset + prev->len)
1496			return true;
1497	}
1498
1499	return false;
1500}
1501
1502/*
1503 * A sibling call is a tail-call to another symbol -- to differentiate from a
1504 * recursive tail-call which is to the same symbol.
1505 */
1506static bool jump_is_sibling_call(struct objtool_file *file,
1507				 struct instruction *from, struct instruction *to)
1508{
1509	struct symbol *fs = from->sym;
1510	struct symbol *ts = to->sym;
1511
1512	/* Not a sibling call if from/to a symbol hole */
1513	if (!fs || !ts)
1514		return false;
1515
1516	/* Not a sibling call if not targeting the start of a symbol. */
1517	if (!is_first_func_insn(file, to, ts))
1518		return false;
1519
1520	/* Disallow sibling calls into STT_NOTYPE */
1521	if (ts->type == STT_NOTYPE)
1522		return false;
1523
1524	/* Must not be self to be a sibling */
1525	return fs->pfunc != ts->pfunc;
1526}
1527
1528/*
1529 * Find the destination instructions for all jumps.
1530 */
1531static int add_jump_destinations(struct objtool_file *file)
1532{
1533	struct instruction *insn, *jump_dest;
1534	struct reloc *reloc;
1535	struct section *dest_sec;
1536	unsigned long dest_off;
1537
1538	for_each_insn(file, insn) {
1539		if (insn->jump_dest) {
1540			/*
1541			 * handle_group_alt() may have previously set
1542			 * 'jump_dest' for some alternatives.
1543			 */
1544			continue;
1545		}
1546		if (!is_static_jump(insn))
1547			continue;
1548
1549		reloc = insn_reloc(file, insn);
1550		if (!reloc) {
1551			dest_sec = insn->sec;
1552			dest_off = arch_jump_destination(insn);
1553		} else if (reloc->sym->type == STT_SECTION) {
1554			dest_sec = reloc->sym->sec;
1555			dest_off = arch_dest_reloc_offset(reloc->addend);
1556		} else if (reloc->sym->retpoline_thunk) {
1557			add_retpoline_call(file, insn);
1558			continue;
1559		} else if (reloc->sym->return_thunk) {
1560			add_return_call(file, insn, true);
1561			continue;
1562		} else if (insn_func(insn)) {
1563			/*
1564			 * External sibling call or internal sibling call with
1565			 * STT_FUNC reloc.
1566			 */
1567			add_call_dest(file, insn, reloc->sym, true);
1568			continue;
1569		} else if (reloc->sym->sec->idx) {
1570			dest_sec = reloc->sym->sec;
1571			dest_off = reloc->sym->sym.st_value +
1572				   arch_dest_reloc_offset(reloc->addend);
1573		} else {
1574			/* non-func asm code jumping to another file */
1575			continue;
1576		}
1577
1578		jump_dest = find_insn(file, dest_sec, dest_off);
1579		if (!jump_dest) {
1580			struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1581
1582			/*
1583			 * This is a special case for zen_untrain_ret().
1584			 * It jumps to __x86_return_thunk(), but objtool
1585			 * can't find the thunk's starting RET
1586			 * instruction, because the RET is also in the
1587			 * middle of another instruction.  Objtool only
1588			 * knows about the outer instruction.
1589			 */
1590			if (sym && sym->return_thunk) {
1591				add_return_call(file, insn, false);
1592				continue;
1593			}
1594
1595			WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1596				  insn->sec, insn->offset, dest_sec->name,
1597				  dest_off);
1598			return -1;
1599		}
1600
1601		/*
1602		 * Cross-function jump.
1603		 */
1604		if (insn_func(insn) && insn_func(jump_dest) &&
1605		    insn_func(insn) != insn_func(jump_dest)) {
1606
1607			/*
1608			 * For GCC 8+, create parent/child links for any cold
1609			 * subfunctions.  This is _mostly_ redundant with a
1610			 * similar initialization in read_symbols().
1611			 *
1612			 * If a function has aliases, we want the *first* such
1613			 * function in the symbol table to be the subfunction's
1614			 * parent.  In that case we overwrite the
1615			 * initialization done in read_symbols().
1616			 *
1617			 * However this code can't completely replace the
1618			 * read_symbols() code because this doesn't detect the
1619			 * case where the parent function's only reference to a
1620			 * subfunction is through a jump table.
1621			 */
1622			if (!strstr(insn_func(insn)->name, ".cold") &&
1623			    strstr(insn_func(jump_dest)->name, ".cold")) {
1624				insn_func(insn)->cfunc = insn_func(jump_dest);
1625				insn_func(jump_dest)->pfunc = insn_func(insn);
1626			}
1627		}
1628
1629		if (jump_is_sibling_call(file, insn, jump_dest)) {
1630			/*
1631			 * Internal sibling call without reloc or with
1632			 * STT_SECTION reloc.
1633			 */
1634			add_call_dest(file, insn, insn_func(jump_dest), true);
1635			continue;
1636		}
1637
1638		insn->jump_dest = jump_dest;
1639	}
1640
1641	return 0;
1642}
1643
1644static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1645{
1646	struct symbol *call_dest;
1647
1648	call_dest = find_func_by_offset(sec, offset);
1649	if (!call_dest)
1650		call_dest = find_symbol_by_offset(sec, offset);
1651
1652	return call_dest;
1653}
1654
1655/*
1656 * Find the destination instructions for all calls.
1657 */
1658static int add_call_destinations(struct objtool_file *file)
1659{
1660	struct instruction *insn;
1661	unsigned long dest_off;
1662	struct symbol *dest;
1663	struct reloc *reloc;
1664
1665	for_each_insn(file, insn) {
1666		if (insn->type != INSN_CALL)
1667			continue;
1668
1669		reloc = insn_reloc(file, insn);
1670		if (!reloc) {
1671			dest_off = arch_jump_destination(insn);
1672			dest = find_call_destination(insn->sec, dest_off);
1673
1674			add_call_dest(file, insn, dest, false);
1675
1676			if (insn->ignore)
1677				continue;
1678
1679			if (!insn->call_dest) {
1680				WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1681				return -1;
1682			}
1683
1684			if (insn_func(insn) && insn->call_dest->type != STT_FUNC) {
1685				WARN_FUNC("unsupported call to non-function",
1686					  insn->sec, insn->offset);
1687				return -1;
1688			}
1689
1690		} else if (reloc->sym->type == STT_SECTION) {
1691			dest_off = arch_dest_reloc_offset(reloc->addend);
1692			dest = find_call_destination(reloc->sym->sec, dest_off);
1693			if (!dest) {
1694				WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1695					  insn->sec, insn->offset,
1696					  reloc->sym->sec->name,
1697					  dest_off);
1698				return -1;
1699			}
1700
1701			add_call_dest(file, insn, dest, false);
1702
1703		} else if (reloc->sym->retpoline_thunk) {
1704			add_retpoline_call(file, insn);
1705
1706		} else
1707			add_call_dest(file, insn, reloc->sym, false);
1708	}
1709
1710	return 0;
1711}
1712
1713/*
1714 * The .alternatives section requires some extra special care over and above
1715 * other special sections because alternatives are patched in place.
1716 */
1717static int handle_group_alt(struct objtool_file *file,
1718			    struct special_alt *special_alt,
1719			    struct instruction *orig_insn,
1720			    struct instruction **new_insn)
1721{
1722	struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1723	struct alt_group *orig_alt_group, *new_alt_group;
1724	unsigned long dest_off;
1725
1726
1727	orig_alt_group = malloc(sizeof(*orig_alt_group));
1728	if (!orig_alt_group) {
1729		WARN("malloc failed");
1730		return -1;
1731	}
1732	orig_alt_group->cfi = calloc(special_alt->orig_len,
1733				     sizeof(struct cfi_state *));
1734	if (!orig_alt_group->cfi) {
1735		WARN("calloc failed");
1736		return -1;
1737	}
1738
1739	last_orig_insn = NULL;
1740	insn = orig_insn;
1741	sec_for_each_insn_from(file, insn) {
1742		if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1743			break;
1744
1745		insn->alt_group = orig_alt_group;
1746		last_orig_insn = insn;
1747	}
1748	orig_alt_group->orig_group = NULL;
1749	orig_alt_group->first_insn = orig_insn;
1750	orig_alt_group->last_insn = last_orig_insn;
1751
1752
1753	new_alt_group = malloc(sizeof(*new_alt_group));
1754	if (!new_alt_group) {
1755		WARN("malloc failed");
1756		return -1;
1757	}
1758
1759	if (special_alt->new_len < special_alt->orig_len) {
1760		/*
1761		 * Insert a fake nop at the end to make the replacement
1762		 * alt_group the same size as the original.  This is needed to
1763		 * allow propagate_alt_cfi() to do its magic.  When the last
1764		 * instruction affects the stack, the instruction after it (the
1765		 * nop) will propagate the new state to the shared CFI array.
1766		 */
1767		nop = malloc(sizeof(*nop));
1768		if (!nop) {
1769			WARN("malloc failed");
1770			return -1;
1771		}
1772		memset(nop, 0, sizeof(*nop));
1773		INIT_LIST_HEAD(&nop->alts);
1774		INIT_LIST_HEAD(&nop->stack_ops);
1775
1776		nop->sec = special_alt->new_sec;
1777		nop->offset = special_alt->new_off + special_alt->new_len;
1778		nop->len = special_alt->orig_len - special_alt->new_len;
1779		nop->type = INSN_NOP;
1780		nop->sym = orig_insn->sym;
1781		nop->alt_group = new_alt_group;
1782		nop->ignore = orig_insn->ignore_alts;
1783	}
1784
1785	if (!special_alt->new_len) {
1786		*new_insn = nop;
1787		goto end;
1788	}
1789
1790	insn = *new_insn;
1791	sec_for_each_insn_from(file, insn) {
1792		struct reloc *alt_reloc;
1793
1794		if (insn->offset >= special_alt->new_off + special_alt->new_len)
1795			break;
1796
1797		last_new_insn = insn;
1798
1799		insn->ignore = orig_insn->ignore_alts;
1800		insn->sym = orig_insn->sym;
1801		insn->alt_group = new_alt_group;
1802
1803		/*
1804		 * Since alternative replacement code is copy/pasted by the
1805		 * kernel after applying relocations, generally such code can't
1806		 * have relative-address relocation references to outside the
1807		 * .altinstr_replacement section, unless the arch's
1808		 * alternatives code can adjust the relative offsets
1809		 * accordingly.
1810		 */
1811		alt_reloc = insn_reloc(file, insn);
1812		if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1813		    !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1814
1815			WARN_FUNC("unsupported relocation in alternatives section",
1816				  insn->sec, insn->offset);
1817			return -1;
1818		}
1819
1820		if (!is_static_jump(insn))
1821			continue;
1822
1823		if (!insn->immediate)
1824			continue;
1825
1826		dest_off = arch_jump_destination(insn);
1827		if (dest_off == special_alt->new_off + special_alt->new_len) {
1828			insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1829			if (!insn->jump_dest) {
1830				WARN_FUNC("can't find alternative jump destination",
1831					  insn->sec, insn->offset);
1832				return -1;
1833			}
1834		}
1835	}
1836
1837	if (!last_new_insn) {
1838		WARN_FUNC("can't find last new alternative instruction",
1839			  special_alt->new_sec, special_alt->new_off);
1840		return -1;
1841	}
1842
1843	if (nop)
1844		list_add(&nop->list, &last_new_insn->list);
1845end:
1846	new_alt_group->orig_group = orig_alt_group;
1847	new_alt_group->first_insn = *new_insn;
1848	new_alt_group->last_insn = nop ? : last_new_insn;
1849	new_alt_group->cfi = orig_alt_group->cfi;
1850	return 0;
1851}
1852
1853/*
1854 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1855 * If the original instruction is a jump, make the alt entry an effective nop
1856 * by just skipping the original instruction.
1857 */
1858static int handle_jump_alt(struct objtool_file *file,
1859			   struct special_alt *special_alt,
1860			   struct instruction *orig_insn,
1861			   struct instruction **new_insn)
1862{
1863	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1864	    orig_insn->type != INSN_NOP) {
1865
1866		WARN_FUNC("unsupported instruction at jump label",
1867			  orig_insn->sec, orig_insn->offset);
1868		return -1;
1869	}
1870
1871	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1872		struct reloc *reloc = insn_reloc(file, orig_insn);
1873
1874		if (reloc) {
1875			reloc->type = R_NONE;
1876			elf_write_reloc(file->elf, reloc);
1877		}
1878		elf_write_insn(file->elf, orig_insn->sec,
1879			       orig_insn->offset, orig_insn->len,
1880			       arch_nop_insn(orig_insn->len));
1881		orig_insn->type = INSN_NOP;
1882	}
1883
1884	if (orig_insn->type == INSN_NOP) {
1885		if (orig_insn->len == 2)
1886			file->jl_nop_short++;
1887		else
1888			file->jl_nop_long++;
1889
1890		return 0;
1891	}
1892
1893	if (orig_insn->len == 2)
1894		file->jl_short++;
1895	else
1896		file->jl_long++;
1897
1898	*new_insn = list_next_entry(orig_insn, list);
1899	return 0;
1900}
1901
1902/*
1903 * Read all the special sections which have alternate instructions which can be
1904 * patched in or redirected to at runtime.  Each instruction having alternate
1905 * instruction(s) has them added to its insn->alts list, which will be
1906 * traversed in validate_branch().
1907 */
1908static int add_special_section_alts(struct objtool_file *file)
1909{
1910	struct list_head special_alts;
1911	struct instruction *orig_insn, *new_insn;
1912	struct special_alt *special_alt, *tmp;
1913	struct alternative *alt;
1914	int ret;
1915
1916	ret = special_get_alts(file->elf, &special_alts);
1917	if (ret)
1918		return ret;
1919
1920	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1921
1922		orig_insn = find_insn(file, special_alt->orig_sec,
1923				      special_alt->orig_off);
1924		if (!orig_insn) {
1925			WARN_FUNC("special: can't find orig instruction",
1926				  special_alt->orig_sec, special_alt->orig_off);
1927			ret = -1;
1928			goto out;
1929		}
1930
1931		new_insn = NULL;
1932		if (!special_alt->group || special_alt->new_len) {
1933			new_insn = find_insn(file, special_alt->new_sec,
1934					     special_alt->new_off);
1935			if (!new_insn) {
1936				WARN_FUNC("special: can't find new instruction",
1937					  special_alt->new_sec,
1938					  special_alt->new_off);
1939				ret = -1;
1940				goto out;
1941			}
1942		}
1943
1944		if (special_alt->group) {
1945			if (!special_alt->orig_len) {
1946				WARN_FUNC("empty alternative entry",
1947					  orig_insn->sec, orig_insn->offset);
1948				continue;
1949			}
1950
1951			ret = handle_group_alt(file, special_alt, orig_insn,
1952					       &new_insn);
1953			if (ret)
1954				goto out;
1955		} else if (special_alt->jump_or_nop) {
1956			ret = handle_jump_alt(file, special_alt, orig_insn,
1957					      &new_insn);
1958			if (ret)
1959				goto out;
1960		}
1961
1962		alt = malloc(sizeof(*alt));
1963		if (!alt) {
1964			WARN("malloc failed");
1965			ret = -1;
1966			goto out;
1967		}
1968
1969		alt->insn = new_insn;
1970		alt->skip_orig = special_alt->skip_orig;
1971		orig_insn->ignore_alts |= special_alt->skip_alt;
1972		list_add_tail(&alt->list, &orig_insn->alts);
1973
1974		list_del(&special_alt->list);
1975		free(special_alt);
1976	}
1977
1978	if (opts.stats) {
1979		printf("jl\\\tNOP\tJMP\n");
1980		printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1981		printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1982	}
1983
1984out:
1985	return ret;
1986}
1987
1988static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1989			    struct reloc *table)
1990{
1991	struct reloc *reloc = table;
1992	struct instruction *dest_insn;
1993	struct alternative *alt;
1994	struct symbol *pfunc = insn_func(insn)->pfunc;
1995	unsigned int prev_offset = 0;
1996
1997	/*
1998	 * Each @reloc is a switch table relocation which points to the target
1999	 * instruction.
2000	 */
2001	list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
2002
2003		/* Check for the end of the table: */
2004		if (reloc != table && reloc->jump_table_start)
2005			break;
2006
2007		/* Make sure the table entries are consecutive: */
2008		if (prev_offset && reloc->offset != prev_offset + 8)
2009			break;
2010
2011		/* Detect function pointers from contiguous objects: */
2012		if (reloc->sym->sec == pfunc->sec &&
2013		    reloc->addend == pfunc->offset)
2014			break;
2015
2016		dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
2017		if (!dest_insn)
2018			break;
2019
2020		/* Make sure the destination is in the same function: */
2021		if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2022			break;
2023
2024		alt = malloc(sizeof(*alt));
2025		if (!alt) {
2026			WARN("malloc failed");
2027			return -1;
2028		}
2029
2030		alt->insn = dest_insn;
2031		list_add_tail(&alt->list, &insn->alts);
2032		prev_offset = reloc->offset;
2033	}
2034
2035	if (!prev_offset) {
2036		WARN_FUNC("can't find switch jump table",
2037			  insn->sec, insn->offset);
2038		return -1;
2039	}
2040
2041	return 0;
2042}
2043
2044/*
2045 * find_jump_table() - Given a dynamic jump, find the switch jump table
2046 * associated with it.
2047 */
2048static struct reloc *find_jump_table(struct objtool_file *file,
2049				      struct symbol *func,
2050				      struct instruction *insn)
2051{
2052	struct reloc *table_reloc;
2053	struct instruction *dest_insn, *orig_insn = insn;
2054
2055	/*
2056	 * Backward search using the @first_jump_src links, these help avoid
2057	 * much of the 'in between' code. Which avoids us getting confused by
2058	 * it.
2059	 */
2060	for (;
2061	     insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2062	     insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2063
2064		if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2065			break;
2066
2067		/* allow small jumps within the range */
2068		if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2069		    insn->jump_dest &&
2070		    (insn->jump_dest->offset <= insn->offset ||
2071		     insn->jump_dest->offset > orig_insn->offset))
2072		    break;
2073
2074		table_reloc = arch_find_switch_table(file, insn);
2075		if (!table_reloc)
2076			continue;
2077		dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
2078		if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2079			continue;
2080
2081		return table_reloc;
2082	}
2083
2084	return NULL;
2085}
2086
2087/*
2088 * First pass: Mark the head of each jump table so that in the next pass,
2089 * we know when a given jump table ends and the next one starts.
2090 */
2091static void mark_func_jump_tables(struct objtool_file *file,
2092				    struct symbol *func)
2093{
2094	struct instruction *insn, *last = NULL;
2095	struct reloc *reloc;
2096
2097	func_for_each_insn(file, func, insn) {
2098		if (!last)
2099			last = insn;
2100
2101		/*
2102		 * Store back-pointers for unconditional forward jumps such
2103		 * that find_jump_table() can back-track using those and
2104		 * avoid some potentially confusing code.
2105		 */
2106		if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2107		    insn->offset > last->offset &&
2108		    insn->jump_dest->offset > insn->offset &&
2109		    !insn->jump_dest->first_jump_src) {
2110
2111			insn->jump_dest->first_jump_src = insn;
2112			last = insn->jump_dest;
2113		}
2114
2115		if (insn->type != INSN_JUMP_DYNAMIC)
2116			continue;
2117
2118		reloc = find_jump_table(file, func, insn);
2119		if (reloc) {
2120			reloc->jump_table_start = true;
2121			insn->jump_table = reloc;
2122		}
2123	}
2124}
2125
2126static int add_func_jump_tables(struct objtool_file *file,
2127				  struct symbol *func)
2128{
2129	struct instruction *insn;
2130	int ret;
2131
2132	func_for_each_insn(file, func, insn) {
2133		if (!insn->jump_table)
2134			continue;
2135
2136		ret = add_jump_table(file, insn, insn->jump_table);
2137		if (ret)
2138			return ret;
2139	}
2140
2141	return 0;
2142}
2143
2144/*
2145 * For some switch statements, gcc generates a jump table in the .rodata
2146 * section which contains a list of addresses within the function to jump to.
2147 * This finds these jump tables and adds them to the insn->alts lists.
2148 */
2149static int add_jump_table_alts(struct objtool_file *file)
2150{
2151	struct section *sec;
2152	struct symbol *func;
2153	int ret;
2154
2155	if (!file->rodata)
2156		return 0;
2157
2158	for_each_sec(file, sec) {
2159		list_for_each_entry(func, &sec->symbol_list, list) {
2160			if (func->type != STT_FUNC)
2161				continue;
2162
2163			mark_func_jump_tables(file, func);
2164			ret = add_func_jump_tables(file, func);
2165			if (ret)
2166				return ret;
2167		}
2168	}
2169
2170	return 0;
2171}
2172
2173static void set_func_state(struct cfi_state *state)
2174{
2175	state->cfa = initial_func_cfi.cfa;
2176	memcpy(&state->regs, &initial_func_cfi.regs,
2177	       CFI_NUM_REGS * sizeof(struct cfi_reg));
2178	state->stack_size = initial_func_cfi.cfa.offset;
2179}
2180
2181static int read_unwind_hints(struct objtool_file *file)
2182{
2183	struct cfi_state cfi = init_cfi;
2184	struct section *sec, *relocsec;
2185	struct unwind_hint *hint;
2186	struct instruction *insn;
2187	struct reloc *reloc;
2188	int i;
2189
2190	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2191	if (!sec)
2192		return 0;
2193
2194	relocsec = sec->reloc;
2195	if (!relocsec) {
2196		WARN("missing .rela.discard.unwind_hints section");
2197		return -1;
2198	}
2199
2200	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2201		WARN("struct unwind_hint size mismatch");
2202		return -1;
2203	}
2204
2205	file->hints = true;
2206
2207	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2208		hint = (struct unwind_hint *)sec->data->d_buf + i;
2209
2210		reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2211		if (!reloc) {
2212			WARN("can't find reloc for unwind_hints[%d]", i);
2213			return -1;
2214		}
2215
2216		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2217		if (!insn) {
2218			WARN("can't find insn for unwind_hints[%d]", i);
2219			return -1;
2220		}
2221
2222		insn->hint = true;
2223
2224		if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2225			insn->hint = false;
2226			insn->save = true;
2227			continue;
2228		}
2229
2230		if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2231			insn->restore = true;
2232			continue;
2233		}
2234
2235		if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2236			struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2237
2238			if (sym && sym->bind == STB_GLOBAL) {
2239				if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2240					WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
2241						  insn->sec, insn->offset);
2242				}
2243
2244				insn->entry = 1;
2245			}
2246		}
2247
2248		if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
2249			hint->type = UNWIND_HINT_TYPE_CALL;
2250			insn->entry = 1;
2251		}
2252
2253		if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2254			insn->cfi = &func_cfi;
2255			continue;
2256		}
2257
2258		if (insn->cfi)
2259			cfi = *(insn->cfi);
2260
2261		if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2262			WARN_FUNC("unsupported unwind_hint sp base reg %d",
2263				  insn->sec, insn->offset, hint->sp_reg);
2264			return -1;
2265		}
2266
2267		cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2268		cfi.type = hint->type;
2269		cfi.end = hint->end;
2270
2271		insn->cfi = cfi_hash_find_or_add(&cfi);
2272	}
2273
2274	return 0;
2275}
2276
2277static int read_noendbr_hints(struct objtool_file *file)
2278{
2279	struct section *sec;
2280	struct instruction *insn;
2281	struct reloc *reloc;
2282
2283	sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2284	if (!sec)
2285		return 0;
2286
2287	list_for_each_entry(reloc, &sec->reloc_list, list) {
2288		insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2289		if (!insn) {
2290			WARN("bad .discard.noendbr entry");
2291			return -1;
2292		}
2293
2294		insn->noendbr = 1;
2295	}
2296
2297	return 0;
2298}
2299
2300static int read_retpoline_hints(struct objtool_file *file)
2301{
2302	struct section *sec;
2303	struct instruction *insn;
2304	struct reloc *reloc;
2305
2306	sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2307	if (!sec)
2308		return 0;
2309
2310	list_for_each_entry(reloc, &sec->reloc_list, list) {
2311		if (reloc->sym->type != STT_SECTION) {
2312			WARN("unexpected relocation symbol type in %s", sec->name);
2313			return -1;
2314		}
2315
2316		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2317		if (!insn) {
2318			WARN("bad .discard.retpoline_safe entry");
2319			return -1;
2320		}
2321
2322		if (insn->type != INSN_JUMP_DYNAMIC &&
2323		    insn->type != INSN_CALL_DYNAMIC &&
2324		    insn->type != INSN_RETURN &&
2325		    insn->type != INSN_NOP) {
2326			WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
2327				  insn->sec, insn->offset);
2328			return -1;
2329		}
2330
2331		insn->retpoline_safe = true;
2332	}
2333
2334	return 0;
2335}
2336
2337static int read_instr_hints(struct objtool_file *file)
2338{
2339	struct section *sec;
2340	struct instruction *insn;
2341	struct reloc *reloc;
2342
2343	sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2344	if (!sec)
2345		return 0;
2346
2347	list_for_each_entry(reloc, &sec->reloc_list, list) {
2348		if (reloc->sym->type != STT_SECTION) {
2349			WARN("unexpected relocation symbol type in %s", sec->name);
2350			return -1;
2351		}
2352
2353		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2354		if (!insn) {
2355			WARN("bad .discard.instr_end entry");
2356			return -1;
2357		}
2358
2359		insn->instr--;
2360	}
2361
2362	sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2363	if (!sec)
2364		return 0;
2365
2366	list_for_each_entry(reloc, &sec->reloc_list, list) {
2367		if (reloc->sym->type != STT_SECTION) {
2368			WARN("unexpected relocation symbol type in %s", sec->name);
2369			return -1;
2370		}
2371
2372		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2373		if (!insn) {
2374			WARN("bad .discard.instr_begin entry");
2375			return -1;
2376		}
2377
2378		insn->instr++;
2379	}
2380
2381	return 0;
2382}
2383
2384static int read_intra_function_calls(struct objtool_file *file)
2385{
2386	struct instruction *insn;
2387	struct section *sec;
2388	struct reloc *reloc;
2389
2390	sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2391	if (!sec)
2392		return 0;
2393
2394	list_for_each_entry(reloc, &sec->reloc_list, list) {
2395		unsigned long dest_off;
2396
2397		if (reloc->sym->type != STT_SECTION) {
2398			WARN("unexpected relocation symbol type in %s",
2399			     sec->name);
2400			return -1;
2401		}
2402
2403		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2404		if (!insn) {
2405			WARN("bad .discard.intra_function_call entry");
2406			return -1;
2407		}
2408
2409		if (insn->type != INSN_CALL) {
2410			WARN_FUNC("intra_function_call not a direct call",
2411				  insn->sec, insn->offset);
2412			return -1;
2413		}
2414
2415		/*
2416		 * Treat intra-function CALLs as JMPs, but with a stack_op.
2417		 * See add_call_destinations(), which strips stack_ops from
2418		 * normal CALLs.
2419		 */
2420		insn->type = INSN_JUMP_UNCONDITIONAL;
2421
2422		dest_off = arch_jump_destination(insn);
2423		insn->jump_dest = find_insn(file, insn->sec, dest_off);
2424		if (!insn->jump_dest) {
2425			WARN_FUNC("can't find call dest at %s+0x%lx",
2426				  insn->sec, insn->offset,
2427				  insn->sec->name, dest_off);
2428			return -1;
2429		}
2430	}
2431
2432	return 0;
2433}
2434
2435/*
2436 * Return true if name matches an instrumentation function, where calls to that
2437 * function from noinstr code can safely be removed, but compilers won't do so.
2438 */
2439static bool is_profiling_func(const char *name)
2440{
2441	/*
2442	 * Many compilers cannot disable KCOV with a function attribute.
2443	 */
2444	if (!strncmp(name, "__sanitizer_cov_", 16))
2445		return true;
2446
2447	/*
2448	 * Some compilers currently do not remove __tsan_func_entry/exit nor
2449	 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2450	 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2451	 * minimum Clang version is 14.0, this can be removed.
2452	 */
2453	if (!strncmp(name, "__tsan_func_", 12) ||
2454	    !strcmp(name, "__tsan_atomic_signal_fence"))
2455		return true;
2456
2457	return false;
2458}
2459
2460static int classify_symbols(struct objtool_file *file)
2461{
2462	struct section *sec;
2463	struct symbol *func;
2464
2465	for_each_sec(file, sec) {
2466		list_for_each_entry(func, &sec->symbol_list, list) {
2467			if (func->bind != STB_GLOBAL)
2468				continue;
2469
2470			if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2471				     strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2472				func->static_call_tramp = true;
2473
2474			if (arch_is_retpoline(func))
2475				func->retpoline_thunk = true;
2476
2477			if (arch_is_rethunk(func))
2478				func->return_thunk = true;
2479
2480			if (arch_ftrace_match(func->name))
2481				func->fentry = true;
2482
2483			if (is_profiling_func(func->name))
2484				func->profiling_func = true;
2485		}
2486	}
2487
2488	return 0;
2489}
2490
2491static void mark_rodata(struct objtool_file *file)
2492{
2493	struct section *sec;
2494	bool found = false;
2495
2496	/*
2497	 * Search for the following rodata sections, each of which can
2498	 * potentially contain jump tables:
2499	 *
2500	 * - .rodata: can contain GCC switch tables
2501	 * - .rodata.<func>: same, if -fdata-sections is being used
2502	 * - .rodata..c_jump_table: contains C annotated jump tables
2503	 *
2504	 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2505	 */
2506	for_each_sec(file, sec) {
2507		if (!strncmp(sec->name, ".rodata", 7) &&
2508		    !strstr(sec->name, ".str1.")) {
2509			sec->rodata = true;
2510			found = true;
2511		}
2512	}
2513
2514	file->rodata = found;
2515}
2516
2517static int decode_sections(struct objtool_file *file)
2518{
2519	int ret;
2520
2521	mark_rodata(file);
2522
2523	ret = init_pv_ops(file);
2524	if (ret)
2525		return ret;
2526
2527	/*
2528	 * Must be before add_{jump_call}_destination.
2529	 */
2530	ret = classify_symbols(file);
2531	if (ret)
2532		return ret;
2533
2534	ret = decode_instructions(file);
2535	if (ret)
2536		return ret;
2537
2538	add_ignores(file);
2539	add_uaccess_safe(file);
2540
2541	ret = add_ignore_alternatives(file);
2542	if (ret)
2543		return ret;
2544
2545	/*
2546	 * Must be before read_unwind_hints() since that needs insn->noendbr.
2547	 */
2548	ret = read_noendbr_hints(file);
2549	if (ret)
2550		return ret;
2551
2552	/*
2553	 * Must be before add_jump_destinations(), which depends on 'func'
2554	 * being set for alternatives, to enable proper sibling call detection.
2555	 */
2556	if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2557		ret = add_special_section_alts(file);
2558		if (ret)
2559			return ret;
2560	}
2561
2562	ret = add_jump_destinations(file);
2563	if (ret)
2564		return ret;
2565
2566	/*
2567	 * Must be before add_call_destination(); it changes INSN_CALL to
2568	 * INSN_JUMP.
2569	 */
2570	ret = read_intra_function_calls(file);
2571	if (ret)
2572		return ret;
2573
2574	ret = add_call_destinations(file);
2575	if (ret)
2576		return ret;
2577
2578	/*
2579	 * Must be after add_call_destinations() such that it can override
2580	 * dead_end_function() marks.
2581	 */
2582	ret = add_dead_ends(file);
2583	if (ret)
2584		return ret;
2585
2586	ret = add_jump_table_alts(file);
2587	if (ret)
2588		return ret;
2589
2590	ret = read_unwind_hints(file);
2591	if (ret)
2592		return ret;
2593
2594	ret = read_retpoline_hints(file);
2595	if (ret)
2596		return ret;
2597
2598	ret = read_instr_hints(file);
2599	if (ret)
2600		return ret;
2601
2602	return 0;
2603}
2604
2605static bool is_fentry_call(struct instruction *insn)
2606{
2607	if (insn->type == INSN_CALL &&
2608	    insn->call_dest &&
2609	    insn->call_dest->fentry)
2610		return true;
2611
2612	return false;
2613}
2614
2615static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2616{
2617	struct cfi_state *cfi = &state->cfi;
2618	int i;
2619
2620	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2621		return true;
2622
2623	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2624		return true;
2625
2626	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2627		return true;
2628
2629	for (i = 0; i < CFI_NUM_REGS; i++) {
2630		if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2631		    cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2632			return true;
2633	}
2634
2635	return false;
2636}
2637
2638static bool check_reg_frame_pos(const struct cfi_reg *reg,
2639				int expected_offset)
2640{
2641	return reg->base == CFI_CFA &&
2642	       reg->offset == expected_offset;
2643}
2644
2645static bool has_valid_stack_frame(struct insn_state *state)
2646{
2647	struct cfi_state *cfi = &state->cfi;
2648
2649	if (cfi->cfa.base == CFI_BP &&
2650	    check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2651	    check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2652		return true;
2653
2654	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2655		return true;
2656
2657	return false;
2658}
2659
2660static int update_cfi_state_regs(struct instruction *insn,
2661				  struct cfi_state *cfi,
2662				  struct stack_op *op)
2663{
2664	struct cfi_reg *cfa = &cfi->cfa;
2665
2666	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2667		return 0;
2668
2669	/* push */
2670	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2671		cfa->offset += 8;
2672
2673	/* pop */
2674	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2675		cfa->offset -= 8;
2676
2677	/* add immediate to sp */
2678	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2679	    op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2680		cfa->offset -= op->src.offset;
2681
2682	return 0;
2683}
2684
2685static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2686{
2687	if (arch_callee_saved_reg(reg) &&
2688	    cfi->regs[reg].base == CFI_UNDEFINED) {
2689		cfi->regs[reg].base = base;
2690		cfi->regs[reg].offset = offset;
2691	}
2692}
2693
2694static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2695{
2696	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2697	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2698}
2699
2700/*
2701 * A note about DRAP stack alignment:
2702 *
2703 * GCC has the concept of a DRAP register, which is used to help keep track of
2704 * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2705 * register.  The typical DRAP pattern is:
2706 *
2707 *   4c 8d 54 24 08		lea    0x8(%rsp),%r10
2708 *   48 83 e4 c0		and    $0xffffffffffffffc0,%rsp
2709 *   41 ff 72 f8		pushq  -0x8(%r10)
2710 *   55				push   %rbp
2711 *   48 89 e5			mov    %rsp,%rbp
2712 *				(more pushes)
2713 *   41 52			push   %r10
2714 *				...
2715 *   41 5a			pop    %r10
2716 *				(more pops)
2717 *   5d				pop    %rbp
2718 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2719 *   c3				retq
2720 *
2721 * There are some variations in the epilogues, like:
2722 *
2723 *   5b				pop    %rbx
2724 *   41 5a			pop    %r10
2725 *   41 5c			pop    %r12
2726 *   41 5d			pop    %r13
2727 *   41 5e			pop    %r14
2728 *   c9				leaveq
2729 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2730 *   c3				retq
2731 *
2732 * and:
2733 *
2734 *   4c 8b 55 e8		mov    -0x18(%rbp),%r10
2735 *   48 8b 5d e0		mov    -0x20(%rbp),%rbx
2736 *   4c 8b 65 f0		mov    -0x10(%rbp),%r12
2737 *   4c 8b 6d f8		mov    -0x8(%rbp),%r13
2738 *   c9				leaveq
2739 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2740 *   c3				retq
2741 *
2742 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2743 * restored beforehand:
2744 *
2745 *   41 55			push   %r13
2746 *   4c 8d 6c 24 10		lea    0x10(%rsp),%r13
2747 *   48 83 e4 f0		and    $0xfffffffffffffff0,%rsp
2748 *				...
2749 *   49 8d 65 f0		lea    -0x10(%r13),%rsp
2750 *   41 5d			pop    %r13
2751 *   c3				retq
2752 */
2753static int update_cfi_state(struct instruction *insn,
2754			    struct instruction *next_insn,
2755			    struct cfi_state *cfi, struct stack_op *op)
2756{
2757	struct cfi_reg *cfa = &cfi->cfa;
2758	struct cfi_reg *regs = cfi->regs;
2759
2760	/* stack operations don't make sense with an undefined CFA */
2761	if (cfa->base == CFI_UNDEFINED) {
2762		if (insn_func(insn)) {
2763			WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2764			return -1;
2765		}
2766		return 0;
2767	}
2768
2769	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2770	    cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2771		return update_cfi_state_regs(insn, cfi, op);
2772
2773	switch (op->dest.type) {
2774
2775	case OP_DEST_REG:
2776		switch (op->src.type) {
2777
2778		case OP_SRC_REG:
2779			if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2780			    cfa->base == CFI_SP &&
2781			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2782
2783				/* mov %rsp, %rbp */
2784				cfa->base = op->dest.reg;
2785				cfi->bp_scratch = false;
2786			}
2787
2788			else if (op->src.reg == CFI_SP &&
2789				 op->dest.reg == CFI_BP && cfi->drap) {
2790
2791				/* drap: mov %rsp, %rbp */
2792				regs[CFI_BP].base = CFI_BP;
2793				regs[CFI_BP].offset = -cfi->stack_size;
2794				cfi->bp_scratch = false;
2795			}
2796
2797			else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2798
2799				/*
2800				 * mov %rsp, %reg
2801				 *
2802				 * This is needed for the rare case where GCC
2803				 * does:
2804				 *
2805				 *   mov    %rsp, %rax
2806				 *   ...
2807				 *   mov    %rax, %rsp
2808				 */
2809				cfi->vals[op->dest.reg].base = CFI_CFA;
2810				cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2811			}
2812
2813			else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2814				 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2815
2816				/*
2817				 * mov %rbp, %rsp
2818				 *
2819				 * Restore the original stack pointer (Clang).
2820				 */
2821				cfi->stack_size = -cfi->regs[CFI_BP].offset;
2822			}
2823
2824			else if (op->dest.reg == cfa->base) {
2825
2826				/* mov %reg, %rsp */
2827				if (cfa->base == CFI_SP &&
2828				    cfi->vals[op->src.reg].base == CFI_CFA) {
2829
2830					/*
2831					 * This is needed for the rare case
2832					 * where GCC does something dumb like:
2833					 *
2834					 *   lea    0x8(%rsp), %rcx
2835					 *   ...
2836					 *   mov    %rcx, %rsp
2837					 */
2838					cfa->offset = -cfi->vals[op->src.reg].offset;
2839					cfi->stack_size = cfa->offset;
2840
2841				} else if (cfa->base == CFI_SP &&
2842					   cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2843					   cfi->vals[op->src.reg].offset == cfa->offset) {
2844
2845					/*
2846					 * Stack swizzle:
2847					 *
2848					 * 1: mov %rsp, (%[tos])
2849					 * 2: mov %[tos], %rsp
2850					 *    ...
2851					 * 3: pop %rsp
2852					 *
2853					 * Where:
2854					 *
2855					 * 1 - places a pointer to the previous
2856					 *     stack at the Top-of-Stack of the
2857					 *     new stack.
2858					 *
2859					 * 2 - switches to the new stack.
2860					 *
2861					 * 3 - pops the Top-of-Stack to restore
2862					 *     the original stack.
2863					 *
2864					 * Note: we set base to SP_INDIRECT
2865					 * here and preserve offset. Therefore
2866					 * when the unwinder reaches ToS it
2867					 * will dereference SP and then add the
2868					 * offset to find the next frame, IOW:
2869					 * (%rsp) + offset.
2870					 */
2871					cfa->base = CFI_SP_INDIRECT;
2872
2873				} else {
2874					cfa->base = CFI_UNDEFINED;
2875					cfa->offset = 0;
2876				}
2877			}
2878
2879			else if (op->dest.reg == CFI_SP &&
2880				 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2881				 cfi->vals[op->src.reg].offset == cfa->offset) {
2882
2883				/*
2884				 * The same stack swizzle case 2) as above. But
2885				 * because we can't change cfa->base, case 3)
2886				 * will become a regular POP. Pretend we're a
2887				 * PUSH so things don't go unbalanced.
2888				 */
2889				cfi->stack_size += 8;
2890			}
2891
2892
2893			break;
2894
2895		case OP_SRC_ADD:
2896			if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2897
2898				/* add imm, %rsp */
2899				cfi->stack_size -= op->src.offset;
2900				if (cfa->base == CFI_SP)
2901					cfa->offset -= op->src.offset;
2902				break;
2903			}
2904
2905			if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2906
2907				/* lea disp(%rbp), %rsp */
2908				cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2909				break;
2910			}
2911
2912			if (!cfi->drap && op->src.reg == CFI_SP &&
2913			    op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2914			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2915
2916				/* lea disp(%rsp), %rbp */
2917				cfa->base = CFI_BP;
2918				cfa->offset -= op->src.offset;
2919				cfi->bp_scratch = false;
2920				break;
2921			}
2922
2923			if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2924
2925				/* drap: lea disp(%rsp), %drap */
2926				cfi->drap_reg = op->dest.reg;
2927
2928				/*
2929				 * lea disp(%rsp), %reg
2930				 *
2931				 * This is needed for the rare case where GCC
2932				 * does something dumb like:
2933				 *
2934				 *   lea    0x8(%rsp), %rcx
2935				 *   ...
2936				 *   mov    %rcx, %rsp
2937				 */
2938				cfi->vals[op->dest.reg].base = CFI_CFA;
2939				cfi->vals[op->dest.reg].offset = \
2940					-cfi->stack_size + op->src.offset;
2941
2942				break;
2943			}
2944
2945			if (cfi->drap && op->dest.reg == CFI_SP &&
2946			    op->src.reg == cfi->drap_reg) {
2947
2948				 /* drap: lea disp(%drap), %rsp */
2949				cfa->base = CFI_SP;
2950				cfa->offset = cfi->stack_size = -op->src.offset;
2951				cfi->drap_reg = CFI_UNDEFINED;
2952				cfi->drap = false;
2953				break;
2954			}
2955
2956			if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2957				WARN_FUNC("unsupported stack register modification",
2958					  insn->sec, insn->offset);
2959				return -1;
2960			}
2961
2962			break;
2963
2964		case OP_SRC_AND:
2965			if (op->dest.reg != CFI_SP ||
2966			    (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2967			    (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2968				WARN_FUNC("unsupported stack pointer realignment",
2969					  insn->sec, insn->offset);
2970				return -1;
2971			}
2972
2973			if (cfi->drap_reg != CFI_UNDEFINED) {
2974				/* drap: and imm, %rsp */
2975				cfa->base = cfi->drap_reg;
2976				cfa->offset = cfi->stack_size = 0;
2977				cfi->drap = true;
2978			}
2979
2980			/*
2981			 * Older versions of GCC (4.8ish) realign the stack
2982			 * without DRAP, with a frame pointer.
2983			 */
2984
2985			break;
2986
2987		case OP_SRC_POP:
2988		case OP_SRC_POPF:
2989			if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2990
2991				/* pop %rsp; # restore from a stack swizzle */
2992				cfa->base = CFI_SP;
2993				break;
2994			}
2995
2996			if (!cfi->drap && op->dest.reg == cfa->base) {
2997
2998				/* pop %rbp */
2999				cfa->base = CFI_SP;
3000			}
3001
3002			if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3003			    op->dest.reg == cfi->drap_reg &&
3004			    cfi->drap_offset == -cfi->stack_size) {
3005
3006				/* drap: pop %drap */
3007				cfa->base = cfi->drap_reg;
3008				cfa->offset = 0;
3009				cfi->drap_offset = -1;
3010
3011			} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3012
3013				/* pop %reg */
3014				restore_reg(cfi, op->dest.reg);
3015			}
3016
3017			cfi->stack_size -= 8;
3018			if (cfa->base == CFI_SP)
3019				cfa->offset -= 8;
3020
3021			break;
3022
3023		case OP_SRC_REG_INDIRECT:
3024			if (!cfi->drap && op->dest.reg == cfa->base &&
3025			    op->dest.reg == CFI_BP) {
3026
3027				/* mov disp(%rsp), %rbp */
3028				cfa->base = CFI_SP;
3029				cfa->offset = cfi->stack_size;
3030			}
3031
3032			if (cfi->drap && op->src.reg == CFI_BP &&
3033			    op->src.offset == cfi->drap_offset) {
3034
3035				/* drap: mov disp(%rbp), %drap */
3036				cfa->base = cfi->drap_reg;
3037				cfa->offset = 0;
3038				cfi->drap_offset = -1;
3039			}
3040
3041			if (cfi->drap && op->src.reg == CFI_BP &&
3042			    op->src.offset == regs[op->dest.reg].offset) {
3043
3044				/* drap: mov disp(%rbp), %reg */
3045				restore_reg(cfi, op->dest.reg);
3046
3047			} else if (op->src.reg == cfa->base &&
3048			    op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3049
3050				/* mov disp(%rbp), %reg */
3051				/* mov disp(%rsp), %reg */
3052				restore_reg(cfi, op->dest.reg);
3053
3054			} else if (op->src.reg == CFI_SP &&
3055				   op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3056
3057				/* mov disp(%rsp), %reg */
3058				restore_reg(cfi, op->dest.reg);
3059			}
3060
3061			break;
3062
3063		default:
3064			WARN_FUNC("unknown stack-related instruction",
3065				  insn->sec, insn->offset);
3066			return -1;
3067		}
3068
3069		break;
3070
3071	case OP_DEST_PUSH:
3072	case OP_DEST_PUSHF:
3073		cfi->stack_size += 8;
3074		if (cfa->base == CFI_SP)
3075			cfa->offset += 8;
3076
3077		if (op->src.type != OP_SRC_REG)
3078			break;
3079
3080		if (cfi->drap) {
3081			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3082
3083				/* drap: push %drap */
3084				cfa->base = CFI_BP_INDIRECT;
3085				cfa->offset = -cfi->stack_size;
3086
3087				/* save drap so we know when to restore it */
3088				cfi->drap_offset = -cfi->stack_size;
3089
3090			} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3091
3092				/* drap: push %rbp */
3093				cfi->stack_size = 0;
3094
3095			} else {
3096
3097				/* drap: push %reg */
3098				save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3099			}
3100
3101		} else {
3102
3103			/* push %reg */
3104			save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3105		}
3106
3107		/* detect when asm code uses rbp as a scratch register */
3108		if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3109		    cfa->base != CFI_BP)
3110			cfi->bp_scratch = true;
3111		break;
3112
3113	case OP_DEST_REG_INDIRECT:
3114
3115		if (cfi->drap) {
3116			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3117
3118				/* drap: mov %drap, disp(%rbp) */
3119				cfa->base = CFI_BP_INDIRECT;
3120				cfa->offset = op->dest.offset;
3121
3122				/* save drap offset so we know when to restore it */
3123				cfi->drap_offset = op->dest.offset;
3124			} else {
3125
3126				/* drap: mov reg, disp(%rbp) */
3127				save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3128			}
3129
3130		} else if (op->dest.reg == cfa->base) {
3131
3132			/* mov reg, disp(%rbp) */
3133			/* mov reg, disp(%rsp) */
3134			save_reg(cfi, op->src.reg, CFI_CFA,
3135				 op->dest.offset - cfi->cfa.offset);
3136
3137		} else if (op->dest.reg == CFI_SP) {
3138
3139			/* mov reg, disp(%rsp) */
3140			save_reg(cfi, op->src.reg, CFI_CFA,
3141				 op->dest.offset - cfi->stack_size);
3142
3143		} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3144
3145			/* mov %rsp, (%reg); # setup a stack swizzle. */
3146			cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3147			cfi->vals[op->dest.reg].offset = cfa->offset;
3148		}
3149
3150		break;
3151
3152	case OP_DEST_MEM:
3153		if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3154			WARN_FUNC("unknown stack-related memory operation",
3155				  insn->sec, insn->offset);
3156			return -1;
3157		}
3158
3159		/* pop mem */
3160		cfi->stack_size -= 8;
3161		if (cfa->base == CFI_SP)
3162			cfa->offset -= 8;
3163
3164		break;
3165
3166	default:
3167		WARN_FUNC("unknown stack-related instruction",
3168			  insn->sec, insn->offset);
3169		return -1;
3170	}
3171
3172	return 0;
3173}
3174
3175/*
3176 * The stack layouts of alternatives instructions can sometimes diverge when
3177 * they have stack modifications.  That's fine as long as the potential stack
3178 * layouts don't conflict at any given potential instruction boundary.
3179 *
3180 * Flatten the CFIs of the different alternative code streams (both original
3181 * and replacement) into a single shared CFI array which can be used to detect
3182 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3183 */
3184static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3185{
3186	struct cfi_state **alt_cfi;
3187	int group_off;
3188
3189	if (!insn->alt_group)
3190		return 0;
3191
3192	if (!insn->cfi) {
3193		WARN("CFI missing");
3194		return -1;
3195	}
3196
3197	alt_cfi = insn->alt_group->cfi;
3198	group_off = insn->offset - insn->alt_group->first_insn->offset;
3199
3200	if (!alt_cfi[group_off]) {
3201		alt_cfi[group_off] = insn->cfi;
3202	} else {
3203		if (cficmp(alt_cfi[group_off], insn->cfi)) {
3204			WARN_FUNC("stack layout conflict in alternatives",
3205				  insn->sec, insn->offset);
3206			return -1;
3207		}
3208	}
3209
3210	return 0;
3211}
3212
3213static int handle_insn_ops(struct instruction *insn,
3214			   struct instruction *next_insn,
3215			   struct insn_state *state)
3216{
3217	struct stack_op *op;
3218
3219	list_for_each_entry(op, &insn->stack_ops, list) {
3220
3221		if (update_cfi_state(insn, next_insn, &state->cfi, op))
3222			return 1;
3223
3224		if (!insn->alt_group)
3225			continue;
3226
3227		if (op->dest.type == OP_DEST_PUSHF) {
3228			if (!state->uaccess_stack) {
3229				state->uaccess_stack = 1;
3230			} else if (state->uaccess_stack >> 31) {
3231				WARN_FUNC("PUSHF stack exhausted",
3232					  insn->sec, insn->offset);
3233				return 1;
3234			}
3235			state->uaccess_stack <<= 1;
3236			state->uaccess_stack  |= state->uaccess;
3237		}
3238
3239		if (op->src.type == OP_SRC_POPF) {
3240			if (state->uaccess_stack) {
3241				state->uaccess = state->uaccess_stack & 1;
3242				state->uaccess_stack >>= 1;
3243				if (state->uaccess_stack == 1)
3244					state->uaccess_stack = 0;
3245			}
3246		}
3247	}
3248
3249	return 0;
3250}
3251
3252static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3253{
3254	struct cfi_state *cfi1 = insn->cfi;
3255	int i;
3256
3257	if (!cfi1) {
3258		WARN("CFI missing");
3259		return false;
3260	}
3261
3262	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3263
3264		WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3265			  insn->sec, insn->offset,
3266			  cfi1->cfa.base, cfi1->cfa.offset,
3267			  cfi2->cfa.base, cfi2->cfa.offset);
3268
3269	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3270		for (i = 0; i < CFI_NUM_REGS; i++) {
3271			if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3272				    sizeof(struct cfi_reg)))
3273				continue;
3274
3275			WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3276				  insn->sec, insn->offset,
3277				  i, cfi1->regs[i].base, cfi1->regs[i].offset,
3278				  i, cfi2->regs[i].base, cfi2->regs[i].offset);
3279			break;
3280		}
3281
3282	} else if (cfi1->type != cfi2->type) {
3283
3284		WARN_FUNC("stack state mismatch: type1=%d type2=%d",
3285			  insn->sec, insn->offset, cfi1->type, cfi2->type);
3286
3287	} else if (cfi1->drap != cfi2->drap ||
3288		   (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3289		   (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3290
3291		WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3292			  insn->sec, insn->offset,
3293			  cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3294			  cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3295
3296	} else
3297		return true;
3298
3299	return false;
3300}
3301
3302static inline bool func_uaccess_safe(struct symbol *func)
3303{
3304	if (func)
3305		return func->uaccess_safe;
3306
3307	return false;
3308}
3309
3310static inline const char *call_dest_name(struct instruction *insn)
3311{
3312	static char pvname[19];
3313	struct reloc *rel;
3314	int idx;
3315
3316	if (insn->call_dest)
3317		return insn->call_dest->name;
3318
3319	rel = insn_reloc(NULL, insn);
3320	if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3321		idx = (rel->addend / sizeof(void *));
3322		snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3323		return pvname;
3324	}
3325
3326	return "{dynamic}";
3327}
3328
3329static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3330{
3331	struct symbol *target;
3332	struct reloc *rel;
3333	int idx;
3334
3335	rel = insn_reloc(file, insn);
3336	if (!rel || strcmp(rel->sym->name, "pv_ops"))
3337		return false;
3338
3339	idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3340
3341	if (file->pv_ops[idx].clean)
3342		return true;
3343
3344	file->pv_ops[idx].clean = true;
3345
3346	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3347		if (!target->sec->noinstr) {
3348			WARN("pv_ops[%d]: %s", idx, target->name);
3349			file->pv_ops[idx].clean = false;
3350		}
3351	}
3352
3353	return file->pv_ops[idx].clean;
3354}
3355
3356static inline bool noinstr_call_dest(struct objtool_file *file,
3357				     struct instruction *insn,
3358				     struct symbol *func)
3359{
3360	/*
3361	 * We can't deal with indirect function calls at present;
3362	 * assume they're instrumented.
3363	 */
3364	if (!func) {
3365		if (file->pv_ops)
3366			return pv_call_dest(file, insn);
3367
3368		return false;
3369	}
3370
3371	/*
3372	 * If the symbol is from a noinstr section; we good.
3373	 */
3374	if (func->sec->noinstr)
3375		return true;
3376
3377	/*
3378	 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3379	 * something 'BAD' happened. At the risk of taking the machine down,
3380	 * let them proceed to get the message out.
3381	 */
3382	if (!strncmp(func->name, "__ubsan_handle_", 15))
3383		return true;
3384
3385	return false;
3386}
3387
3388static int validate_call(struct objtool_file *file,
3389			 struct instruction *insn,
3390			 struct insn_state *state)
3391{
3392	if (state->noinstr && state->instr <= 0 &&
3393	    !noinstr_call_dest(file, insn, insn->call_dest)) {
3394		WARN_FUNC("call to %s() leaves .noinstr.text section",
3395				insn->sec, insn->offset, call_dest_name(insn));
3396		return 1;
3397	}
3398
3399	if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3400		WARN_FUNC("call to %s() with UACCESS enabled",
3401				insn->sec, insn->offset, call_dest_name(insn));
3402		return 1;
3403	}
3404
3405	if (state->df) {
3406		WARN_FUNC("call to %s() with DF set",
3407				insn->sec, insn->offset, call_dest_name(insn));
3408		return 1;
3409	}
3410
3411	return 0;
3412}
3413
3414static int validate_sibling_call(struct objtool_file *file,
3415				 struct instruction *insn,
3416				 struct insn_state *state)
3417{
3418	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3419		WARN_FUNC("sibling call from callable instruction with modified stack frame",
3420				insn->sec, insn->offset);
3421		return 1;
3422	}
3423
3424	return validate_call(file, insn, state);
3425}
3426
3427static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3428{
3429	if (state->noinstr && state->instr > 0) {
3430		WARN_FUNC("return with instrumentation enabled",
3431			  insn->sec, insn->offset);
3432		return 1;
3433	}
3434
3435	if (state->uaccess && !func_uaccess_safe(func)) {
3436		WARN_FUNC("return with UACCESS enabled",
3437			  insn->sec, insn->offset);
3438		return 1;
3439	}
3440
3441	if (!state->uaccess && func_uaccess_safe(func)) {
3442		WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3443			  insn->sec, insn->offset);
3444		return 1;
3445	}
3446
3447	if (state->df) {
3448		WARN_FUNC("return with DF set",
3449			  insn->sec, insn->offset);
3450		return 1;
3451	}
3452
3453	if (func && has_modified_stack_frame(insn, state)) {
3454		WARN_FUNC("return with modified stack frame",
3455			  insn->sec, insn->offset);
3456		return 1;
3457	}
3458
3459	if (state->cfi.bp_scratch) {
3460		WARN_FUNC("BP used as a scratch register",
3461			  insn->sec, insn->offset);
3462		return 1;
3463	}
3464
3465	return 0;
3466}
3467
3468static struct instruction *next_insn_to_validate(struct objtool_file *file,
3469						 struct instruction *insn)
3470{
3471	struct alt_group *alt_group = insn->alt_group;
3472
3473	/*
3474	 * Simulate the fact that alternatives are patched in-place.  When the
3475	 * end of a replacement alt_group is reached, redirect objtool flow to
3476	 * the end of the original alt_group.
3477	 */
3478	if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3479		return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3480
3481	return next_insn_same_sec(file, insn);
3482}
3483
3484/*
3485 * Follow the branch starting at the given instruction, and recursively follow
3486 * any other branches (jumps).  Meanwhile, track the frame pointer state at
3487 * each instruction and validate all the rules described in
3488 * tools/objtool/Documentation/objtool.txt.
3489 */
3490static int validate_branch(struct objtool_file *file, struct symbol *func,
3491			   struct instruction *insn, struct insn_state state)
3492{
3493	struct alternative *alt;
3494	struct instruction *next_insn, *prev_insn = NULL;
3495	struct section *sec;
3496	u8 visited;
3497	int ret;
3498
3499	sec = insn->sec;
3500
3501	while (1) {
3502		next_insn = next_insn_to_validate(file, insn);
3503
3504		if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3505			/* Ignore KCFI type preambles, which always fall through */
3506			if (!strncmp(func->name, "__cfi_", 6) ||
3507			    !strncmp(func->name, "__pfx_", 6))
3508				return 0;
3509
3510			WARN("%s() falls through to next function %s()",
3511			     func->name, insn_func(insn)->name);
3512			return 1;
3513		}
3514
3515		if (func && insn->ignore) {
3516			WARN_FUNC("BUG: why am I validating an ignored function?",
3517				  sec, insn->offset);
3518			return 1;
3519		}
3520
3521		visited = VISITED_BRANCH << state.uaccess;
3522		if (insn->visited & VISITED_BRANCH_MASK) {
3523			if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3524				return 1;
3525
3526			if (insn->visited & visited)
3527				return 0;
3528		} else {
3529			nr_insns_visited++;
3530		}
3531
3532		if (state.noinstr)
3533			state.instr += insn->instr;
3534
3535		if (insn->hint) {
3536			if (insn->restore) {
3537				struct instruction *save_insn, *i;
3538
3539				i = insn;
3540				save_insn = NULL;
3541
3542				sym_for_each_insn_continue_reverse(file, func, i) {
3543					if (i->save) {
3544						save_insn = i;
3545						break;
3546					}
3547				}
3548
3549				if (!save_insn) {
3550					WARN_FUNC("no corresponding CFI save for CFI restore",
3551						  sec, insn->offset);
3552					return 1;
3553				}
3554
3555				if (!save_insn->visited) {
3556					WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
3557						  sec, insn->offset);
3558					return 1;
3559				}
3560
3561				insn->cfi = save_insn->cfi;
3562				nr_cfi_reused++;
3563			}
3564
3565			state.cfi = *insn->cfi;
3566		} else {
3567			/* XXX track if we actually changed state.cfi */
3568
3569			if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3570				insn->cfi = prev_insn->cfi;
3571				nr_cfi_reused++;
3572			} else {
3573				insn->cfi = cfi_hash_find_or_add(&state.cfi);
3574			}
3575		}
3576
3577		insn->visited |= visited;
3578
3579		if (propagate_alt_cfi(file, insn))
3580			return 1;
3581
3582		if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3583			bool skip_orig = false;
3584
3585			list_for_each_entry(alt, &insn->alts, list) {
3586				if (alt->skip_orig)
3587					skip_orig = true;
3588
3589				ret = validate_branch(file, func, alt->insn, state);
3590				if (ret) {
3591					if (opts.backtrace)
3592						BT_FUNC("(alt)", insn);
3593					return ret;
3594				}
3595			}
3596
3597			if (skip_orig)
3598				return 0;
3599		}
3600
3601		if (handle_insn_ops(insn, next_insn, &state))
3602			return 1;
3603
3604		switch (insn->type) {
3605
3606		case INSN_RETURN:
3607			return validate_return(func, insn, &state);
3608
3609		case INSN_CALL:
3610		case INSN_CALL_DYNAMIC:
3611			ret = validate_call(file, insn, &state);
3612			if (ret)
3613				return ret;
3614
3615			if (opts.stackval && func && !is_fentry_call(insn) &&
3616			    !has_valid_stack_frame(&state)) {
3617				WARN_FUNC("call without frame pointer save/setup",
3618					  sec, insn->offset);
3619				return 1;
3620			}
3621
3622			if (insn->dead_end)
3623				return 0;
3624
3625			break;
3626
3627		case INSN_JUMP_CONDITIONAL:
3628		case INSN_JUMP_UNCONDITIONAL:
3629			if (is_sibling_call(insn)) {
3630				ret = validate_sibling_call(file, insn, &state);
3631				if (ret)
3632					return ret;
3633
3634			} else if (insn->jump_dest) {
3635				ret = validate_branch(file, func,
3636						      insn->jump_dest, state);
3637				if (ret) {
3638					if (opts.backtrace)
3639						BT_FUNC("(branch)", insn);
3640					return ret;
3641				}
3642			}
3643
3644			if (insn->type == INSN_JUMP_UNCONDITIONAL)
3645				return 0;
3646
3647			break;
3648
3649		case INSN_JUMP_DYNAMIC:
3650		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3651			if (is_sibling_call(insn)) {
3652				ret = validate_sibling_call(file, insn, &state);
3653				if (ret)
3654					return ret;
3655			}
3656
3657			if (insn->type == INSN_JUMP_DYNAMIC)
3658				return 0;
3659
3660			break;
3661
3662		case INSN_CONTEXT_SWITCH:
3663			if (func && (!next_insn || !next_insn->hint)) {
3664				WARN_FUNC("unsupported instruction in callable function",
3665					  sec, insn->offset);
3666				return 1;
3667			}
3668			return 0;
3669
3670		case INSN_STAC:
3671			if (state.uaccess) {
3672				WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3673				return 1;
3674			}
3675
3676			state.uaccess = true;
3677			break;
3678
3679		case INSN_CLAC:
3680			if (!state.uaccess && func) {
3681				WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3682				return 1;
3683			}
3684
3685			if (func_uaccess_safe(func) && !state.uaccess_stack) {
3686				WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3687				return 1;
3688			}
3689
3690			state.uaccess = false;
3691			break;
3692
3693		case INSN_STD:
3694			if (state.df) {
3695				WARN_FUNC("recursive STD", sec, insn->offset);
3696				return 1;
3697			}
3698
3699			state.df = true;
3700			break;
3701
3702		case INSN_CLD:
3703			if (!state.df && func) {
3704				WARN_FUNC("redundant CLD", sec, insn->offset);
3705				return 1;
3706			}
3707
3708			state.df = false;
3709			break;
3710
3711		default:
3712			break;
3713		}
3714
3715		if (insn->dead_end)
3716			return 0;
3717
3718		if (!next_insn) {
3719			if (state.cfi.cfa.base == CFI_UNDEFINED)
3720				return 0;
3721			WARN("%s: unexpected end of section", sec->name);
3722			return 1;
3723		}
3724
3725		prev_insn = insn;
3726		insn = next_insn;
3727	}
3728
3729	return 0;
3730}
3731
3732static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3733{
3734	struct instruction *insn;
3735	struct insn_state state;
3736	int ret, warnings = 0;
3737
3738	if (!file->hints)
3739		return 0;
3740
3741	init_insn_state(file, &state, sec);
3742
3743	if (sec) {
3744		insn = find_insn(file, sec, 0);
3745		if (!insn)
3746			return 0;
3747	} else {
3748		insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3749	}
3750
3751	while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3752		if (insn->hint && !insn->visited && !insn->ignore) {
3753			ret = validate_branch(file, insn_func(insn), insn, state);
3754			if (ret && opts.backtrace)
3755				BT_FUNC("<=== (hint)", insn);
3756			warnings += ret;
3757		}
3758
3759		insn = list_next_entry(insn, list);
3760	}
3761
3762	return warnings;
3763}
3764
3765/*
3766 * Validate rethunk entry constraint: must untrain RET before the first RET.
3767 *
3768 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
3769 * before an actual RET instruction.
3770 */
3771static int validate_entry(struct objtool_file *file, struct instruction *insn)
3772{
3773	struct instruction *next, *dest;
3774	int ret, warnings = 0;
3775
3776	for (;;) {
3777		next = next_insn_to_validate(file, insn);
3778
3779		if (insn->visited & VISITED_ENTRY)
3780			return 0;
3781
3782		insn->visited |= VISITED_ENTRY;
3783
3784		if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3785			struct alternative *alt;
3786			bool skip_orig = false;
3787
3788			list_for_each_entry(alt, &insn->alts, list) {
3789				if (alt->skip_orig)
3790					skip_orig = true;
3791
3792				ret = validate_entry(file, alt->insn);
3793				if (ret) {
3794				        if (opts.backtrace)
3795						BT_FUNC("(alt)", insn);
3796					return ret;
3797				}
3798			}
3799
3800			if (skip_orig)
3801				return 0;
3802		}
3803
3804		switch (insn->type) {
3805
3806		case INSN_CALL_DYNAMIC:
3807		case INSN_JUMP_DYNAMIC:
3808		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3809			WARN_FUNC("early indirect call", insn->sec, insn->offset);
3810			return 1;
3811
3812		case INSN_JUMP_UNCONDITIONAL:
3813		case INSN_JUMP_CONDITIONAL:
3814			if (!is_sibling_call(insn)) {
3815				if (!insn->jump_dest) {
3816					WARN_FUNC("unresolved jump target after linking?!?",
3817						  insn->sec, insn->offset);
3818					return -1;
3819				}
3820				ret = validate_entry(file, insn->jump_dest);
3821				if (ret) {
3822					if (opts.backtrace) {
3823						BT_FUNC("(branch%s)", insn,
3824							insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3825					}
3826					return ret;
3827				}
3828
3829				if (insn->type == INSN_JUMP_UNCONDITIONAL)
3830					return 0;
3831
3832				break;
3833			}
3834
3835			/* fallthrough */
3836		case INSN_CALL:
3837			dest = find_insn(file, insn->call_dest->sec,
3838					 insn->call_dest->offset);
3839			if (!dest) {
3840				WARN("Unresolved function after linking!?: %s",
3841				     insn->call_dest->name);
3842				return -1;
3843			}
3844
3845			ret = validate_entry(file, dest);
3846			if (ret) {
3847				if (opts.backtrace)
3848					BT_FUNC("(call)", insn);
3849				return ret;
3850			}
3851			/*
3852			 * If a call returns without error, it must have seen UNTRAIN_RET.
3853			 * Therefore any non-error return is a success.
3854			 */
3855			return 0;
3856
3857		case INSN_RETURN:
3858			WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
3859			return 1;
3860
3861		case INSN_NOP:
3862			if (insn->retpoline_safe)
3863				return 0;
3864			break;
3865
3866		default:
3867			break;
3868		}
3869
3870		if (!next) {
3871			WARN_FUNC("teh end!", insn->sec, insn->offset);
3872			return -1;
3873		}
3874		insn = next;
3875	}
3876
3877	return warnings;
3878}
3879
3880/*
3881 * Validate that all branches starting at 'insn->entry' encounter UNRET_END
3882 * before RET.
3883 */
3884static int validate_unret(struct objtool_file *file)
3885{
3886	struct instruction *insn;
3887	int ret, warnings = 0;
3888
3889	for_each_insn(file, insn) {
3890		if (!insn->entry)
3891			continue;
3892
3893		ret = validate_entry(file, insn);
3894		if (ret < 0) {
3895			WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
3896			return ret;
3897		}
3898		warnings += ret;
3899	}
3900
3901	return warnings;
3902}
3903
3904static int validate_retpoline(struct objtool_file *file)
3905{
3906	struct instruction *insn;
3907	int warnings = 0;
3908
3909	for_each_insn(file, insn) {
3910		if (insn->type != INSN_JUMP_DYNAMIC &&
3911		    insn->type != INSN_CALL_DYNAMIC &&
3912		    insn->type != INSN_RETURN)
3913			continue;
3914
3915		if (insn->retpoline_safe)
3916			continue;
3917
3918		if (insn->sec->init)
3919			continue;
3920
3921		if (insn->type == INSN_RETURN) {
3922			if (opts.rethunk) {
3923				WARN_FUNC("'naked' return found in RETHUNK build",
3924					  insn->sec, insn->offset);
3925			} else
3926				continue;
3927		} else {
3928			WARN_FUNC("indirect %s found in RETPOLINE build",
3929				  insn->sec, insn->offset,
3930				  insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3931		}
3932
3933		warnings++;
3934	}
3935
3936	return warnings;
3937}
3938
3939static bool is_kasan_insn(struct instruction *insn)
3940{
3941	return (insn->type == INSN_CALL &&
3942		!strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3943}
3944
3945static bool is_ubsan_insn(struct instruction *insn)
3946{
3947	return (insn->type == INSN_CALL &&
3948		!strcmp(insn->call_dest->name,
3949			"__ubsan_handle_builtin_unreachable"));
3950}
3951
3952static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3953{
3954	int i;
3955	struct instruction *prev_insn;
3956
3957	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3958		return true;
3959
3960	/*
3961	 * Ignore alternative replacement instructions.  This can happen
3962	 * when a whitelisted function uses one of the ALTERNATIVE macros.
3963	 */
3964	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3965	    !strcmp(insn->sec->name, ".altinstr_aux"))
3966		return true;
3967
3968	/*
3969	 * Whole archive runs might encounter dead code from weak symbols.
3970	 * This is where the linker will have dropped the weak symbol in
3971	 * favour of a regular symbol, but leaves the code in place.
3972	 *
3973	 * In this case we'll find a piece of code (whole function) that is not
3974	 * covered by a !section symbol. Ignore them.
3975	 */
3976	if (opts.link && !insn_func(insn)) {
3977		int size = find_symbol_hole_containing(insn->sec, insn->offset);
3978		unsigned long end = insn->offset + size;
3979
3980		if (!size) /* not a hole */
3981			return false;
3982
3983		if (size < 0) /* hole until the end */
3984			return true;
3985
3986		sec_for_each_insn_continue(file, insn) {
3987			/*
3988			 * If we reach a visited instruction at or before the
3989			 * end of the hole, ignore the unreachable.
3990			 */
3991			if (insn->visited)
3992				return true;
3993
3994			if (insn->offset >= end)
3995				break;
3996
3997			/*
3998			 * If this hole jumps to a .cold function, mark it ignore too.
3999			 */
4000			if (insn->jump_dest && insn_func(insn->jump_dest) &&
4001			    strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4002				struct instruction *dest = insn->jump_dest;
4003				func_for_each_insn(file, insn_func(dest), dest)
4004					dest->ignore = true;
4005			}
4006		}
4007
4008		return false;
4009	}
4010
4011	if (!insn_func(insn))
4012		return false;
4013
4014	if (insn_func(insn)->static_call_tramp)
4015		return true;
4016
4017	/*
4018	 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4019	 * __builtin_unreachable().  The BUG() macro has an unreachable() after
4020	 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4021	 * (or occasionally a JMP to UD2).
4022	 *
4023	 * It may also insert a UD2 after calling a __noreturn function.
4024	 */
4025	prev_insn = list_prev_entry(insn, list);
4026	if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
4027	    (insn->type == INSN_BUG ||
4028	     (insn->type == INSN_JUMP_UNCONDITIONAL &&
4029	      insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4030		return true;
4031
4032	/*
4033	 * Check if this (or a subsequent) instruction is related to
4034	 * CONFIG_UBSAN or CONFIG_KASAN.
4035	 *
4036	 * End the search at 5 instructions to avoid going into the weeds.
4037	 */
4038	for (i = 0; i < 5; i++) {
4039
4040		if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4041			return true;
4042
4043		if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4044			if (insn->jump_dest &&
4045			    insn_func(insn->jump_dest) == insn_func(insn)) {
4046				insn = insn->jump_dest;
4047				continue;
4048			}
4049
4050			break;
4051		}
4052
4053		if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4054			break;
4055
4056		insn = list_next_entry(insn, list);
4057	}
4058
4059	return false;
4060}
4061
4062static int add_prefix_symbol(struct objtool_file *file, struct symbol *func,
4063			     struct instruction *insn)
4064{
4065	if (!opts.prefix)
4066		return 0;
4067
4068	for (;;) {
4069		struct instruction *prev = list_prev_entry(insn, list);
4070		u64 offset;
4071
4072		if (&prev->list == &file->insn_list)
4073			break;
4074
4075		if (prev->type != INSN_NOP)
4076			break;
4077
4078		offset = func->offset - prev->offset;
4079		if (offset >= opts.prefix) {
4080			if (offset == opts.prefix) {
4081				/*
4082				 * Since the sec->symbol_list is ordered by
4083				 * offset (see elf_add_symbol()) the added
4084				 * symbol will not be seen by the iteration in
4085				 * validate_section().
4086				 *
4087				 * Hence the lack of list_for_each_entry_safe()
4088				 * there.
4089				 *
4090				 * The direct concequence is that prefix symbols
4091				 * don't get visited (because pointless), except
4092				 * for the logic in ignore_unreachable_insn()
4093				 * that needs the terminating insn to be visited
4094				 * otherwise it will report the hole.
4095				 *
4096				 * Hence mark the first instruction of the
4097				 * prefix symbol as visisted.
4098				 */
4099				prev->visited |= VISITED_BRANCH;
4100				elf_create_prefix_symbol(file->elf, func, opts.prefix);
4101			}
4102			break;
4103		}
4104		insn = prev;
4105	}
4106
4107	return 0;
4108}
4109
4110static int validate_symbol(struct objtool_file *file, struct section *sec,
4111			   struct symbol *sym, struct insn_state *state)
4112{
4113	struct instruction *insn;
4114	int ret;
4115
4116	if (!sym->len) {
4117		WARN("%s() is missing an ELF size annotation", sym->name);
4118		return 1;
4119	}
4120
4121	if (sym->pfunc != sym || sym->alias != sym)
4122		return 0;
4123
4124	insn = find_insn(file, sec, sym->offset);
4125	if (!insn || insn->ignore || insn->visited)
4126		return 0;
4127
4128	add_prefix_symbol(file, sym, insn);
4129
4130	state->uaccess = sym->uaccess_safe;
4131
4132	ret = validate_branch(file, insn_func(insn), insn, *state);
4133	if (ret && opts.backtrace)
4134		BT_FUNC("<=== (sym)", insn);
4135	return ret;
4136}
4137
4138static int validate_section(struct objtool_file *file, struct section *sec)
4139{
4140	struct insn_state state;
4141	struct symbol *func;
4142	int warnings = 0;
4143
4144	list_for_each_entry(func, &sec->symbol_list, list) {
4145		if (func->type != STT_FUNC)
4146			continue;
4147
4148		init_insn_state(file, &state, sec);
4149		set_func_state(&state.cfi);
4150
4151		warnings += validate_symbol(file, sec, func, &state);
4152	}
4153
4154	return warnings;
4155}
4156
4157static int validate_noinstr_sections(struct objtool_file *file)
4158{
4159	struct section *sec;
4160	int warnings = 0;
4161
4162	sec = find_section_by_name(file->elf, ".noinstr.text");
4163	if (sec) {
4164		warnings += validate_section(file, sec);
4165		warnings += validate_unwind_hints(file, sec);
4166	}
4167
4168	sec = find_section_by_name(file->elf, ".entry.text");
4169	if (sec) {
4170		warnings += validate_section(file, sec);
4171		warnings += validate_unwind_hints(file, sec);
4172	}
4173
4174	return warnings;
4175}
4176
4177static int validate_functions(struct objtool_file *file)
4178{
4179	struct section *sec;
4180	int warnings = 0;
4181
4182	for_each_sec(file, sec) {
4183		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4184			continue;
4185
4186		warnings += validate_section(file, sec);
4187	}
4188
4189	return warnings;
4190}
4191
4192static void mark_endbr_used(struct instruction *insn)
4193{
4194	if (!list_empty(&insn->call_node))
4195		list_del_init(&insn->call_node);
4196}
4197
4198static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4199{
4200	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4201	struct instruction *first;
4202
4203	if (!sym)
4204		return false;
4205
4206	first = find_insn(file, sym->sec, sym->offset);
4207	if (!first)
4208		return false;
4209
4210	if (first->type != INSN_ENDBR && !first->noendbr)
4211		return false;
4212
4213	return insn->offset == sym->offset + sym->len;
4214}
4215
4216static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4217{
4218	struct instruction *dest;
4219	struct reloc *reloc;
4220	unsigned long off;
4221	int warnings = 0;
4222
4223	/*
4224	 * Looking for function pointer load relocations.  Ignore
4225	 * direct/indirect branches:
4226	 */
4227	switch (insn->type) {
4228	case INSN_CALL:
4229	case INSN_CALL_DYNAMIC:
4230	case INSN_JUMP_CONDITIONAL:
4231	case INSN_JUMP_UNCONDITIONAL:
4232	case INSN_JUMP_DYNAMIC:
4233	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4234	case INSN_RETURN:
4235	case INSN_NOP:
4236		return 0;
4237	default:
4238		break;
4239	}
4240
4241	for (reloc = insn_reloc(file, insn);
4242	     reloc;
4243	     reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4244					      reloc->offset + 1,
4245					      (insn->offset + insn->len) - (reloc->offset + 1))) {
4246
4247		/*
4248		 * static_call_update() references the trampoline, which
4249		 * doesn't have (or need) ENDBR.  Skip warning in that case.
4250		 */
4251		if (reloc->sym->static_call_tramp)
4252			continue;
4253
4254		off = reloc->sym->offset;
4255		if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
4256			off += arch_dest_reloc_offset(reloc->addend);
4257		else
4258			off += reloc->addend;
4259
4260		dest = find_insn(file, reloc->sym->sec, off);
4261		if (!dest)
4262			continue;
4263
4264		if (dest->type == INSN_ENDBR) {
4265			mark_endbr_used(dest);
4266			continue;
4267		}
4268
4269		if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
4270			/*
4271			 * Anything from->to self is either _THIS_IP_ or
4272			 * IRET-to-self.
4273			 *
4274			 * There is no sane way to annotate _THIS_IP_ since the
4275			 * compiler treats the relocation as a constant and is
4276			 * happy to fold in offsets, skewing any annotation we
4277			 * do, leading to vast amounts of false-positives.
4278			 *
4279			 * There's also compiler generated _THIS_IP_ through
4280			 * KCOV and such which we have no hope of annotating.
4281			 *
4282			 * As such, blanket accept self-references without
4283			 * issue.
4284			 */
4285			continue;
4286		}
4287
4288		/*
4289		 * Accept anything ANNOTATE_NOENDBR.
4290		 */
4291		if (dest->noendbr)
4292			continue;
4293
4294		/*
4295		 * Accept if this is the instruction after a symbol
4296		 * that is (no)endbr -- typical code-range usage.
4297		 */
4298		if (noendbr_range(file, dest))
4299			continue;
4300
4301		WARN_FUNC("relocation to !ENDBR: %s",
4302			  insn->sec, insn->offset,
4303			  offstr(dest->sec, dest->offset));
4304
4305		warnings++;
4306	}
4307
4308	return warnings;
4309}
4310
4311static int validate_ibt_data_reloc(struct objtool_file *file,
4312				   struct reloc *reloc)
4313{
4314	struct instruction *dest;
4315
4316	dest = find_insn(file, reloc->sym->sec,
4317			 reloc->sym->offset + reloc->addend);
4318	if (!dest)
4319		return 0;
4320
4321	if (dest->type == INSN_ENDBR) {
4322		mark_endbr_used(dest);
4323		return 0;
4324	}
4325
4326	if (dest->noendbr)
4327		return 0;
4328
4329	WARN_FUNC("data relocation to !ENDBR: %s",
4330		  reloc->sec->base, reloc->offset,
4331		  offstr(dest->sec, dest->offset));
4332
4333	return 1;
4334}
4335
4336/*
4337 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4338 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4339 * NOPs) later, in create_ibt_endbr_seal_sections().
4340 */
4341static int validate_ibt(struct objtool_file *file)
4342{
4343	struct section *sec;
4344	struct reloc *reloc;
4345	struct instruction *insn;
4346	int warnings = 0;
4347
4348	for_each_insn(file, insn)
4349		warnings += validate_ibt_insn(file, insn);
4350
4351	for_each_sec(file, sec) {
4352
4353		/* Already done by validate_ibt_insn() */
4354		if (sec->sh.sh_flags & SHF_EXECINSTR)
4355			continue;
4356
4357		if (!sec->reloc)
4358			continue;
4359
4360		/*
4361		 * These sections can reference text addresses, but not with
4362		 * the intent to indirect branch to them.
4363		 */
4364		if ((!strncmp(sec->name, ".discard", 8) &&
4365		     strcmp(sec->name, ".discard.ibt_endbr_noseal"))	||
4366		    !strncmp(sec->name, ".debug", 6)			||
4367		    !strcmp(sec->name, ".altinstructions")		||
4368		    !strcmp(sec->name, ".ibt_endbr_seal")		||
4369		    !strcmp(sec->name, ".orc_unwind_ip")		||
4370		    !strcmp(sec->name, ".parainstructions")		||
4371		    !strcmp(sec->name, ".retpoline_sites")		||
4372		    !strcmp(sec->name, ".smp_locks")			||
4373		    !strcmp(sec->name, ".static_call_sites")		||
4374		    !strcmp(sec->name, "_error_injection_whitelist")	||
4375		    !strcmp(sec->name, "_kprobe_blacklist")		||
4376		    !strcmp(sec->name, "__bug_table")			||
4377		    !strcmp(sec->name, "__ex_table")			||
4378		    !strcmp(sec->name, "__jump_table")			||
4379		    !strcmp(sec->name, "__mcount_loc")			||
4380		    !strcmp(sec->name, ".kcfi_traps")			||
4381		    strstr(sec->name, "__patchable_function_entries"))
4382			continue;
4383
4384		list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
4385			warnings += validate_ibt_data_reloc(file, reloc);
4386	}
4387
4388	return warnings;
4389}
4390
4391static int validate_sls(struct objtool_file *file)
4392{
4393	struct instruction *insn, *next_insn;
4394	int warnings = 0;
4395
4396	for_each_insn(file, insn) {
4397		next_insn = next_insn_same_sec(file, insn);
4398
4399		if (insn->retpoline_safe)
4400			continue;
4401
4402		switch (insn->type) {
4403		case INSN_RETURN:
4404			if (!next_insn || next_insn->type != INSN_TRAP) {
4405				WARN_FUNC("missing int3 after ret",
4406					  insn->sec, insn->offset);
4407				warnings++;
4408			}
4409
4410			break;
4411		case INSN_JUMP_DYNAMIC:
4412			if (!next_insn || next_insn->type != INSN_TRAP) {
4413				WARN_FUNC("missing int3 after indirect jump",
4414					  insn->sec, insn->offset);
4415				warnings++;
4416			}
4417			break;
4418		default:
4419			break;
4420		}
4421	}
4422
4423	return warnings;
4424}
4425
4426static int validate_reachable_instructions(struct objtool_file *file)
4427{
4428	struct instruction *insn;
4429
4430	if (file->ignore_unreachables)
4431		return 0;
4432
4433	for_each_insn(file, insn) {
4434		if (insn->visited || ignore_unreachable_insn(file, insn))
4435			continue;
4436
4437		WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
4438		return 1;
4439	}
4440
4441	return 0;
4442}
4443
4444int check(struct objtool_file *file)
4445{
4446	int ret, warnings = 0;
4447
4448	arch_initial_func_cfi_state(&initial_func_cfi);
4449	init_cfi_state(&init_cfi);
4450	init_cfi_state(&func_cfi);
4451	set_func_state(&func_cfi);
4452
4453	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4454		goto out;
4455
4456	cfi_hash_add(&init_cfi);
4457	cfi_hash_add(&func_cfi);
4458
4459	ret = decode_sections(file);
4460	if (ret < 0)
4461		goto out;
4462
4463	warnings += ret;
4464
4465	if (list_empty(&file->insn_list))
4466		goto out;
4467
4468	if (opts.retpoline) {
4469		ret = validate_retpoline(file);
4470		if (ret < 0)
4471			return ret;
4472		warnings += ret;
4473	}
4474
4475	if (opts.stackval || opts.orc || opts.uaccess) {
4476		ret = validate_functions(file);
4477		if (ret < 0)
4478			goto out;
4479		warnings += ret;
4480
4481		ret = validate_unwind_hints(file, NULL);
4482		if (ret < 0)
4483			goto out;
4484		warnings += ret;
4485
4486		if (!warnings) {
4487			ret = validate_reachable_instructions(file);
4488			if (ret < 0)
4489				goto out;
4490			warnings += ret;
4491		}
4492
4493	} else if (opts.noinstr) {
4494		ret = validate_noinstr_sections(file);
4495		if (ret < 0)
4496			goto out;
4497		warnings += ret;
4498	}
4499
4500	if (opts.unret) {
4501		/*
4502		 * Must be after validate_branch() and friends, it plays
4503		 * further games with insn->visited.
4504		 */
4505		ret = validate_unret(file);
4506		if (ret < 0)
4507			return ret;
4508		warnings += ret;
4509	}
4510
4511	if (opts.ibt) {
4512		ret = validate_ibt(file);
4513		if (ret < 0)
4514			goto out;
4515		warnings += ret;
4516	}
4517
4518	if (opts.sls) {
4519		ret = validate_sls(file);
4520		if (ret < 0)
4521			goto out;
4522		warnings += ret;
4523	}
4524
4525	if (opts.static_call) {
4526		ret = create_static_call_sections(file);
4527		if (ret < 0)
4528			goto out;
4529		warnings += ret;
4530	}
4531
4532	if (opts.retpoline) {
4533		ret = create_retpoline_sites_sections(file);
4534		if (ret < 0)
4535			goto out;
4536		warnings += ret;
4537	}
4538
4539	if (opts.cfi) {
4540		ret = create_cfi_sections(file);
4541		if (ret < 0)
4542			goto out;
4543		warnings += ret;
4544	}
4545
4546	if (opts.rethunk) {
4547		ret = create_return_sites_sections(file);
4548		if (ret < 0)
4549			goto out;
4550		warnings += ret;
4551
4552		if (opts.hack_skylake) {
4553			ret = create_direct_call_sections(file);
4554			if (ret < 0)
4555				goto out;
4556			warnings += ret;
4557		}
4558	}
4559
4560	if (opts.mcount) {
4561		ret = create_mcount_loc_sections(file);
4562		if (ret < 0)
4563			goto out;
4564		warnings += ret;
4565	}
4566
4567	if (opts.ibt) {
4568		ret = create_ibt_endbr_seal_sections(file);
4569		if (ret < 0)
4570			goto out;
4571		warnings += ret;
4572	}
4573
4574	if (opts.orc && !list_empty(&file->insn_list)) {
4575		ret = orc_create(file);
4576		if (ret < 0)
4577			goto out;
4578		warnings += ret;
4579	}
4580
4581
4582	if (opts.stats) {
4583		printf("nr_insns_visited: %ld\n", nr_insns_visited);
4584		printf("nr_cfi: %ld\n", nr_cfi);
4585		printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4586		printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4587	}
4588
4589out:
4590	/*
4591	 *  For now, don't fail the kernel build on fatal warnings.  These
4592	 *  errors are still fairly common due to the growing matrix of
4593	 *  supported toolchains and their recent pace of change.
4594	 */
4595	return 0;
4596}