1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4 */
   5
   6#include <string.h>
   7#include <stdlib.h>
   8#include <inttypes.h>
   9#include <sys/mman.h>
  10
  11#include <objtool/builtin.h>
  12#include <objtool/cfi.h>
  13#include <objtool/arch.h>
  14#include <objtool/check.h>
  15#include <objtool/special.h>
  16#include <objtool/warn.h>
  17#include <objtool/endianness.h>
  18
  19#include <linux/objtool_types.h>
  20#include <linux/hashtable.h>
  21#include <linux/kernel.h>
  22#include <linux/static_call_types.h>
  23
  24struct alternative {
  25	struct alternative *next;
  26	struct instruction *insn;
  27	bool skip_orig;
  28};
  29
  30static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  31
  32static struct cfi_init_state initial_func_cfi;
  33static struct cfi_state init_cfi;
  34static struct cfi_state func_cfi;
  35static struct cfi_state force_undefined_cfi;
  36
  37struct instruction *find_insn(struct objtool_file *file,
  38			      struct section *sec, unsigned long offset)
  39{
  40	struct instruction *insn;
  41
  42	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43		if (insn->sec == sec && insn->offset == offset)
  44			return insn;
  45	}
  46
  47	return NULL;
  48}
  49
  50struct instruction *next_insn_same_sec(struct objtool_file *file,
  51				       struct instruction *insn)
  52{
  53	if (insn->idx == INSN_CHUNK_MAX)
  54		return find_insn(file, insn->sec, insn->offset + insn->len);
  55
  56	insn++;
  57	if (!insn->len)
  58		return NULL;
  59
  60	return insn;
  61}
  62
  63static struct instruction *next_insn_same_func(struct objtool_file *file,
  64					       struct instruction *insn)
  65{
  66	struct instruction *next = next_insn_same_sec(file, insn);
  67	struct symbol *func = insn_func(insn);
  68
  69	if (!func)
  70		return NULL;
  71
  72	if (next && insn_func(next) == func)
  73		return next;
  74
  75	/* Check if we're already in the subfunction: */
  76	if (func == func->cfunc)
  77		return NULL;
  78
  79	/* Move to the subfunction: */
  80	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  81}
  82
  83static struct instruction *prev_insn_same_sec(struct objtool_file *file,
  84					      struct instruction *insn)
  85{
  86	if (insn->idx == 0) {
  87		if (insn->prev_len)
  88			return find_insn(file, insn->sec, insn->offset - insn->prev_len);
  89		return NULL;
  90	}
  91
  92	return insn - 1;
  93}
  94
  95static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  96					      struct instruction *insn)
  97{
  98	struct instruction *prev = prev_insn_same_sec(file, insn);
  99
 100	if (prev && insn_func(prev) == insn_func(insn))
 101		return prev;
 102
 103	return NULL;
 104}
 105
 106#define for_each_insn(file, insn)					\
 107	for (struct section *__sec, *__fake = (struct section *)1;	\
 108	     __fake; __fake = NULL)					\
 109		for_each_sec(file, __sec)				\
 110			sec_for_each_insn(file, __sec, insn)
 111
 112#define func_for_each_insn(file, func, insn)				\
 113	for (insn = find_insn(file, func->sec, func->offset);		\
 114	     insn;							\
 115	     insn = next_insn_same_func(file, insn))
 116
 117#define sym_for_each_insn(file, sym, insn)				\
 118	for (insn = find_insn(file, sym->sec, sym->offset);		\
 119	     insn && insn->offset < sym->offset + sym->len;		\
 120	     insn = next_insn_same_sec(file, insn))
 121
 122#define sym_for_each_insn_continue_reverse(file, sym, insn)		\
 123	for (insn = prev_insn_same_sec(file, insn);			\
 124	     insn && insn->offset >= sym->offset;			\
 125	     insn = prev_insn_same_sec(file, insn))
 126
 127#define sec_for_each_insn_from(file, insn)				\
 128	for (; insn; insn = next_insn_same_sec(file, insn))
 129
 130#define sec_for_each_insn_continue(file, insn)				\
 131	for (insn = next_insn_same_sec(file, insn); insn;		\
 132	     insn = next_insn_same_sec(file, insn))
 133
 134static inline struct symbol *insn_call_dest(struct instruction *insn)
 135{
 136	if (insn->type == INSN_JUMP_DYNAMIC ||
 137	    insn->type == INSN_CALL_DYNAMIC)
 138		return NULL;
 139
 140	return insn->_call_dest;
 141}
 142
 143static inline struct reloc *insn_jump_table(struct instruction *insn)
 144{
 145	if (insn->type == INSN_JUMP_DYNAMIC ||
 146	    insn->type == INSN_CALL_DYNAMIC)
 147		return insn->_jump_table;
 148
 149	return NULL;
 150}
 151
 152static bool is_jump_table_jump(struct instruction *insn)
 153{
 154	struct alt_group *alt_group = insn->alt_group;
 155
 156	if (insn_jump_table(insn))
 157		return true;
 158
 159	/* Retpoline alternative for a jump table? */
 160	return alt_group && alt_group->orig_group &&
 161	       insn_jump_table(alt_group->orig_group->first_insn);
 162}
 163
 164static bool is_sibling_call(struct instruction *insn)
 165{
 166	/*
 167	 * Assume only STT_FUNC calls have jump-tables.
 168	 */
 169	if (insn_func(insn)) {
 170		/* An indirect jump is either a sibling call or a jump to a table. */
 171		if (insn->type == INSN_JUMP_DYNAMIC)
 172			return !is_jump_table_jump(insn);
 173	}
 174
 175	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
 176	return (is_static_jump(insn) && insn_call_dest(insn));
 177}
 178
 179/*
 180 * This checks to see if the given function is a "noreturn" function.
 181 *
 182 * For global functions which are outside the scope of this object file, we
 183 * have to keep a manual list of them.
 184 *
 185 * For local functions, we have to detect them manually by simply looking for
 186 * the lack of a return instruction.
 187 */
 188static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 189				int recursion)
 190{
 191	int i;
 192	struct instruction *insn;
 193	bool empty = true;
 194
 195#define NORETURN(func) __stringify(func),
 196	static const char * const global_noreturns[] = {
 197#include "noreturns.h"
 198	};
 199#undef NORETURN
 200
 201	if (!func)
 202		return false;
 203
 204	if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
 205		for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 206			if (!strcmp(func->name, global_noreturns[i]))
 207				return true;
 208
 209	if (func->bind == STB_WEAK)
 210		return false;
 211
 212	if (!func->len)
 213		return false;
 214
 215	insn = find_insn(file, func->sec, func->offset);
 216	if (!insn || !insn_func(insn))
 217		return false;
 218
 219	func_for_each_insn(file, func, insn) {
 220		empty = false;
 221
 222		if (insn->type == INSN_RETURN)
 223			return false;
 224	}
 225
 226	if (empty)
 227		return false;
 228
 229	/*
 230	 * A function can have a sibling call instead of a return.  In that
 231	 * case, the function's dead-end status depends on whether the target
 232	 * of the sibling call returns.
 233	 */
 234	func_for_each_insn(file, func, insn) {
 235		if (is_sibling_call(insn)) {
 236			struct instruction *dest = insn->jump_dest;
 237
 238			if (!dest)
 239				/* sibling call to another file */
 240				return false;
 241
 242			/* local sibling call */
 243			if (recursion == 5) {
 244				/*
 245				 * Infinite recursion: two functions have
 246				 * sibling calls to each other.  This is a very
 247				 * rare case.  It means they aren't dead ends.
 248				 */
 249				return false;
 250			}
 251
 252			return __dead_end_function(file, insn_func(dest), recursion+1);
 253		}
 254	}
 255
 256	return true;
 257}
 258
 259static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 260{
 261	return __dead_end_function(file, func, 0);
 262}
 263
 264static void init_cfi_state(struct cfi_state *cfi)
 265{
 266	int i;
 267
 268	for (i = 0; i < CFI_NUM_REGS; i++) {
 269		cfi->regs[i].base = CFI_UNDEFINED;
 270		cfi->vals[i].base = CFI_UNDEFINED;
 271	}
 272	cfi->cfa.base = CFI_UNDEFINED;
 273	cfi->drap_reg = CFI_UNDEFINED;
 274	cfi->drap_offset = -1;
 275}
 276
 277static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 278			    struct section *sec)
 279{
 280	memset(state, 0, sizeof(*state));
 281	init_cfi_state(&state->cfi);
 282
 283	/*
 284	 * We need the full vmlinux for noinstr validation, otherwise we can
 285	 * not correctly determine insn_call_dest(insn)->sec (external symbols
 286	 * do not have a section).
 287	 */
 288	if (opts.link && opts.noinstr && sec)
 289		state->noinstr = sec->noinstr;
 290}
 291
 292static struct cfi_state *cfi_alloc(void)
 293{
 294	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
 295	if (!cfi) {
 296		WARN("calloc failed");
 297		exit(1);
 298	}
 299	nr_cfi++;
 300	return cfi;
 301}
 302
 303static int cfi_bits;
 304static struct hlist_head *cfi_hash;
 305
 306static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 307{
 308	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 309		      (void *)cfi2 + sizeof(cfi2->hash),
 310		      sizeof(struct cfi_state) - sizeof(struct hlist_node));
 311}
 312
 313static inline u32 cfi_key(struct cfi_state *cfi)
 314{
 315	return jhash((void *)cfi + sizeof(cfi->hash),
 316		     sizeof(*cfi) - sizeof(cfi->hash), 0);
 317}
 318
 319static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 320{
 321	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 322	struct cfi_state *obj;
 323
 324	hlist_for_each_entry(obj, head, hash) {
 325		if (!cficmp(cfi, obj)) {
 326			nr_cfi_cache++;
 327			return obj;
 328		}
 329	}
 330
 331	obj = cfi_alloc();
 332	*obj = *cfi;
 333	hlist_add_head(&obj->hash, head);
 334
 335	return obj;
 336}
 337
 338static void cfi_hash_add(struct cfi_state *cfi)
 339{
 340	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 341
 342	hlist_add_head(&cfi->hash, head);
 343}
 344
 345static void *cfi_hash_alloc(unsigned long size)
 346{
 347	cfi_bits = max(10, ilog2(size));
 348	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 349			PROT_READ|PROT_WRITE,
 350			MAP_PRIVATE|MAP_ANON, -1, 0);
 351	if (cfi_hash == (void *)-1L) {
 352		WARN("mmap fail cfi_hash");
 353		cfi_hash = NULL;
 354	}  else if (opts.stats) {
 355		printf("cfi_bits: %d\n", cfi_bits);
 356	}
 357
 358	return cfi_hash;
 359}
 360
 361static unsigned long nr_insns;
 362static unsigned long nr_insns_visited;
 363
 364/*
 365 * Call the arch-specific instruction decoder for all the instructions and add
 366 * them to the global instruction list.
 367 */
 368static int decode_instructions(struct objtool_file *file)
 369{
 370	struct section *sec;
 371	struct symbol *func;
 372	unsigned long offset;
 373	struct instruction *insn;
 374	int ret;
 375
 376	for_each_sec(file, sec) {
 377		struct instruction *insns = NULL;
 378		u8 prev_len = 0;
 379		u8 idx = 0;
 380
 381		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 382			continue;
 383
 384		if (strcmp(sec->name, ".altinstr_replacement") &&
 385		    strcmp(sec->name, ".altinstr_aux") &&
 386		    strncmp(sec->name, ".discard.", 9))
 387			sec->text = true;
 388
 389		if (!strcmp(sec->name, ".noinstr.text") ||
 390		    !strcmp(sec->name, ".entry.text") ||
 391		    !strcmp(sec->name, ".cpuidle.text") ||
 392		    !strncmp(sec->name, ".text..__x86.", 13))
 393			sec->noinstr = true;
 394
 395		/*
 396		 * .init.text code is ran before userspace and thus doesn't
 397		 * strictly need retpolines, except for modules which are
 398		 * loaded late, they very much do need retpoline in their
 399		 * .init.text
 400		 */
 401		if (!strcmp(sec->name, ".init.text") && !opts.module)
 402			sec->init = true;
 403
 404		for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 405			if (!insns || idx == INSN_CHUNK_MAX) {
 406				insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
 407				if (!insns) {
 408					WARN("malloc failed");
 409					return -1;
 410				}
 411				idx = 0;
 412			} else {
 413				idx++;
 414			}
 415			insn = &insns[idx];
 416			insn->idx = idx;
 417
 418			INIT_LIST_HEAD(&insn->call_node);
 419			insn->sec = sec;
 420			insn->offset = offset;
 421			insn->prev_len = prev_len;
 422
 423			ret = arch_decode_instruction(file, sec, offset,
 424						      sec->sh.sh_size - offset,
 425						      insn);
 426			if (ret)
 427				return ret;
 428
 429			prev_len = insn->len;
 430
 431			/*
 432			 * By default, "ud2" is a dead end unless otherwise
 433			 * annotated, because GCC 7 inserts it for certain
 434			 * divide-by-zero cases.
 435			 */
 436			if (insn->type == INSN_BUG)
 437				insn->dead_end = true;
 438
 439			hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 440			nr_insns++;
 441		}
 442
 443//		printf("%s: last chunk used: %d\n", sec->name, (int)idx);
 444
 445		sec_for_each_sym(sec, func) {
 446			if (func->type != STT_NOTYPE && func->type != STT_FUNC)
 447				continue;
 448
 449			if (func->offset == sec->sh.sh_size) {
 450				/* Heuristic: likely an "end" symbol */
 451				if (func->type == STT_NOTYPE)
 452					continue;
 453				WARN("%s(): STT_FUNC at end of section",
 454				     func->name);
 455				return -1;
 456			}
 457
 458			if (func->embedded_insn || func->alias != func)
 459				continue;
 460
 461			if (!find_insn(file, sec, func->offset)) {
 462				WARN("%s(): can't find starting instruction",
 463				     func->name);
 464				return -1;
 465			}
 466
 467			sym_for_each_insn(file, func, insn) {
 468				insn->sym = func;
 469				if (func->type == STT_FUNC &&
 470				    insn->type == INSN_ENDBR &&
 471				    list_empty(&insn->call_node)) {
 472					if (insn->offset == func->offset) {
 473						list_add_tail(&insn->call_node, &file->endbr_list);
 474						file->nr_endbr++;
 475					} else {
 476						file->nr_endbr_int++;
 477					}
 478				}
 479			}
 480		}
 481	}
 482
 483	if (opts.stats)
 484		printf("nr_insns: %lu\n", nr_insns);
 485
 486	return 0;
 487}
 488
 489/*
 490 * Read the pv_ops[] .data table to find the static initialized values.
 491 */
 492static int add_pv_ops(struct objtool_file *file, const char *symname)
 493{
 494	struct symbol *sym, *func;
 495	unsigned long off, end;
 496	struct reloc *reloc;
 497	int idx;
 498
 499	sym = find_symbol_by_name(file->elf, symname);
 500	if (!sym)
 501		return 0;
 502
 503	off = sym->offset;
 504	end = off + sym->len;
 505	for (;;) {
 506		reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 507		if (!reloc)
 508			break;
 509
 510		func = reloc->sym;
 511		if (func->type == STT_SECTION)
 512			func = find_symbol_by_offset(reloc->sym->sec,
 513						     reloc_addend(reloc));
 514
 515		idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
 516
 517		objtool_pv_add(file, idx, func);
 518
 519		off = reloc_offset(reloc) + 1;
 520		if (off > end)
 521			break;
 522	}
 523
 524	return 0;
 525}
 526
 527/*
 528 * Allocate and initialize file->pv_ops[].
 529 */
 530static int init_pv_ops(struct objtool_file *file)
 531{
 532	static const char *pv_ops_tables[] = {
 533		"pv_ops",
 534		"xen_cpu_ops",
 535		"xen_irq_ops",
 536		"xen_mmu_ops",
 537		NULL,
 538	};
 539	const char *pv_ops;
 540	struct symbol *sym;
 541	int idx, nr;
 542
 543	if (!opts.noinstr)
 544		return 0;
 545
 546	file->pv_ops = NULL;
 547
 548	sym = find_symbol_by_name(file->elf, "pv_ops");
 549	if (!sym)
 550		return 0;
 551
 552	nr = sym->len / sizeof(unsigned long);
 553	file->pv_ops = calloc(sizeof(struct pv_state), nr);
 554	if (!file->pv_ops)
 555		return -1;
 556
 557	for (idx = 0; idx < nr; idx++)
 558		INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 559
 560	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 561		add_pv_ops(file, pv_ops);
 562
 563	return 0;
 564}
 565
 566static struct instruction *find_last_insn(struct objtool_file *file,
 567					  struct section *sec)
 568{
 569	struct instruction *insn = NULL;
 570	unsigned int offset;
 571	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 572
 573	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 574		insn = find_insn(file, sec, offset);
 575
 576	return insn;
 577}
 578
 579/*
 580 * Mark "ud2" instructions and manually annotated dead ends.
 581 */
 582static int add_dead_ends(struct objtool_file *file)
 583{
 584	struct section *rsec;
 585	struct reloc *reloc;
 586	struct instruction *insn;
 587	s64 addend;
 588
 589	/*
 590	 * Check for manually annotated dead ends.
 591	 */
 592	rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 593	if (!rsec)
 594		goto reachable;
 595
 596	for_each_reloc(rsec, reloc) {
 597
 598		if (reloc->sym->type != STT_SECTION) {
 599			WARN("unexpected relocation symbol type in %s", rsec->name);
 600			return -1;
 601		}
 602
 603		addend = reloc_addend(reloc);
 604
 605		insn = find_insn(file, reloc->sym->sec, addend);
 606		if (insn)
 607			insn = prev_insn_same_sec(file, insn);
 608		else if (addend == reloc->sym->sec->sh.sh_size) {
 609			insn = find_last_insn(file, reloc->sym->sec);
 610			if (!insn) {
 611				WARN("can't find unreachable insn at %s+0x%" PRIx64,
 612				     reloc->sym->sec->name, addend);
 613				return -1;
 614			}
 615		} else {
 616			WARN("can't find unreachable insn at %s+0x%" PRIx64,
 617			     reloc->sym->sec->name, addend);
 618			return -1;
 619		}
 620
 621		insn->dead_end = true;
 622	}
 623
 624reachable:
 625	/*
 626	 * These manually annotated reachable checks are needed for GCC 4.4,
 627	 * where the Linux unreachable() macro isn't supported.  In that case
 628	 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 629	 * not a dead end.
 630	 */
 631	rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
 632	if (!rsec)
 633		return 0;
 634
 635	for_each_reloc(rsec, reloc) {
 636
 637		if (reloc->sym->type != STT_SECTION) {
 638			WARN("unexpected relocation symbol type in %s", rsec->name);
 639			return -1;
 640		}
 641
 642		addend = reloc_addend(reloc);
 643
 644		insn = find_insn(file, reloc->sym->sec, addend);
 645		if (insn)
 646			insn = prev_insn_same_sec(file, insn);
 647		else if (addend == reloc->sym->sec->sh.sh_size) {
 648			insn = find_last_insn(file, reloc->sym->sec);
 649			if (!insn) {
 650				WARN("can't find reachable insn at %s+0x%" PRIx64,
 651				     reloc->sym->sec->name, addend);
 652				return -1;
 653			}
 654		} else {
 655			WARN("can't find reachable insn at %s+0x%" PRIx64,
 656			     reloc->sym->sec->name, addend);
 657			return -1;
 658		}
 659
 660		insn->dead_end = false;
 661	}
 662
 663	return 0;
 664}
 665
 666static int create_static_call_sections(struct objtool_file *file)
 667{
 668	struct static_call_site *site;
 669	struct section *sec;
 670	struct instruction *insn;
 671	struct symbol *key_sym;
 672	char *key_name, *tmp;
 673	int idx;
 674
 675	sec = find_section_by_name(file->elf, ".static_call_sites");
 676	if (sec) {
 677		INIT_LIST_HEAD(&file->static_call_list);
 678		WARN("file already has .static_call_sites section, skipping");
 679		return 0;
 680	}
 681
 682	if (list_empty(&file->static_call_list))
 683		return 0;
 684
 685	idx = 0;
 686	list_for_each_entry(insn, &file->static_call_list, call_node)
 687		idx++;
 688
 689	sec = elf_create_section_pair(file->elf, ".static_call_sites",
 690				      sizeof(*site), idx, idx * 2);
 691	if (!sec)
 692		return -1;
 693
 694	/* Allow modules to modify the low bits of static_call_site::key */
 695	sec->sh.sh_flags |= SHF_WRITE;
 696
 697	idx = 0;
 698	list_for_each_entry(insn, &file->static_call_list, call_node) {
 699
 700		/* populate reloc for 'addr' */
 701		if (!elf_init_reloc_text_sym(file->elf, sec,
 702					     idx * sizeof(*site), idx * 2,
 703					     insn->sec, insn->offset))
 704			return -1;
 705
 706		/* find key symbol */
 707		key_name = strdup(insn_call_dest(insn)->name);
 708		if (!key_name) {
 709			perror("strdup");
 710			return -1;
 711		}
 712		if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 713			    STATIC_CALL_TRAMP_PREFIX_LEN)) {
 714			WARN("static_call: trampoline name malformed: %s", key_name);
 715			free(key_name);
 716			return -1;
 717		}
 718		tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 719		memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 720
 721		key_sym = find_symbol_by_name(file->elf, tmp);
 722		if (!key_sym) {
 723			if (!opts.module) {
 724				WARN("static_call: can't find static_call_key symbol: %s", tmp);
 725				free(key_name);
 726				return -1;
 727			}
 728
 729			/*
 730			 * For modules(), the key might not be exported, which
 731			 * means the module can make static calls but isn't
 732			 * allowed to change them.
 733			 *
 734			 * In that case we temporarily set the key to be the
 735			 * trampoline address.  This is fixed up in
 736			 * static_call_add_module().
 737			 */
 738			key_sym = insn_call_dest(insn);
 739		}
 740		free(key_name);
 741
 742		/* populate reloc for 'key' */
 743		if (!elf_init_reloc_data_sym(file->elf, sec,
 744					     idx * sizeof(*site) + 4,
 745					     (idx * 2) + 1, key_sym,
 746					     is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 747			return -1;
 748
 749		idx++;
 750	}
 751
 752	return 0;
 753}
 754
 755static int create_retpoline_sites_sections(struct objtool_file *file)
 756{
 757	struct instruction *insn;
 758	struct section *sec;
 759	int idx;
 760
 761	sec = find_section_by_name(file->elf, ".retpoline_sites");
 762	if (sec) {
 763		WARN("file already has .retpoline_sites, skipping");
 764		return 0;
 765	}
 766
 767	idx = 0;
 768	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 769		idx++;
 770
 771	if (!idx)
 772		return 0;
 773
 774	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
 775				      sizeof(int), idx, idx);
 776	if (!sec)
 777		return -1;
 778
 779	idx = 0;
 780	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 781
 782		if (!elf_init_reloc_text_sym(file->elf, sec,
 783					     idx * sizeof(int), idx,
 784					     insn->sec, insn->offset))
 785			return -1;
 786
 787		idx++;
 788	}
 789
 790	return 0;
 791}
 792
 793static int create_return_sites_sections(struct objtool_file *file)
 794{
 795	struct instruction *insn;
 796	struct section *sec;
 797	int idx;
 798
 799	sec = find_section_by_name(file->elf, ".return_sites");
 800	if (sec) {
 801		WARN("file already has .return_sites, skipping");
 802		return 0;
 803	}
 804
 805	idx = 0;
 806	list_for_each_entry(insn, &file->return_thunk_list, call_node)
 807		idx++;
 808
 809	if (!idx)
 810		return 0;
 811
 812	sec = elf_create_section_pair(file->elf, ".return_sites",
 813				      sizeof(int), idx, idx);
 814	if (!sec)
 815		return -1;
 816
 817	idx = 0;
 818	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 819
 820		if (!elf_init_reloc_text_sym(file->elf, sec,
 821					     idx * sizeof(int), idx,
 822					     insn->sec, insn->offset))
 823			return -1;
 824
 825		idx++;
 826	}
 827
 828	return 0;
 829}
 830
 831static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 832{
 833	struct instruction *insn;
 834	struct section *sec;
 835	int idx;
 836
 837	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 838	if (sec) {
 839		WARN("file already has .ibt_endbr_seal, skipping");
 840		return 0;
 841	}
 842
 843	idx = 0;
 844	list_for_each_entry(insn, &file->endbr_list, call_node)
 845		idx++;
 846
 847	if (opts.stats) {
 848		printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 849		printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 850		printf("ibt: superfluous ENDBR:       %d\n", idx);
 851	}
 852
 853	if (!idx)
 854		return 0;
 855
 856	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
 857				      sizeof(int), idx, idx);
 858	if (!sec)
 859		return -1;
 860
 861	idx = 0;
 862	list_for_each_entry(insn, &file->endbr_list, call_node) {
 863
 864		int *site = (int *)sec->data->d_buf + idx;
 865		struct symbol *sym = insn->sym;
 866		*site = 0;
 867
 868		if (opts.module && sym && sym->type == STT_FUNC &&
 869		    insn->offset == sym->offset &&
 870		    (!strcmp(sym->name, "init_module") ||
 871		     !strcmp(sym->name, "cleanup_module")))
 872			WARN("%s(): not an indirect call target", sym->name);
 873
 874		if (!elf_init_reloc_text_sym(file->elf, sec,
 875					     idx * sizeof(int), idx,
 876					     insn->sec, insn->offset))
 877			return -1;
 878
 879		idx++;
 880	}
 881
 882	return 0;
 883}
 884
 885static int create_cfi_sections(struct objtool_file *file)
 886{
 887	struct section *sec;
 888	struct symbol *sym;
 889	int idx;
 890
 891	sec = find_section_by_name(file->elf, ".cfi_sites");
 892	if (sec) {
 893		INIT_LIST_HEAD(&file->call_list);
 894		WARN("file already has .cfi_sites section, skipping");
 895		return 0;
 896	}
 897
 898	idx = 0;
 899	for_each_sym(file, sym) {
 900		if (sym->type != STT_FUNC)
 901			continue;
 902
 903		if (strncmp(sym->name, "__cfi_", 6))
 904			continue;
 905
 906		idx++;
 907	}
 908
 909	sec = elf_create_section_pair(file->elf, ".cfi_sites",
 910				      sizeof(unsigned int), idx, idx);
 911	if (!sec)
 912		return -1;
 913
 914	idx = 0;
 915	for_each_sym(file, sym) {
 916		if (sym->type != STT_FUNC)
 917			continue;
 918
 919		if (strncmp(sym->name, "__cfi_", 6))
 920			continue;
 921
 922		if (!elf_init_reloc_text_sym(file->elf, sec,
 923					     idx * sizeof(unsigned int), idx,
 924					     sym->sec, sym->offset))
 925			return -1;
 926
 927		idx++;
 928	}
 929
 930	return 0;
 931}
 932
 933static int create_mcount_loc_sections(struct objtool_file *file)
 934{
 935	size_t addr_size = elf_addr_size(file->elf);
 936	struct instruction *insn;
 937	struct section *sec;
 938	int idx;
 939
 940	sec = find_section_by_name(file->elf, "__mcount_loc");
 941	if (sec) {
 942		INIT_LIST_HEAD(&file->mcount_loc_list);
 943		WARN("file already has __mcount_loc section, skipping");
 944		return 0;
 945	}
 946
 947	if (list_empty(&file->mcount_loc_list))
 948		return 0;
 949
 950	idx = 0;
 951	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 952		idx++;
 953
 954	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
 955				      idx, idx);
 956	if (!sec)
 957		return -1;
 958
 959	sec->sh.sh_addralign = addr_size;
 960
 961	idx = 0;
 962	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 963
 964		struct reloc *reloc;
 965
 966		reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
 967					       insn->sec, insn->offset);
 968		if (!reloc)
 969			return -1;
 970
 971		set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
 972
 973		idx++;
 974	}
 975
 976	return 0;
 977}
 978
 979static int create_direct_call_sections(struct objtool_file *file)
 980{
 981	struct instruction *insn;
 982	struct section *sec;
 983	int idx;
 984
 985	sec = find_section_by_name(file->elf, ".call_sites");
 986	if (sec) {
 987		INIT_LIST_HEAD(&file->call_list);
 988		WARN("file already has .call_sites section, skipping");
 989		return 0;
 990	}
 991
 992	if (list_empty(&file->call_list))
 993		return 0;
 994
 995	idx = 0;
 996	list_for_each_entry(insn, &file->call_list, call_node)
 997		idx++;
 998
 999	sec = elf_create_section_pair(file->elf, ".call_sites",
1000				      sizeof(unsigned int), idx, idx);
1001	if (!sec)
1002		return -1;
1003
1004	idx = 0;
1005	list_for_each_entry(insn, &file->call_list, call_node) {
1006
1007		if (!elf_init_reloc_text_sym(file->elf, sec,
1008					     idx * sizeof(unsigned int), idx,
1009					     insn->sec, insn->offset))
1010			return -1;
1011
1012		idx++;
1013	}
1014
1015	return 0;
1016}
1017
1018/*
1019 * Warnings shouldn't be reported for ignored functions.
1020 */
1021static void add_ignores(struct objtool_file *file)
1022{
1023	struct instruction *insn;
1024	struct section *rsec;
1025	struct symbol *func;
1026	struct reloc *reloc;
1027
1028	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1029	if (!rsec)
1030		return;
1031
1032	for_each_reloc(rsec, reloc) {
1033		switch (reloc->sym->type) {
1034		case STT_FUNC:
1035			func = reloc->sym;
1036			break;
1037
1038		case STT_SECTION:
1039			func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1040			if (!func)
1041				continue;
1042			break;
1043
1044		default:
1045			WARN("unexpected relocation symbol type in %s: %d",
1046			     rsec->name, reloc->sym->type);
1047			continue;
1048		}
1049
1050		func_for_each_insn(file, func, insn)
1051			insn->ignore = true;
1052	}
1053}
1054
1055/*
1056 * This is a whitelist of functions that is allowed to be called with AC set.
1057 * The list is meant to be minimal and only contains compiler instrumentation
1058 * ABI and a few functions used to implement *_{to,from}_user() functions.
1059 *
1060 * These functions must not directly change AC, but may PUSHF/POPF.
1061 */
1062static const char *uaccess_safe_builtin[] = {
1063	/* KASAN */
1064	"kasan_report",
1065	"kasan_check_range",
1066	/* KASAN out-of-line */
1067	"__asan_loadN_noabort",
1068	"__asan_load1_noabort",
1069	"__asan_load2_noabort",
1070	"__asan_load4_noabort",
1071	"__asan_load8_noabort",
1072	"__asan_load16_noabort",
1073	"__asan_storeN_noabort",
1074	"__asan_store1_noabort",
1075	"__asan_store2_noabort",
1076	"__asan_store4_noabort",
1077	"__asan_store8_noabort",
1078	"__asan_store16_noabort",
1079	"__kasan_check_read",
1080	"__kasan_check_write",
1081	/* KASAN in-line */
1082	"__asan_report_load_n_noabort",
1083	"__asan_report_load1_noabort",
1084	"__asan_report_load2_noabort",
1085	"__asan_report_load4_noabort",
1086	"__asan_report_load8_noabort",
1087	"__asan_report_load16_noabort",
1088	"__asan_report_store_n_noabort",
1089	"__asan_report_store1_noabort",
1090	"__asan_report_store2_noabort",
1091	"__asan_report_store4_noabort",
1092	"__asan_report_store8_noabort",
1093	"__asan_report_store16_noabort",
1094	/* KCSAN */
1095	"__kcsan_check_access",
1096	"__kcsan_mb",
1097	"__kcsan_wmb",
1098	"__kcsan_rmb",
1099	"__kcsan_release",
1100	"kcsan_found_watchpoint",
1101	"kcsan_setup_watchpoint",
1102	"kcsan_check_scoped_accesses",
1103	"kcsan_disable_current",
1104	"kcsan_enable_current_nowarn",
1105	/* KCSAN/TSAN */
1106	"__tsan_func_entry",
1107	"__tsan_func_exit",
1108	"__tsan_read_range",
1109	"__tsan_write_range",
1110	"__tsan_read1",
1111	"__tsan_read2",
1112	"__tsan_read4",
1113	"__tsan_read8",
1114	"__tsan_read16",
1115	"__tsan_write1",
1116	"__tsan_write2",
1117	"__tsan_write4",
1118	"__tsan_write8",
1119	"__tsan_write16",
1120	"__tsan_read_write1",
1121	"__tsan_read_write2",
1122	"__tsan_read_write4",
1123	"__tsan_read_write8",
1124	"__tsan_read_write16",
1125	"__tsan_volatile_read1",
1126	"__tsan_volatile_read2",
1127	"__tsan_volatile_read4",
1128	"__tsan_volatile_read8",
1129	"__tsan_volatile_read16",
1130	"__tsan_volatile_write1",
1131	"__tsan_volatile_write2",
1132	"__tsan_volatile_write4",
1133	"__tsan_volatile_write8",
1134	"__tsan_volatile_write16",
1135	"__tsan_atomic8_load",
1136	"__tsan_atomic16_load",
1137	"__tsan_atomic32_load",
1138	"__tsan_atomic64_load",
1139	"__tsan_atomic8_store",
1140	"__tsan_atomic16_store",
1141	"__tsan_atomic32_store",
1142	"__tsan_atomic64_store",
1143	"__tsan_atomic8_exchange",
1144	"__tsan_atomic16_exchange",
1145	"__tsan_atomic32_exchange",
1146	"__tsan_atomic64_exchange",
1147	"__tsan_atomic8_fetch_add",
1148	"__tsan_atomic16_fetch_add",
1149	"__tsan_atomic32_fetch_add",
1150	"__tsan_atomic64_fetch_add",
1151	"__tsan_atomic8_fetch_sub",
1152	"__tsan_atomic16_fetch_sub",
1153	"__tsan_atomic32_fetch_sub",
1154	"__tsan_atomic64_fetch_sub",
1155	"__tsan_atomic8_fetch_and",
1156	"__tsan_atomic16_fetch_and",
1157	"__tsan_atomic32_fetch_and",
1158	"__tsan_atomic64_fetch_and",
1159	"__tsan_atomic8_fetch_or",
1160	"__tsan_atomic16_fetch_or",
1161	"__tsan_atomic32_fetch_or",
1162	"__tsan_atomic64_fetch_or",
1163	"__tsan_atomic8_fetch_xor",
1164	"__tsan_atomic16_fetch_xor",
1165	"__tsan_atomic32_fetch_xor",
1166	"__tsan_atomic64_fetch_xor",
1167	"__tsan_atomic8_fetch_nand",
1168	"__tsan_atomic16_fetch_nand",
1169	"__tsan_atomic32_fetch_nand",
1170	"__tsan_atomic64_fetch_nand",
1171	"__tsan_atomic8_compare_exchange_strong",
1172	"__tsan_atomic16_compare_exchange_strong",
1173	"__tsan_atomic32_compare_exchange_strong",
1174	"__tsan_atomic64_compare_exchange_strong",
1175	"__tsan_atomic8_compare_exchange_weak",
1176	"__tsan_atomic16_compare_exchange_weak",
1177	"__tsan_atomic32_compare_exchange_weak",
1178	"__tsan_atomic64_compare_exchange_weak",
1179	"__tsan_atomic8_compare_exchange_val",
1180	"__tsan_atomic16_compare_exchange_val",
1181	"__tsan_atomic32_compare_exchange_val",
1182	"__tsan_atomic64_compare_exchange_val",
1183	"__tsan_atomic_thread_fence",
1184	"__tsan_atomic_signal_fence",
1185	"__tsan_unaligned_read16",
1186	"__tsan_unaligned_write16",
1187	/* KCOV */
1188	"write_comp_data",
1189	"check_kcov_mode",
1190	"__sanitizer_cov_trace_pc",
1191	"__sanitizer_cov_trace_const_cmp1",
1192	"__sanitizer_cov_trace_const_cmp2",
1193	"__sanitizer_cov_trace_const_cmp4",
1194	"__sanitizer_cov_trace_const_cmp8",
1195	"__sanitizer_cov_trace_cmp1",
1196	"__sanitizer_cov_trace_cmp2",
1197	"__sanitizer_cov_trace_cmp4",
1198	"__sanitizer_cov_trace_cmp8",
1199	"__sanitizer_cov_trace_switch",
1200	/* KMSAN */
1201	"kmsan_copy_to_user",
1202	"kmsan_report",
1203	"kmsan_unpoison_entry_regs",
1204	"kmsan_unpoison_memory",
1205	"__msan_chain_origin",
1206	"__msan_get_context_state",
1207	"__msan_instrument_asm_store",
1208	"__msan_metadata_ptr_for_load_1",
1209	"__msan_metadata_ptr_for_load_2",
1210	"__msan_metadata_ptr_for_load_4",
1211	"__msan_metadata_ptr_for_load_8",
1212	"__msan_metadata_ptr_for_load_n",
1213	"__msan_metadata_ptr_for_store_1",
1214	"__msan_metadata_ptr_for_store_2",
1215	"__msan_metadata_ptr_for_store_4",
1216	"__msan_metadata_ptr_for_store_8",
1217	"__msan_metadata_ptr_for_store_n",
1218	"__msan_poison_alloca",
1219	"__msan_warning",
1220	/* UBSAN */
1221	"ubsan_type_mismatch_common",
1222	"__ubsan_handle_type_mismatch",
1223	"__ubsan_handle_type_mismatch_v1",
1224	"__ubsan_handle_shift_out_of_bounds",
1225	"__ubsan_handle_load_invalid_value",
1226	/* STACKLEAK */
1227	"stackleak_track_stack",
1228	/* misc */
1229	"csum_partial_copy_generic",
1230	"copy_mc_fragile",
1231	"copy_mc_fragile_handle_tail",
1232	"copy_mc_enhanced_fast_string",
1233	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1234	"rep_stos_alternative",
1235	"rep_movs_alternative",
1236	"__copy_user_nocache",
1237	NULL
1238};
1239
1240static void add_uaccess_safe(struct objtool_file *file)
1241{
1242	struct symbol *func;
1243	const char **name;
1244
1245	if (!opts.uaccess)
1246		return;
1247
1248	for (name = uaccess_safe_builtin; *name; name++) {
1249		func = find_symbol_by_name(file->elf, *name);
1250		if (!func)
1251			continue;
1252
1253		func->uaccess_safe = true;
1254	}
1255}
1256
1257/*
1258 * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1259 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1260 * But it at least allows objtool to understand the control flow *around* the
1261 * retpoline.
1262 */
1263static int add_ignore_alternatives(struct objtool_file *file)
1264{
1265	struct section *rsec;
1266	struct reloc *reloc;
1267	struct instruction *insn;
1268
1269	rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1270	if (!rsec)
1271		return 0;
1272
1273	for_each_reloc(rsec, reloc) {
1274		if (reloc->sym->type != STT_SECTION) {
1275			WARN("unexpected relocation symbol type in %s", rsec->name);
1276			return -1;
1277		}
1278
1279		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1280		if (!insn) {
1281			WARN("bad .discard.ignore_alts entry");
1282			return -1;
1283		}
1284
1285		insn->ignore_alts = true;
1286	}
1287
1288	return 0;
1289}
1290
1291/*
1292 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1293 * will be added to the .retpoline_sites section.
1294 */
1295__weak bool arch_is_retpoline(struct symbol *sym)
1296{
1297	return false;
1298}
1299
1300/*
1301 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1302 * will be added to the .return_sites section.
1303 */
1304__weak bool arch_is_rethunk(struct symbol *sym)
1305{
1306	return false;
1307}
1308
1309/*
1310 * Symbols that are embedded inside other instructions, because sometimes crazy
1311 * code exists. These are mostly ignored for validation purposes.
1312 */
1313__weak bool arch_is_embedded_insn(struct symbol *sym)
1314{
1315	return false;
1316}
1317
1318static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1319{
1320	struct reloc *reloc;
1321
1322	if (insn->no_reloc)
1323		return NULL;
1324
1325	if (!file)
1326		return NULL;
1327
1328	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1329					 insn->offset, insn->len);
1330	if (!reloc) {
1331		insn->no_reloc = 1;
1332		return NULL;
1333	}
1334
1335	return reloc;
1336}
1337
1338static void remove_insn_ops(struct instruction *insn)
1339{
1340	struct stack_op *op, *next;
1341
1342	for (op = insn->stack_ops; op; op = next) {
1343		next = op->next;
1344		free(op);
1345	}
1346	insn->stack_ops = NULL;
1347}
1348
1349static void annotate_call_site(struct objtool_file *file,
1350			       struct instruction *insn, bool sibling)
1351{
1352	struct reloc *reloc = insn_reloc(file, insn);
1353	struct symbol *sym = insn_call_dest(insn);
1354
1355	if (!sym)
1356		sym = reloc->sym;
1357
1358	/*
1359	 * Alternative replacement code is just template code which is
1360	 * sometimes copied to the original instruction. For now, don't
1361	 * annotate it. (In the future we might consider annotating the
1362	 * original instruction if/when it ever makes sense to do so.)
1363	 */
1364	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1365		return;
1366
1367	if (sym->static_call_tramp) {
1368		list_add_tail(&insn->call_node, &file->static_call_list);
1369		return;
1370	}
1371
1372	if (sym->retpoline_thunk) {
1373		list_add_tail(&insn->call_node, &file->retpoline_call_list);
1374		return;
1375	}
1376
1377	/*
1378	 * Many compilers cannot disable KCOV or sanitizer calls with a function
1379	 * attribute so they need a little help, NOP out any such calls from
1380	 * noinstr text.
1381	 */
1382	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1383		if (reloc)
1384			set_reloc_type(file->elf, reloc, R_NONE);
1385
1386		elf_write_insn(file->elf, insn->sec,
1387			       insn->offset, insn->len,
1388			       sibling ? arch_ret_insn(insn->len)
1389			               : arch_nop_insn(insn->len));
1390
1391		insn->type = sibling ? INSN_RETURN : INSN_NOP;
1392
1393		if (sibling) {
1394			/*
1395			 * We've replaced the tail-call JMP insn by two new
1396			 * insn: RET; INT3, except we only have a single struct
1397			 * insn here. Mark it retpoline_safe to avoid the SLS
1398			 * warning, instead of adding another insn.
1399			 */
1400			insn->retpoline_safe = true;
1401		}
1402
1403		return;
1404	}
1405
1406	if (opts.mcount && sym->fentry) {
1407		if (sibling)
1408			WARN_INSN(insn, "tail call to __fentry__ !?!?");
1409		if (opts.mnop) {
1410			if (reloc)
1411				set_reloc_type(file->elf, reloc, R_NONE);
1412
1413			elf_write_insn(file->elf, insn->sec,
1414				       insn->offset, insn->len,
1415				       arch_nop_insn(insn->len));
1416
1417			insn->type = INSN_NOP;
1418		}
1419
1420		list_add_tail(&insn->call_node, &file->mcount_loc_list);
1421		return;
1422	}
1423
1424	if (insn->type == INSN_CALL && !insn->sec->init)
1425		list_add_tail(&insn->call_node, &file->call_list);
1426
1427	if (!sibling && dead_end_function(file, sym))
1428		insn->dead_end = true;
1429}
1430
1431static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1432			  struct symbol *dest, bool sibling)
1433{
1434	insn->_call_dest = dest;
1435	if (!dest)
1436		return;
1437
1438	/*
1439	 * Whatever stack impact regular CALLs have, should be undone
1440	 * by the RETURN of the called function.
1441	 *
1442	 * Annotated intra-function calls retain the stack_ops but
1443	 * are converted to JUMP, see read_intra_function_calls().
1444	 */
1445	remove_insn_ops(insn);
1446
1447	annotate_call_site(file, insn, sibling);
1448}
1449
1450static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1451{
1452	/*
1453	 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1454	 * so convert them accordingly.
1455	 */
1456	switch (insn->type) {
1457	case INSN_CALL:
1458		insn->type = INSN_CALL_DYNAMIC;
1459		break;
1460	case INSN_JUMP_UNCONDITIONAL:
1461		insn->type = INSN_JUMP_DYNAMIC;
1462		break;
1463	case INSN_JUMP_CONDITIONAL:
1464		insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1465		break;
1466	default:
1467		return;
1468	}
1469
1470	insn->retpoline_safe = true;
1471
1472	/*
1473	 * Whatever stack impact regular CALLs have, should be undone
1474	 * by the RETURN of the called function.
1475	 *
1476	 * Annotated intra-function calls retain the stack_ops but
1477	 * are converted to JUMP, see read_intra_function_calls().
1478	 */
1479	remove_insn_ops(insn);
1480
1481	annotate_call_site(file, insn, false);
1482}
1483
1484static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1485{
1486	/*
1487	 * Return thunk tail calls are really just returns in disguise,
1488	 * so convert them accordingly.
1489	 */
1490	insn->type = INSN_RETURN;
1491	insn->retpoline_safe = true;
1492
1493	if (add)
1494		list_add_tail(&insn->call_node, &file->return_thunk_list);
1495}
1496
1497static bool is_first_func_insn(struct objtool_file *file,
1498			       struct instruction *insn, struct symbol *sym)
1499{
1500	if (insn->offset == sym->offset)
1501		return true;
1502
1503	/* Allow direct CALL/JMP past ENDBR */
1504	if (opts.ibt) {
1505		struct instruction *prev = prev_insn_same_sym(file, insn);
1506
1507		if (prev && prev->type == INSN_ENDBR &&
1508		    insn->offset == sym->offset + prev->len)
1509			return true;
1510	}
1511
1512	return false;
1513}
1514
1515/*
1516 * A sibling call is a tail-call to another symbol -- to differentiate from a
1517 * recursive tail-call which is to the same symbol.
1518 */
1519static bool jump_is_sibling_call(struct objtool_file *file,
1520				 struct instruction *from, struct instruction *to)
1521{
1522	struct symbol *fs = from->sym;
1523	struct symbol *ts = to->sym;
1524
1525	/* Not a sibling call if from/to a symbol hole */
1526	if (!fs || !ts)
1527		return false;
1528
1529	/* Not a sibling call if not targeting the start of a symbol. */
1530	if (!is_first_func_insn(file, to, ts))
1531		return false;
1532
1533	/* Disallow sibling calls into STT_NOTYPE */
1534	if (ts->type == STT_NOTYPE)
1535		return false;
1536
1537	/* Must not be self to be a sibling */
1538	return fs->pfunc != ts->pfunc;
1539}
1540
1541/*
1542 * Find the destination instructions for all jumps.
1543 */
1544static int add_jump_destinations(struct objtool_file *file)
1545{
1546	struct instruction *insn, *jump_dest;
1547	struct reloc *reloc;
1548	struct section *dest_sec;
1549	unsigned long dest_off;
1550
1551	for_each_insn(file, insn) {
1552		if (insn->jump_dest) {
1553			/*
1554			 * handle_group_alt() may have previously set
1555			 * 'jump_dest' for some alternatives.
1556			 */
1557			continue;
1558		}
1559		if (!is_static_jump(insn))
1560			continue;
1561
1562		reloc = insn_reloc(file, insn);
1563		if (!reloc) {
1564			dest_sec = insn->sec;
1565			dest_off = arch_jump_destination(insn);
1566		} else if (reloc->sym->type == STT_SECTION) {
1567			dest_sec = reloc->sym->sec;
1568			dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1569		} else if (reloc->sym->retpoline_thunk) {
1570			add_retpoline_call(file, insn);
1571			continue;
1572		} else if (reloc->sym->return_thunk) {
1573			add_return_call(file, insn, true);
1574			continue;
1575		} else if (insn_func(insn)) {
1576			/*
1577			 * External sibling call or internal sibling call with
1578			 * STT_FUNC reloc.
1579			 */
1580			add_call_dest(file, insn, reloc->sym, true);
1581			continue;
1582		} else if (reloc->sym->sec->idx) {
1583			dest_sec = reloc->sym->sec;
1584			dest_off = reloc->sym->sym.st_value +
1585				   arch_dest_reloc_offset(reloc_addend(reloc));
1586		} else {
1587			/* non-func asm code jumping to another file */
1588			continue;
1589		}
1590
1591		jump_dest = find_insn(file, dest_sec, dest_off);
1592		if (!jump_dest) {
1593			struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1594
1595			/*
1596			 * This is a special case for retbleed_untrain_ret().
1597			 * It jumps to __x86_return_thunk(), but objtool
1598			 * can't find the thunk's starting RET
1599			 * instruction, because the RET is also in the
1600			 * middle of another instruction.  Objtool only
1601			 * knows about the outer instruction.
1602			 */
1603			if (sym && sym->embedded_insn) {
1604				add_return_call(file, insn, false);
1605				continue;
1606			}
1607
1608			WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1609				  dest_sec->name, dest_off);
1610			return -1;
1611		}
1612
1613		/*
1614		 * An intra-TU jump in retpoline.o might not have a relocation
1615		 * for its jump dest, in which case the above
1616		 * add_{retpoline,return}_call() didn't happen.
1617		 */
1618		if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1619			if (jump_dest->sym->retpoline_thunk) {
1620				add_retpoline_call(file, insn);
1621				continue;
1622			}
1623			if (jump_dest->sym->return_thunk) {
1624				add_return_call(file, insn, true);
1625				continue;
1626			}
1627		}
1628
1629		/*
1630		 * Cross-function jump.
1631		 */
1632		if (insn_func(insn) && insn_func(jump_dest) &&
1633		    insn_func(insn) != insn_func(jump_dest)) {
1634
1635			/*
1636			 * For GCC 8+, create parent/child links for any cold
1637			 * subfunctions.  This is _mostly_ redundant with a
1638			 * similar initialization in read_symbols().
1639			 *
1640			 * If a function has aliases, we want the *first* such
1641			 * function in the symbol table to be the subfunction's
1642			 * parent.  In that case we overwrite the
1643			 * initialization done in read_symbols().
1644			 *
1645			 * However this code can't completely replace the
1646			 * read_symbols() code because this doesn't detect the
1647			 * case where the parent function's only reference to a
1648			 * subfunction is through a jump table.
1649			 */
1650			if (!strstr(insn_func(insn)->name, ".cold") &&
1651			    strstr(insn_func(jump_dest)->name, ".cold")) {
1652				insn_func(insn)->cfunc = insn_func(jump_dest);
1653				insn_func(jump_dest)->pfunc = insn_func(insn);
1654			}
1655		}
1656
1657		if (jump_is_sibling_call(file, insn, jump_dest)) {
1658			/*
1659			 * Internal sibling call without reloc or with
1660			 * STT_SECTION reloc.
1661			 */
1662			add_call_dest(file, insn, insn_func(jump_dest), true);
1663			continue;
1664		}
1665
1666		insn->jump_dest = jump_dest;
1667	}
1668
1669	return 0;
1670}
1671
1672static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1673{
1674	struct symbol *call_dest;
1675
1676	call_dest = find_func_by_offset(sec, offset);
1677	if (!call_dest)
1678		call_dest = find_symbol_by_offset(sec, offset);
1679
1680	return call_dest;
1681}
1682
1683/*
1684 * Find the destination instructions for all calls.
1685 */
1686static int add_call_destinations(struct objtool_file *file)
1687{
1688	struct instruction *insn;
1689	unsigned long dest_off;
1690	struct symbol *dest;
1691	struct reloc *reloc;
1692
1693	for_each_insn(file, insn) {
1694		if (insn->type != INSN_CALL)
1695			continue;
1696
1697		reloc = insn_reloc(file, insn);
1698		if (!reloc) {
1699			dest_off = arch_jump_destination(insn);
1700			dest = find_call_destination(insn->sec, dest_off);
1701
1702			add_call_dest(file, insn, dest, false);
1703
1704			if (insn->ignore)
1705				continue;
1706
1707			if (!insn_call_dest(insn)) {
1708				WARN_INSN(insn, "unannotated intra-function call");
1709				return -1;
1710			}
1711
1712			if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1713				WARN_INSN(insn, "unsupported call to non-function");
1714				return -1;
1715			}
1716
1717		} else if (reloc->sym->type == STT_SECTION) {
1718			dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1719			dest = find_call_destination(reloc->sym->sec, dest_off);
1720			if (!dest) {
1721				WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1722					  reloc->sym->sec->name, dest_off);
1723				return -1;
1724			}
1725
1726			add_call_dest(file, insn, dest, false);
1727
1728		} else if (reloc->sym->retpoline_thunk) {
1729			add_retpoline_call(file, insn);
1730
1731		} else
1732			add_call_dest(file, insn, reloc->sym, false);
1733	}
1734
1735	return 0;
1736}
1737
1738/*
1739 * The .alternatives section requires some extra special care over and above
1740 * other special sections because alternatives are patched in place.
1741 */
1742static int handle_group_alt(struct objtool_file *file,
1743			    struct special_alt *special_alt,
1744			    struct instruction *orig_insn,
1745			    struct instruction **new_insn)
1746{
1747	struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1748	struct alt_group *orig_alt_group, *new_alt_group;
1749	unsigned long dest_off;
1750
1751	orig_alt_group = orig_insn->alt_group;
1752	if (!orig_alt_group) {
1753		struct instruction *last_orig_insn = NULL;
1754
1755		orig_alt_group = malloc(sizeof(*orig_alt_group));
1756		if (!orig_alt_group) {
1757			WARN("malloc failed");
1758			return -1;
1759		}
1760		orig_alt_group->cfi = calloc(special_alt->orig_len,
1761					     sizeof(struct cfi_state *));
1762		if (!orig_alt_group->cfi) {
1763			WARN("calloc failed");
1764			return -1;
1765		}
1766
1767		insn = orig_insn;
1768		sec_for_each_insn_from(file, insn) {
1769			if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1770				break;
1771
1772			insn->alt_group = orig_alt_group;
1773			last_orig_insn = insn;
1774		}
1775		orig_alt_group->orig_group = NULL;
1776		orig_alt_group->first_insn = orig_insn;
1777		orig_alt_group->last_insn = last_orig_insn;
1778		orig_alt_group->nop = NULL;
1779	} else {
1780		if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1781		    orig_alt_group->first_insn->offset != special_alt->orig_len) {
1782			WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1783				  orig_alt_group->last_insn->offset +
1784				  orig_alt_group->last_insn->len -
1785				  orig_alt_group->first_insn->offset,
1786				  special_alt->orig_len);
1787			return -1;
1788		}
1789	}
1790
1791	new_alt_group = malloc(sizeof(*new_alt_group));
1792	if (!new_alt_group) {
1793		WARN("malloc failed");
1794		return -1;
1795	}
1796
1797	if (special_alt->new_len < special_alt->orig_len) {
1798		/*
1799		 * Insert a fake nop at the end to make the replacement
1800		 * alt_group the same size as the original.  This is needed to
1801		 * allow propagate_alt_cfi() to do its magic.  When the last
1802		 * instruction affects the stack, the instruction after it (the
1803		 * nop) will propagate the new state to the shared CFI array.
1804		 */
1805		nop = malloc(sizeof(*nop));
1806		if (!nop) {
1807			WARN("malloc failed");
1808			return -1;
1809		}
1810		memset(nop, 0, sizeof(*nop));
1811
1812		nop->sec = special_alt->new_sec;
1813		nop->offset = special_alt->new_off + special_alt->new_len;
1814		nop->len = special_alt->orig_len - special_alt->new_len;
1815		nop->type = INSN_NOP;
1816		nop->sym = orig_insn->sym;
1817		nop->alt_group = new_alt_group;
1818		nop->ignore = orig_insn->ignore_alts;
1819	}
1820
1821	if (!special_alt->new_len) {
1822		*new_insn = nop;
1823		goto end;
1824	}
1825
1826	insn = *new_insn;
1827	sec_for_each_insn_from(file, insn) {
1828		struct reloc *alt_reloc;
1829
1830		if (insn->offset >= special_alt->new_off + special_alt->new_len)
1831			break;
1832
1833		last_new_insn = insn;
1834
1835		insn->ignore = orig_insn->ignore_alts;
1836		insn->sym = orig_insn->sym;
1837		insn->alt_group = new_alt_group;
1838
1839		/*
1840		 * Since alternative replacement code is copy/pasted by the
1841		 * kernel after applying relocations, generally such code can't
1842		 * have relative-address relocation references to outside the
1843		 * .altinstr_replacement section, unless the arch's
1844		 * alternatives code can adjust the relative offsets
1845		 * accordingly.
1846		 */
1847		alt_reloc = insn_reloc(file, insn);
1848		if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1849		    !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1850
1851			WARN_INSN(insn, "unsupported relocation in alternatives section");
1852			return -1;
1853		}
1854
1855		if (!is_static_jump(insn))
1856			continue;
1857
1858		if (!insn->immediate)
1859			continue;
1860
1861		dest_off = arch_jump_destination(insn);
1862		if (dest_off == special_alt->new_off + special_alt->new_len) {
1863			insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1864			if (!insn->jump_dest) {
1865				WARN_INSN(insn, "can't find alternative jump destination");
1866				return -1;
1867			}
1868		}
1869	}
1870
1871	if (!last_new_insn) {
1872		WARN_FUNC("can't find last new alternative instruction",
1873			  special_alt->new_sec, special_alt->new_off);
1874		return -1;
1875	}
1876
1877end:
1878	new_alt_group->orig_group = orig_alt_group;
1879	new_alt_group->first_insn = *new_insn;
1880	new_alt_group->last_insn = last_new_insn;
1881	new_alt_group->nop = nop;
1882	new_alt_group->cfi = orig_alt_group->cfi;
1883	return 0;
1884}
1885
1886/*
1887 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1888 * If the original instruction is a jump, make the alt entry an effective nop
1889 * by just skipping the original instruction.
1890 */
1891static int handle_jump_alt(struct objtool_file *file,
1892			   struct special_alt *special_alt,
1893			   struct instruction *orig_insn,
1894			   struct instruction **new_insn)
1895{
1896	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1897	    orig_insn->type != INSN_NOP) {
1898
1899		WARN_INSN(orig_insn, "unsupported instruction at jump label");
1900		return -1;
1901	}
1902
1903	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1904		struct reloc *reloc = insn_reloc(file, orig_insn);
1905
1906		if (reloc)
1907			set_reloc_type(file->elf, reloc, R_NONE);
1908		elf_write_insn(file->elf, orig_insn->sec,
1909			       orig_insn->offset, orig_insn->len,
1910			       arch_nop_insn(orig_insn->len));
1911		orig_insn->type = INSN_NOP;
1912	}
1913
1914	if (orig_insn->type == INSN_NOP) {
1915		if (orig_insn->len == 2)
1916			file->jl_nop_short++;
1917		else
1918			file->jl_nop_long++;
1919
1920		return 0;
1921	}
1922
1923	if (orig_insn->len == 2)
1924		file->jl_short++;
1925	else
1926		file->jl_long++;
1927
1928	*new_insn = next_insn_same_sec(file, orig_insn);
1929	return 0;
1930}
1931
1932/*
1933 * Read all the special sections which have alternate instructions which can be
1934 * patched in or redirected to at runtime.  Each instruction having alternate
1935 * instruction(s) has them added to its insn->alts list, which will be
1936 * traversed in validate_branch().
1937 */
1938static int add_special_section_alts(struct objtool_file *file)
1939{
1940	struct list_head special_alts;
1941	struct instruction *orig_insn, *new_insn;
1942	struct special_alt *special_alt, *tmp;
1943	struct alternative *alt;
1944	int ret;
1945
1946	ret = special_get_alts(file->elf, &special_alts);
1947	if (ret)
1948		return ret;
1949
1950	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1951
1952		orig_insn = find_insn(file, special_alt->orig_sec,
1953				      special_alt->orig_off);
1954		if (!orig_insn) {
1955			WARN_FUNC("special: can't find orig instruction",
1956				  special_alt->orig_sec, special_alt->orig_off);
1957			ret = -1;
1958			goto out;
1959		}
1960
1961		new_insn = NULL;
1962		if (!special_alt->group || special_alt->new_len) {
1963			new_insn = find_insn(file, special_alt->new_sec,
1964					     special_alt->new_off);
1965			if (!new_insn) {
1966				WARN_FUNC("special: can't find new instruction",
1967					  special_alt->new_sec,
1968					  special_alt->new_off);
1969				ret = -1;
1970				goto out;
1971			}
1972		}
1973
1974		if (special_alt->group) {
1975			if (!special_alt->orig_len) {
1976				WARN_INSN(orig_insn, "empty alternative entry");
1977				continue;
1978			}
1979
1980			ret = handle_group_alt(file, special_alt, orig_insn,
1981					       &new_insn);
1982			if (ret)
1983				goto out;
1984		} else if (special_alt->jump_or_nop) {
1985			ret = handle_jump_alt(file, special_alt, orig_insn,
1986					      &new_insn);
1987			if (ret)
1988				goto out;
1989		}
1990
1991		alt = malloc(sizeof(*alt));
1992		if (!alt) {
1993			WARN("malloc failed");
1994			ret = -1;
1995			goto out;
1996		}
1997
1998		alt->insn = new_insn;
1999		alt->skip_orig = special_alt->skip_orig;
2000		orig_insn->ignore_alts |= special_alt->skip_alt;
2001		alt->next = orig_insn->alts;
2002		orig_insn->alts = alt;
2003
2004		list_del(&special_alt->list);
2005		free(special_alt);
2006	}
2007
2008	if (opts.stats) {
2009		printf("jl\\\tNOP\tJMP\n");
2010		printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2011		printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2012	}
2013
2014out:
2015	return ret;
2016}
2017
2018static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2019			  struct reloc *next_table)
2020{
2021	struct symbol *pfunc = insn_func(insn)->pfunc;
2022	struct reloc *table = insn_jump_table(insn);
2023	struct instruction *dest_insn;
2024	unsigned int prev_offset = 0;
2025	struct reloc *reloc = table;
2026	struct alternative *alt;
2027
2028	/*
2029	 * Each @reloc is a switch table relocation which points to the target
2030	 * instruction.
2031	 */
2032	for_each_reloc_from(table->sec, reloc) {
2033
2034		/* Check for the end of the table: */
2035		if (reloc != table && reloc == next_table)
2036			break;
2037
2038		/* Make sure the table entries are consecutive: */
2039		if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2040			break;
2041
2042		/* Detect function pointers from contiguous objects: */
2043		if (reloc->sym->sec == pfunc->sec &&
2044		    reloc_addend(reloc) == pfunc->offset)
2045			break;
2046
2047		dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2048		if (!dest_insn)
2049			break;
2050
2051		/* Make sure the destination is in the same function: */
2052		if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2053			break;
2054
2055		alt = malloc(sizeof(*alt));
2056		if (!alt) {
2057			WARN("malloc failed");
2058			return -1;
2059		}
2060
2061		alt->insn = dest_insn;
2062		alt->next = insn->alts;
2063		insn->alts = alt;
2064		prev_offset = reloc_offset(reloc);
2065	}
2066
2067	if (!prev_offset) {
2068		WARN_INSN(insn, "can't find switch jump table");
2069		return -1;
2070	}
2071
2072	return 0;
2073}
2074
2075/*
2076 * find_jump_table() - Given a dynamic jump, find the switch jump table
2077 * associated with it.
2078 */
2079static struct reloc *find_jump_table(struct objtool_file *file,
2080				      struct symbol *func,
2081				      struct instruction *insn)
2082{
2083	struct reloc *table_reloc;
2084	struct instruction *dest_insn, *orig_insn = insn;
2085
2086	/*
2087	 * Backward search using the @first_jump_src links, these help avoid
2088	 * much of the 'in between' code. Which avoids us getting confused by
2089	 * it.
2090	 */
2091	for (;
2092	     insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2093	     insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2094
2095		if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2096			break;
2097
2098		/* allow small jumps within the range */
2099		if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2100		    insn->jump_dest &&
2101		    (insn->jump_dest->offset <= insn->offset ||
2102		     insn->jump_dest->offset > orig_insn->offset))
2103		    break;
2104
2105		table_reloc = arch_find_switch_table(file, insn);
2106		if (!table_reloc)
2107			continue;
2108		dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2109		if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2110			continue;
2111
2112		return table_reloc;
2113	}
2114
2115	return NULL;
2116}
2117
2118/*
2119 * First pass: Mark the head of each jump table so that in the next pass,
2120 * we know when a given jump table ends and the next one starts.
2121 */
2122static void mark_func_jump_tables(struct objtool_file *file,
2123				    struct symbol *func)
2124{
2125	struct instruction *insn, *last = NULL;
2126	struct reloc *reloc;
2127
2128	func_for_each_insn(file, func, insn) {
2129		if (!last)
2130			last = insn;
2131
2132		/*
2133		 * Store back-pointers for unconditional forward jumps such
2134		 * that find_jump_table() can back-track using those and
2135		 * avoid some potentially confusing code.
2136		 */
2137		if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2138		    insn->offset > last->offset &&
2139		    insn->jump_dest->offset > insn->offset &&
2140		    !insn->jump_dest->first_jump_src) {
2141
2142			insn->jump_dest->first_jump_src = insn;
2143			last = insn->jump_dest;
2144		}
2145
2146		if (insn->type != INSN_JUMP_DYNAMIC)
2147			continue;
2148
2149		reloc = find_jump_table(file, func, insn);
2150		if (reloc)
2151			insn->_jump_table = reloc;
2152	}
2153}
2154
2155static int add_func_jump_tables(struct objtool_file *file,
2156				  struct symbol *func)
2157{
2158	struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2159	int ret = 0;
2160
2161	func_for_each_insn(file, func, insn) {
2162		if (!insn_jump_table(insn))
2163			continue;
2164
2165		if (!insn_t1) {
2166			insn_t1 = insn;
2167			continue;
2168		}
2169
2170		insn_t2 = insn;
2171
2172		ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2173		if (ret)
2174			return ret;
2175
2176		insn_t1 = insn_t2;
2177	}
2178
2179	if (insn_t1)
2180		ret = add_jump_table(file, insn_t1, NULL);
2181
2182	return ret;
2183}
2184
2185/*
2186 * For some switch statements, gcc generates a jump table in the .rodata
2187 * section which contains a list of addresses within the function to jump to.
2188 * This finds these jump tables and adds them to the insn->alts lists.
2189 */
2190static int add_jump_table_alts(struct objtool_file *file)
2191{
2192	struct symbol *func;
2193	int ret;
2194
2195	if (!file->rodata)
2196		return 0;
2197
2198	for_each_sym(file, func) {
2199		if (func->type != STT_FUNC)
2200			continue;
2201
2202		mark_func_jump_tables(file, func);
2203		ret = add_func_jump_tables(file, func);
2204		if (ret)
2205			return ret;
2206	}
2207
2208	return 0;
2209}
2210
2211static void set_func_state(struct cfi_state *state)
2212{
2213	state->cfa = initial_func_cfi.cfa;
2214	memcpy(&state->regs, &initial_func_cfi.regs,
2215	       CFI_NUM_REGS * sizeof(struct cfi_reg));
2216	state->stack_size = initial_func_cfi.cfa.offset;
2217	state->type = UNWIND_HINT_TYPE_CALL;
2218}
2219
2220static int read_unwind_hints(struct objtool_file *file)
2221{
2222	struct cfi_state cfi = init_cfi;
2223	struct section *sec;
2224	struct unwind_hint *hint;
2225	struct instruction *insn;
2226	struct reloc *reloc;
2227	int i;
2228
2229	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2230	if (!sec)
2231		return 0;
2232
2233	if (!sec->rsec) {
2234		WARN("missing .rela.discard.unwind_hints section");
2235		return -1;
2236	}
2237
2238	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2239		WARN("struct unwind_hint size mismatch");
2240		return -1;
2241	}
2242
2243	file->hints = true;
2244
2245	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2246		hint = (struct unwind_hint *)sec->data->d_buf + i;
2247
2248		reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2249		if (!reloc) {
2250			WARN("can't find reloc for unwind_hints[%d]", i);
2251			return -1;
2252		}
2253
2254		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2255		if (!insn) {
2256			WARN("can't find insn for unwind_hints[%d]", i);
2257			return -1;
2258		}
2259
2260		insn->hint = true;
2261
2262		if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2263			insn->cfi = &force_undefined_cfi;
2264			continue;
2265		}
2266
2267		if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2268			insn->hint = false;
2269			insn->save = true;
2270			continue;
2271		}
2272
2273		if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2274			insn->restore = true;
2275			continue;
2276		}
2277
2278		if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2279			struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2280
2281			if (sym && sym->bind == STB_GLOBAL) {
2282				if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2283					WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2284				}
2285			}
2286		}
2287
2288		if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2289			insn->cfi = &func_cfi;
2290			continue;
2291		}
2292
2293		if (insn->cfi)
2294			cfi = *(insn->cfi);
2295
2296		if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2297			WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2298			return -1;
2299		}
2300
2301		cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2302		cfi.type = hint->type;
2303		cfi.signal = hint->signal;
2304
2305		insn->cfi = cfi_hash_find_or_add(&cfi);
2306	}
2307
2308	return 0;
2309}
2310
2311static int read_noendbr_hints(struct objtool_file *file)
2312{
2313	struct instruction *insn;
2314	struct section *rsec;
2315	struct reloc *reloc;
2316
2317	rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2318	if (!rsec)
2319		return 0;
2320
2321	for_each_reloc(rsec, reloc) {
2322		insn = find_insn(file, reloc->sym->sec,
2323				 reloc->sym->offset + reloc_addend(reloc));
2324		if (!insn) {
2325			WARN("bad .discard.noendbr entry");
2326			return -1;
2327		}
2328
2329		insn->noendbr = 1;
2330	}
2331
2332	return 0;
2333}
2334
2335static int read_retpoline_hints(struct objtool_file *file)
2336{
2337	struct section *rsec;
2338	struct instruction *insn;
2339	struct reloc *reloc;
2340
2341	rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2342	if (!rsec)
2343		return 0;
2344
2345	for_each_reloc(rsec, reloc) {
2346		if (reloc->sym->type != STT_SECTION) {
2347			WARN("unexpected relocation symbol type in %s", rsec->name);
2348			return -1;
2349		}
2350
2351		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2352		if (!insn) {
2353			WARN("bad .discard.retpoline_safe entry");
2354			return -1;
2355		}
2356
2357		if (insn->type != INSN_JUMP_DYNAMIC &&
2358		    insn->type != INSN_CALL_DYNAMIC &&
2359		    insn->type != INSN_RETURN &&
2360		    insn->type != INSN_NOP) {
2361			WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2362			return -1;
2363		}
2364
2365		insn->retpoline_safe = true;
2366	}
2367
2368	return 0;
2369}
2370
2371static int read_instr_hints(struct objtool_file *file)
2372{
2373	struct section *rsec;
2374	struct instruction *insn;
2375	struct reloc *reloc;
2376
2377	rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2378	if (!rsec)
2379		return 0;
2380
2381	for_each_reloc(rsec, reloc) {
2382		if (reloc->sym->type != STT_SECTION) {
2383			WARN("unexpected relocation symbol type in %s", rsec->name);
2384			return -1;
2385		}
2386
2387		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2388		if (!insn) {
2389			WARN("bad .discard.instr_end entry");
2390			return -1;
2391		}
2392
2393		insn->instr--;
2394	}
2395
2396	rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2397	if (!rsec)
2398		return 0;
2399
2400	for_each_reloc(rsec, reloc) {
2401		if (reloc->sym->type != STT_SECTION) {
2402			WARN("unexpected relocation symbol type in %s", rsec->name);
2403			return -1;
2404		}
2405
2406		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2407		if (!insn) {
2408			WARN("bad .discard.instr_begin entry");
2409			return -1;
2410		}
2411
2412		insn->instr++;
2413	}
2414
2415	return 0;
2416}
2417
2418static int read_validate_unret_hints(struct objtool_file *file)
2419{
2420	struct section *rsec;
2421	struct instruction *insn;
2422	struct reloc *reloc;
2423
2424	rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2425	if (!rsec)
2426		return 0;
2427
2428	for_each_reloc(rsec, reloc) {
2429		if (reloc->sym->type != STT_SECTION) {
2430			WARN("unexpected relocation symbol type in %s", rsec->name);
2431			return -1;
2432		}
2433
2434		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2435		if (!insn) {
2436			WARN("bad .discard.instr_end entry");
2437			return -1;
2438		}
2439		insn->unret = 1;
2440	}
2441
2442	return 0;
2443}
2444
2445
2446static int read_intra_function_calls(struct objtool_file *file)
2447{
2448	struct instruction *insn;
2449	struct section *rsec;
2450	struct reloc *reloc;
2451
2452	rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2453	if (!rsec)
2454		return 0;
2455
2456	for_each_reloc(rsec, reloc) {
2457		unsigned long dest_off;
2458
2459		if (reloc->sym->type != STT_SECTION) {
2460			WARN("unexpected relocation symbol type in %s",
2461			     rsec->name);
2462			return -1;
2463		}
2464
2465		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2466		if (!insn) {
2467			WARN("bad .discard.intra_function_call entry");
2468			return -1;
2469		}
2470
2471		if (insn->type != INSN_CALL) {
2472			WARN_INSN(insn, "intra_function_call not a direct call");
2473			return -1;
2474		}
2475
2476		/*
2477		 * Treat intra-function CALLs as JMPs, but with a stack_op.
2478		 * See add_call_destinations(), which strips stack_ops from
2479		 * normal CALLs.
2480		 */
2481		insn->type = INSN_JUMP_UNCONDITIONAL;
2482
2483		dest_off = arch_jump_destination(insn);
2484		insn->jump_dest = find_insn(file, insn->sec, dest_off);
2485		if (!insn->jump_dest) {
2486			WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2487				  insn->sec->name, dest_off);
2488			return -1;
2489		}
2490	}
2491
2492	return 0;
2493}
2494
2495/*
2496 * Return true if name matches an instrumentation function, where calls to that
2497 * function from noinstr code can safely be removed, but compilers won't do so.
2498 */
2499static bool is_profiling_func(const char *name)
2500{
2501	/*
2502	 * Many compilers cannot disable KCOV with a function attribute.
2503	 */
2504	if (!strncmp(name, "__sanitizer_cov_", 16))
2505		return true;
2506
2507	/*
2508	 * Some compilers currently do not remove __tsan_func_entry/exit nor
2509	 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2510	 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2511	 * minimum Clang version is 14.0, this can be removed.
2512	 */
2513	if (!strncmp(name, "__tsan_func_", 12) ||
2514	    !strcmp(name, "__tsan_atomic_signal_fence"))
2515		return true;
2516
2517	return false;
2518}
2519
2520static int classify_symbols(struct objtool_file *file)
2521{
2522	struct symbol *func;
2523
2524	for_each_sym(file, func) {
2525		if (func->bind != STB_GLOBAL)
2526			continue;
2527
2528		if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2529			     strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2530			func->static_call_tramp = true;
2531
2532		if (arch_is_retpoline(func))
2533			func->retpoline_thunk = true;
2534
2535		if (arch_is_rethunk(func))
2536			func->return_thunk = true;
2537
2538		if (arch_is_embedded_insn(func))
2539			func->embedded_insn = true;
2540
2541		if (arch_ftrace_match(func->name))
2542			func->fentry = true;
2543
2544		if (is_profiling_func(func->name))
2545			func->profiling_func = true;
2546	}
2547
2548	return 0;
2549}
2550
2551static void mark_rodata(struct objtool_file *file)
2552{
2553	struct section *sec;
2554	bool found = false;
2555
2556	/*
2557	 * Search for the following rodata sections, each of which can
2558	 * potentially contain jump tables:
2559	 *
2560	 * - .rodata: can contain GCC switch tables
2561	 * - .rodata.<func>: same, if -fdata-sections is being used
2562	 * - .rodata..c_jump_table: contains C annotated jump tables
2563	 *
2564	 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2565	 */
2566	for_each_sec(file, sec) {
2567		if (!strncmp(sec->name, ".rodata", 7) &&
2568		    !strstr(sec->name, ".str1.")) {
2569			sec->rodata = true;
2570			found = true;
2571		}
2572	}
2573
2574	file->rodata = found;
2575}
2576
2577static int decode_sections(struct objtool_file *file)
2578{
2579	int ret;
2580
2581	mark_rodata(file);
2582
2583	ret = init_pv_ops(file);
2584	if (ret)
2585		return ret;
2586
2587	/*
2588	 * Must be before add_{jump_call}_destination.
2589	 */
2590	ret = classify_symbols(file);
2591	if (ret)
2592		return ret;
2593
2594	ret = decode_instructions(file);
2595	if (ret)
2596		return ret;
2597
2598	add_ignores(file);
2599	add_uaccess_safe(file);
2600
2601	ret = add_ignore_alternatives(file);
2602	if (ret)
2603		return ret;
2604
2605	/*
2606	 * Must be before read_unwind_hints() since that needs insn->noendbr.
2607	 */
2608	ret = read_noendbr_hints(file);
2609	if (ret)
2610		return ret;
2611
2612	/*
2613	 * Must be before add_jump_destinations(), which depends on 'func'
2614	 * being set for alternatives, to enable proper sibling call detection.
2615	 */
2616	if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2617		ret = add_special_section_alts(file);
2618		if (ret)
2619			return ret;
2620	}
2621
2622	ret = add_jump_destinations(file);
2623	if (ret)
2624		return ret;
2625
2626	/*
2627	 * Must be before add_call_destination(); it changes INSN_CALL to
2628	 * INSN_JUMP.
2629	 */
2630	ret = read_intra_function_calls(file);
2631	if (ret)
2632		return ret;
2633
2634	ret = add_call_destinations(file);
2635	if (ret)
2636		return ret;
2637
2638	/*
2639	 * Must be after add_call_destinations() such that it can override
2640	 * dead_end_function() marks.
2641	 */
2642	ret = add_dead_ends(file);
2643	if (ret)
2644		return ret;
2645
2646	ret = add_jump_table_alts(file);
2647	if (ret)
2648		return ret;
2649
2650	ret = read_unwind_hints(file);
2651	if (ret)
2652		return ret;
2653
2654	ret = read_retpoline_hints(file);
2655	if (ret)
2656		return ret;
2657
2658	ret = read_instr_hints(file);
2659	if (ret)
2660		return ret;
2661
2662	ret = read_validate_unret_hints(file);
2663	if (ret)
2664		return ret;
2665
2666	return 0;
2667}
2668
2669static bool is_special_call(struct instruction *insn)
2670{
2671	if (insn->type == INSN_CALL) {
2672		struct symbol *dest = insn_call_dest(insn);
2673
2674		if (!dest)
2675			return false;
2676
2677		if (dest->fentry || dest->embedded_insn)
2678			return true;
2679	}
2680
2681	return false;
2682}
2683
2684static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2685{
2686	struct cfi_state *cfi = &state->cfi;
2687	int i;
2688
2689	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2690		return true;
2691
2692	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2693		return true;
2694
2695	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2696		return true;
2697
2698	for (i = 0; i < CFI_NUM_REGS; i++) {
2699		if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2700		    cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2701			return true;
2702	}
2703
2704	return false;
2705}
2706
2707static bool check_reg_frame_pos(const struct cfi_reg *reg,
2708				int expected_offset)
2709{
2710	return reg->base == CFI_CFA &&
2711	       reg->offset == expected_offset;
2712}
2713
2714static bool has_valid_stack_frame(struct insn_state *state)
2715{
2716	struct cfi_state *cfi = &state->cfi;
2717
2718	if (cfi->cfa.base == CFI_BP &&
2719	    check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2720	    check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2721		return true;
2722
2723	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2724		return true;
2725
2726	return false;
2727}
2728
2729static int update_cfi_state_regs(struct instruction *insn,
2730				  struct cfi_state *cfi,
2731				  struct stack_op *op)
2732{
2733	struct cfi_reg *cfa = &cfi->cfa;
2734
2735	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2736		return 0;
2737
2738	/* push */
2739	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2740		cfa->offset += 8;
2741
2742	/* pop */
2743	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2744		cfa->offset -= 8;
2745
2746	/* add immediate to sp */
2747	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2748	    op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2749		cfa->offset -= op->src.offset;
2750
2751	return 0;
2752}
2753
2754static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2755{
2756	if (arch_callee_saved_reg(reg) &&
2757	    cfi->regs[reg].base == CFI_UNDEFINED) {
2758		cfi->regs[reg].base = base;
2759		cfi->regs[reg].offset = offset;
2760	}
2761}
2762
2763static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2764{
2765	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2766	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2767}
2768
2769/*
2770 * A note about DRAP stack alignment:
2771 *
2772 * GCC has the concept of a DRAP register, which is used to help keep track of
2773 * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2774 * register.  The typical DRAP pattern is:
2775 *
2776 *   4c 8d 54 24 08		lea    0x8(%rsp),%r10
2777 *   48 83 e4 c0		and    $0xffffffffffffffc0,%rsp
2778 *   41 ff 72 f8		pushq  -0x8(%r10)
2779 *   55				push   %rbp
2780 *   48 89 e5			mov    %rsp,%rbp
2781 *				(more pushes)
2782 *   41 52			push   %r10
2783 *				...
2784 *   41 5a			pop    %r10
2785 *				(more pops)
2786 *   5d				pop    %rbp
2787 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2788 *   c3				retq
2789 *
2790 * There are some variations in the epilogues, like:
2791 *
2792 *   5b				pop    %rbx
2793 *   41 5a			pop    %r10
2794 *   41 5c			pop    %r12
2795 *   41 5d			pop    %r13
2796 *   41 5e			pop    %r14
2797 *   c9				leaveq
2798 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2799 *   c3				retq
2800 *
2801 * and:
2802 *
2803 *   4c 8b 55 e8		mov    -0x18(%rbp),%r10
2804 *   48 8b 5d e0		mov    -0x20(%rbp),%rbx
2805 *   4c 8b 65 f0		mov    -0x10(%rbp),%r12
2806 *   4c 8b 6d f8		mov    -0x8(%rbp),%r13
2807 *   c9				leaveq
2808 *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2809 *   c3				retq
2810 *
2811 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2812 * restored beforehand:
2813 *
2814 *   41 55			push   %r13
2815 *   4c 8d 6c 24 10		lea    0x10(%rsp),%r13
2816 *   48 83 e4 f0		and    $0xfffffffffffffff0,%rsp
2817 *				...
2818 *   49 8d 65 f0		lea    -0x10(%r13),%rsp
2819 *   41 5d			pop    %r13
2820 *   c3				retq
2821 */
2822static int update_cfi_state(struct instruction *insn,
2823			    struct instruction *next_insn,
2824			    struct cfi_state *cfi, struct stack_op *op)
2825{
2826	struct cfi_reg *cfa = &cfi->cfa;
2827	struct cfi_reg *regs = cfi->regs;
2828
2829	/* ignore UNWIND_HINT_UNDEFINED regions */
2830	if (cfi->force_undefined)
2831		return 0;
2832
2833	/* stack operations don't make sense with an undefined CFA */
2834	if (cfa->base == CFI_UNDEFINED) {
2835		if (insn_func(insn)) {
2836			WARN_INSN(insn, "undefined stack state");
2837			return -1;
2838		}
2839		return 0;
2840	}
2841
2842	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2843	    cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2844		return update_cfi_state_regs(insn, cfi, op);
2845
2846	switch (op->dest.type) {
2847
2848	case OP_DEST_REG:
2849		switch (op->src.type) {
2850
2851		case OP_SRC_REG:
2852			if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2853			    cfa->base == CFI_SP &&
2854			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2855
2856				/* mov %rsp, %rbp */
2857				cfa->base = op->dest.reg;
2858				cfi->bp_scratch = false;
2859			}
2860
2861			else if (op->src.reg == CFI_SP &&
2862				 op->dest.reg == CFI_BP && cfi->drap) {
2863
2864				/* drap: mov %rsp, %rbp */
2865				regs[CFI_BP].base = CFI_BP;
2866				regs[CFI_BP].offset = -cfi->stack_size;
2867				cfi->bp_scratch = false;
2868			}
2869
2870			else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2871
2872				/*
2873				 * mov %rsp, %reg
2874				 *
2875				 * This is needed for the rare case where GCC
2876				 * does:
2877				 *
2878				 *   mov    %rsp, %rax
2879				 *   ...
2880				 *   mov    %rax, %rsp
2881				 */
2882				cfi->vals[op->dest.reg].base = CFI_CFA;
2883				cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2884			}
2885
2886			else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2887				 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2888
2889				/*
2890				 * mov %rbp, %rsp
2891				 *
2892				 * Restore the original stack pointer (Clang).
2893				 */
2894				cfi->stack_size = -cfi->regs[CFI_BP].offset;
2895			}
2896
2897			else if (op->dest.reg == cfa->base) {
2898
2899				/* mov %reg, %rsp */
2900				if (cfa->base == CFI_SP &&
2901				    cfi->vals[op->src.reg].base == CFI_CFA) {
2902
2903					/*
2904					 * This is needed for the rare case
2905					 * where GCC does something dumb like:
2906					 *
2907					 *   lea    0x8(%rsp), %rcx
2908					 *   ...
2909					 *   mov    %rcx, %rsp
2910					 */
2911					cfa->offset = -cfi->vals[op->src.reg].offset;
2912					cfi->stack_size = cfa->offset;
2913
2914				} else if (cfa->base == CFI_SP &&
2915					   cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2916					   cfi->vals[op->src.reg].offset == cfa->offset) {
2917
2918					/*
2919					 * Stack swizzle:
2920					 *
2921					 * 1: mov %rsp, (%[tos])
2922					 * 2: mov %[tos], %rsp
2923					 *    ...
2924					 * 3: pop %rsp
2925					 *
2926					 * Where:
2927					 *
2928					 * 1 - places a pointer to the previous
2929					 *     stack at the Top-of-Stack of the
2930					 *     new stack.
2931					 *
2932					 * 2 - switches to the new stack.
2933					 *
2934					 * 3 - pops the Top-of-Stack to restore
2935					 *     the original stack.
2936					 *
2937					 * Note: we set base to SP_INDIRECT
2938					 * here and preserve offset. Therefore
2939					 * when the unwinder reaches ToS it
2940					 * will dereference SP and then add the
2941					 * offset to find the next frame, IOW:
2942					 * (%rsp) + offset.
2943					 */
2944					cfa->base = CFI_SP_INDIRECT;
2945
2946				} else {
2947					cfa->base = CFI_UNDEFINED;
2948					cfa->offset = 0;
2949				}
2950			}
2951
2952			else if (op->dest.reg == CFI_SP &&
2953				 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2954				 cfi->vals[op->src.reg].offset == cfa->offset) {
2955
2956				/*
2957				 * The same stack swizzle case 2) as above. But
2958				 * because we can't change cfa->base, case 3)
2959				 * will become a regular POP. Pretend we're a
2960				 * PUSH so things don't go unbalanced.
2961				 */
2962				cfi->stack_size += 8;
2963			}
2964
2965
2966			break;
2967
2968		case OP_SRC_ADD:
2969			if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2970
2971				/* add imm, %rsp */
2972				cfi->stack_size -= op->src.offset;
2973				if (cfa->base == CFI_SP)
2974					cfa->offset -= op->src.offset;
2975				break;
2976			}
2977
2978			if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2979
2980				/* lea disp(%rbp), %rsp */
2981				cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2982				break;
2983			}
2984
2985			if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2986
2987				/* drap: lea disp(%rsp), %drap */
2988				cfi->drap_reg = op->dest.reg;
2989
2990				/*
2991				 * lea disp(%rsp), %reg
2992				 *
2993				 * This is needed for the rare case where GCC
2994				 * does something dumb like:
2995				 *
2996				 *   lea    0x8(%rsp), %rcx
2997				 *   ...
2998				 *   mov    %rcx, %rsp
2999				 */
3000				cfi->vals[op->dest.reg].base = CFI_CFA;
3001				cfi->vals[op->dest.reg].offset = \
3002					-cfi->stack_size + op->src.offset;
3003
3004				break;
3005			}
3006
3007			if (cfi->drap && op->dest.reg == CFI_SP &&
3008			    op->src.reg == cfi->drap_reg) {
3009
3010				 /* drap: lea disp(%drap), %rsp */
3011				cfa->base = CFI_SP;
3012				cfa->offset = cfi->stack_size = -op->src.offset;
3013				cfi->drap_reg = CFI_UNDEFINED;
3014				cfi->drap = false;
3015				break;
3016			}
3017
3018			if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3019				WARN_INSN(insn, "unsupported stack register modification");
3020				return -1;
3021			}
3022
3023			break;
3024
3025		case OP_SRC_AND:
3026			if (op->dest.reg != CFI_SP ||
3027			    (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3028			    (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3029				WARN_INSN(insn, "unsupported stack pointer realignment");
3030				return -1;
3031			}
3032
3033			if (cfi->drap_reg != CFI_UNDEFINED) {
3034				/* drap: and imm, %rsp */
3035				cfa->base = cfi->drap_reg;
3036				cfa->offset = cfi->stack_size = 0;
3037				cfi->drap = true;
3038			}
3039
3040			/*
3041			 * Older versions of GCC (4.8ish) realign the stack
3042			 * without DRAP, with a frame pointer.
3043			 */
3044
3045			break;
3046
3047		case OP_SRC_POP:
3048		case OP_SRC_POPF:
3049			if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3050
3051				/* pop %rsp; # restore from a stack swizzle */
3052				cfa->base = CFI_SP;
3053				break;
3054			}
3055
3056			if (!cfi->drap && op->dest.reg == cfa->base) {
3057
3058				/* pop %rbp */
3059				cfa->base = CFI_SP;
3060			}
3061
3062			if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3063			    op->dest.reg == cfi->drap_reg &&
3064			    cfi->drap_offset == -cfi->stack_size) {
3065
3066				/* drap: pop %drap */
3067				cfa->base = cfi->drap_reg;
3068				cfa->offset = 0;
3069				cfi->drap_offset = -1;
3070
3071			} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3072
3073				/* pop %reg */
3074				restore_reg(cfi, op->dest.reg);
3075			}
3076
3077			cfi->stack_size -= 8;
3078			if (cfa->base == CFI_SP)
3079				cfa->offset -= 8;
3080
3081			break;
3082
3083		case OP_SRC_REG_INDIRECT:
3084			if (!cfi->drap && op->dest.reg == cfa->base &&
3085			    op->dest.reg == CFI_BP) {
3086
3087				/* mov disp(%rsp), %rbp */
3088				cfa->base = CFI_SP;
3089				cfa->offset = cfi->stack_size;
3090			}
3091
3092			if (cfi->drap && op->src.reg == CFI_BP &&
3093			    op->src.offset == cfi->drap_offset) {
3094
3095				/* drap: mov disp(%rbp), %drap */
3096				cfa->base = cfi->drap_reg;
3097				cfa->offset = 0;
3098				cfi->drap_offset = -1;
3099			}
3100
3101			if (cfi->drap && op->src.reg == CFI_BP &&
3102			    op->src.offset == regs[op->dest.reg].offset) {
3103
3104				/* drap: mov disp(%rbp), %reg */
3105				restore_reg(cfi, op->dest.reg);
3106
3107			} else if (op->src.reg == cfa->base &&
3108			    op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3109
3110				/* mov disp(%rbp), %reg */
3111				/* mov disp(%rsp), %reg */
3112				restore_reg(cfi, op->dest.reg);
3113
3114			} else if (op->src.reg == CFI_SP &&
3115				   op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3116
3117				/* mov disp(%rsp), %reg */
3118				restore_reg(cfi, op->dest.reg);
3119			}
3120
3121			break;
3122
3123		default:
3124			WARN_INSN(insn, "unknown stack-related instruction");
3125			return -1;
3126		}
3127
3128		break;
3129
3130	case OP_DEST_PUSH:
3131	case OP_DEST_PUSHF:
3132		cfi->stack_size += 8;
3133		if (cfa->base == CFI_SP)
3134			cfa->offset += 8;
3135
3136		if (op->src.type != OP_SRC_REG)
3137			break;
3138
3139		if (cfi->drap) {
3140			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3141
3142				/* drap: push %drap */
3143				cfa->base = CFI_BP_INDIRECT;
3144				cfa->offset = -cfi->stack_size;
3145
3146				/* save drap so we know when to restore it */
3147				cfi->drap_offset = -cfi->stack_size;
3148
3149			} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3150
3151				/* drap: push %rbp */
3152				cfi->stack_size = 0;
3153
3154			} else {
3155
3156				/* drap: push %reg */
3157				save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3158			}
3159
3160		} else {
3161
3162			/* push %reg */
3163			save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3164		}
3165
3166		/* detect when asm code uses rbp as a scratch register */
3167		if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3168		    cfa->base != CFI_BP)
3169			cfi->bp_scratch = true;
3170		break;
3171
3172	case OP_DEST_REG_INDIRECT:
3173
3174		if (cfi->drap) {
3175			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3176
3177				/* drap: mov %drap, disp(%rbp) */
3178				cfa->base = CFI_BP_INDIRECT;
3179				cfa->offset = op->dest.offset;
3180
3181				/* save drap offset so we know when to restore it */
3182				cfi->drap_offset = op->dest.offset;
3183			} else {
3184
3185				/* drap: mov reg, disp(%rbp) */
3186				save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3187			}
3188
3189		} else if (op->dest.reg == cfa->base) {
3190
3191			/* mov reg, disp(%rbp) */
3192			/* mov reg, disp(%rsp) */
3193			save_reg(cfi, op->src.reg, CFI_CFA,
3194				 op->dest.offset - cfi->cfa.offset);
3195
3196		} else if (op->dest.reg == CFI_SP) {
3197
3198			/* mov reg, disp(%rsp) */
3199			save_reg(cfi, op->src.reg, CFI_CFA,
3200				 op->dest.offset - cfi->stack_size);
3201
3202		} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3203
3204			/* mov %rsp, (%reg); # setup a stack swizzle. */
3205			cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3206			cfi->vals[op->dest.reg].offset = cfa->offset;
3207		}
3208
3209		break;
3210
3211	case OP_DEST_MEM:
3212		if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3213			WARN_INSN(insn, "unknown stack-related memory operation");
3214			return -1;
3215		}
3216
3217		/* pop mem */
3218		cfi->stack_size -= 8;
3219		if (cfa->base == CFI_SP)
3220			cfa->offset -= 8;
3221
3222		break;
3223
3224	default:
3225		WARN_INSN(insn, "unknown stack-related instruction");
3226		return -1;
3227	}
3228
3229	return 0;
3230}
3231
3232/*
3233 * The stack layouts of alternatives instructions can sometimes diverge when
3234 * they have stack modifications.  That's fine as long as the potential stack
3235 * layouts don't conflict at any given potential instruction boundary.
3236 *
3237 * Flatten the CFIs of the different alternative code streams (both original
3238 * and replacement) into a single shared CFI array which can be used to detect
3239 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3240 */
3241static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3242{
3243	struct cfi_state **alt_cfi;
3244	int group_off;
3245
3246	if (!insn->alt_group)
3247		return 0;
3248
3249	if (!insn->cfi) {
3250		WARN("CFI missing");
3251		return -1;
3252	}
3253
3254	alt_cfi = insn->alt_group->cfi;
3255	group_off = insn->offset - insn->alt_group->first_insn->offset;
3256
3257	if (!alt_cfi[group_off]) {
3258		alt_cfi[group_off] = insn->cfi;
3259	} else {
3260		if (cficmp(alt_cfi[group_off], insn->cfi)) {
3261			struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3262			struct instruction *orig = orig_group->first_insn;
3263			char *where = offstr(insn->sec, insn->offset);
3264			WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3265			free(where);
3266			return -1;
3267		}
3268	}
3269
3270	return 0;
3271}
3272
3273static int handle_insn_ops(struct instruction *insn,
3274			   struct instruction *next_insn,
3275			   struct insn_state *state)
3276{
3277	struct stack_op *op;
3278
3279	for (op = insn->stack_ops; op; op = op->next) {
3280
3281		if (update_cfi_state(insn, next_insn, &state->cfi, op))
3282			return 1;
3283
3284		if (!insn->alt_group)
3285			continue;
3286
3287		if (op->dest.type == OP_DEST_PUSHF) {
3288			if (!state->uaccess_stack) {
3289				state->uaccess_stack = 1;
3290			} else if (state->uaccess_stack >> 31) {
3291				WARN_INSN(insn, "PUSHF stack exhausted");
3292				return 1;
3293			}
3294			state->uaccess_stack <<= 1;
3295			state->uaccess_stack  |= state->uaccess;
3296		}
3297
3298		if (op->src.type == OP_SRC_POPF) {
3299			if (state->uaccess_stack) {
3300				state->uaccess = state->uaccess_stack & 1;
3301				state->uaccess_stack >>= 1;
3302				if (state->uaccess_stack == 1)
3303					state->uaccess_stack = 0;
3304			}
3305		}
3306	}
3307
3308	return 0;
3309}
3310
3311static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3312{
3313	struct cfi_state *cfi1 = insn->cfi;
3314	int i;
3315
3316	if (!cfi1) {
3317		WARN("CFI missing");
3318		return false;
3319	}
3320
3321	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3322
3323		WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3324			  cfi1->cfa.base, cfi1->cfa.offset,
3325			  cfi2->cfa.base, cfi2->cfa.offset);
3326
3327	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3328		for (i = 0; i < CFI_NUM_REGS; i++) {
3329			if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3330				    sizeof(struct cfi_reg)))
3331				continue;
3332
3333			WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3334				  i, cfi1->regs[i].base, cfi1->regs[i].offset,
3335				  i, cfi2->regs[i].base, cfi2->regs[i].offset);
3336			break;
3337		}
3338
3339	} else if (cfi1->type != cfi2->type) {
3340
3341		WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3342			  cfi1->type, cfi2->type);
3343
3344	} else if (cfi1->drap != cfi2->drap ||
3345		   (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3346		   (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3347
3348		WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3349			  cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3350			  cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3351
3352	} else
3353		return true;
3354
3355	return false;
3356}
3357
3358static inline bool func_uaccess_safe(struct symbol *func)
3359{
3360	if (func)
3361		return func->uaccess_safe;
3362
3363	return false;
3364}
3365
3366static inline const char *call_dest_name(struct instruction *insn)
3367{
3368	static char pvname[19];
3369	struct reloc *reloc;
3370	int idx;
3371
3372	if (insn_call_dest(insn))
3373		return insn_call_dest(insn)->name;
3374
3375	reloc = insn_reloc(NULL, insn);
3376	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3377		idx = (reloc_addend(reloc) / sizeof(void *));
3378		snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3379		return pvname;
3380	}
3381
3382	return "{dynamic}";
3383}
3384
3385static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3386{
3387	struct symbol *target;
3388	struct reloc *reloc;
3389	int idx;
3390
3391	reloc = insn_reloc(file, insn);
3392	if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3393		return false;
3394
3395	idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3396
3397	if (file->pv_ops[idx].clean)
3398		return true;
3399
3400	file->pv_ops[idx].clean = true;
3401
3402	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3403		if (!target->sec->noinstr) {
3404			WARN("pv_ops[%d]: %s", idx, target->name);
3405			file->pv_ops[idx].clean = false;
3406		}
3407	}
3408
3409	return file->pv_ops[idx].clean;
3410}
3411
3412static inline bool noinstr_call_dest(struct objtool_file *file,
3413				     struct instruction *insn,
3414				     struct symbol *func)
3415{
3416	/*
3417	 * We can't deal with indirect function calls at present;
3418	 * assume they're instrumented.
3419	 */
3420	if (!func) {
3421		if (file->pv_ops)
3422			return pv_call_dest(file, insn);
3423
3424		return false;
3425	}
3426
3427	/*
3428	 * If the symbol is from a noinstr section; we good.
3429	 */
3430	if (func->sec->noinstr)
3431		return true;
3432
3433	/*
3434	 * If the symbol is a static_call trampoline, we can't tell.
3435	 */
3436	if (func->static_call_tramp)
3437		return true;
3438
3439	/*
3440	 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3441	 * something 'BAD' happened. At the risk of taking the machine down,
3442	 * let them proceed to get the message out.
3443	 */
3444	if (!strncmp(func->name, "__ubsan_handle_", 15))
3445		return true;
3446
3447	return false;
3448}
3449
3450static int validate_call(struct objtool_file *file,
3451			 struct instruction *insn,
3452			 struct insn_state *state)
3453{
3454	if (state->noinstr && state->instr <= 0 &&
3455	    !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3456		WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3457		return 1;
3458	}
3459
3460	if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3461		WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3462		return 1;
3463	}
3464
3465	if (state->df) {
3466		WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3467		return 1;
3468	}
3469
3470	return 0;
3471}
3472
3473static int validate_sibling_call(struct objtool_file *file,
3474				 struct instruction *insn,
3475				 struct insn_state *state)
3476{
3477	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3478		WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3479		return 1;
3480	}
3481
3482	return validate_call(file, insn, state);
3483}
3484
3485static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3486{
3487	if (state->noinstr && state->instr > 0) {
3488		WARN_INSN(insn, "return with instrumentation enabled");
3489		return 1;
3490	}
3491
3492	if (state->uaccess && !func_uaccess_safe(func)) {
3493		WARN_INSN(insn, "return with UACCESS enabled");
3494		return 1;
3495	}
3496
3497	if (!state->uaccess && func_uaccess_safe(func)) {
3498		WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3499		return 1;
3500	}
3501
3502	if (state->df) {
3503		WARN_INSN(insn, "return with DF set");
3504		return 1;
3505	}
3506
3507	if (func && has_modified_stack_frame(insn, state)) {
3508		WARN_INSN(insn, "return with modified stack frame");
3509		return 1;
3510	}
3511
3512	if (state->cfi.bp_scratch) {
3513		WARN_INSN(insn, "BP used as a scratch register");
3514		return 1;
3515	}
3516
3517	return 0;
3518}
3519
3520static struct instruction *next_insn_to_validate(struct objtool_file *file,
3521						 struct instruction *insn)
3522{
3523	struct alt_group *alt_group = insn->alt_group;
3524
3525	/*
3526	 * Simulate the fact that alternatives are patched in-place.  When the
3527	 * end of a replacement alt_group is reached, redirect objtool flow to
3528	 * the end of the original alt_group.
3529	 *
3530	 * insn->alts->insn -> alt_group->first_insn
3531	 *		       ...
3532	 *		       alt_group->last_insn
3533	 *		       [alt_group->nop]      -> next(orig_group->last_insn)
3534	 */
3535	if (alt_group) {
3536		if (alt_group->nop) {
3537			/* ->nop implies ->orig_group */
3538			if (insn == alt_group->last_insn)
3539				return alt_group->nop;
3540			if (insn == alt_group->nop)
3541				goto next_orig;
3542		}
3543		if (insn == alt_group->last_insn && alt_group->orig_group)
3544			goto next_orig;
3545	}
3546
3547	return next_insn_same_sec(file, insn);
3548
3549next_orig:
3550	return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3551}
3552
3553/*
3554 * Follow the branch starting at the given instruction, and recursively follow
3555 * any other branches (jumps).  Meanwhile, track the frame pointer state at
3556 * each instruction and validate all the rules described in
3557 * tools/objtool/Documentation/objtool.txt.
3558 */
3559static int validate_branch(struct objtool_file *file, struct symbol *func,
3560			   struct instruction *insn, struct insn_state state)
3561{
3562	struct alternative *alt;
3563	struct instruction *next_insn, *prev_insn = NULL;
3564	struct section *sec;
3565	u8 visited;
3566	int ret;
3567
3568	sec = insn->sec;
3569
3570	while (1) {
3571		next_insn = next_insn_to_validate(file, insn);
3572
3573		if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3574			/* Ignore KCFI type preambles, which always fall through */
3575			if (!strncmp(func->name, "__cfi_", 6) ||
3576			    !strncmp(func->name, "__pfx_", 6))
3577				return 0;
3578
3579			WARN("%s() falls through to next function %s()",
3580			     func->name, insn_func(insn)->name);
3581			return 1;
3582		}
3583
3584		if (func && insn->ignore) {
3585			WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3586			return 1;
3587		}
3588
3589		visited = VISITED_BRANCH << state.uaccess;
3590		if (insn->visited & VISITED_BRANCH_MASK) {
3591			if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3592				return 1;
3593
3594			if (insn->visited & visited)
3595				return 0;
3596		} else {
3597			nr_insns_visited++;
3598		}
3599
3600		if (state.noinstr)
3601			state.instr += insn->instr;
3602
3603		if (insn->hint) {
3604			if (insn->restore) {
3605				struct instruction *save_insn, *i;
3606
3607				i = insn;
3608				save_insn = NULL;
3609
3610				sym_for_each_insn_continue_reverse(file, func, i) {
3611					if (i->save) {
3612						save_insn = i;
3613						break;
3614					}
3615				}
3616
3617				if (!save_insn) {
3618					WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3619					return 1;
3620				}
3621
3622				if (!save_insn->visited) {
3623					WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3624					return 1;
3625				}
3626
3627				insn->cfi = save_insn->cfi;
3628				nr_cfi_reused++;
3629			}
3630
3631			state.cfi = *insn->cfi;
3632		} else {
3633			/* XXX track if we actually changed state.cfi */
3634
3635			if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3636				insn->cfi = prev_insn->cfi;
3637				nr_cfi_reused++;
3638			} else {
3639				insn->cfi = cfi_hash_find_or_add(&state.cfi);
3640			}
3641		}
3642
3643		insn->visited |= visited;
3644
3645		if (propagate_alt_cfi(file, insn))
3646			return 1;
3647
3648		if (!insn->ignore_alts && insn->alts) {
3649			bool skip_orig = false;
3650
3651			for (alt = insn->alts; alt; alt = alt->next) {
3652				if (alt->skip_orig)
3653					skip_orig = true;
3654
3655				ret = validate_branch(file, func, alt->insn, state);
3656				if (ret) {
3657					BT_INSN(insn, "(alt)");
3658					return ret;
3659				}
3660			}
3661
3662			if (skip_orig)
3663				return 0;
3664		}
3665
3666		if (handle_insn_ops(insn, next_insn, &state))
3667			return 1;
3668
3669		switch (insn->type) {
3670
3671		case INSN_RETURN:
3672			return validate_return(func, insn, &state);
3673
3674		case INSN_CALL:
3675		case INSN_CALL_DYNAMIC:
3676			ret = validate_call(file, insn, &state);
3677			if (ret)
3678				return ret;
3679
3680			if (opts.stackval && func && !is_special_call(insn) &&
3681			    !has_valid_stack_frame(&state)) {
3682				WARN_INSN(insn, "call without frame pointer save/setup");
3683				return 1;
3684			}
3685
3686			if (insn->dead_end)
3687				return 0;
3688
3689			break;
3690
3691		case INSN_JUMP_CONDITIONAL:
3692		case INSN_JUMP_UNCONDITIONAL:
3693			if (is_sibling_call(insn)) {
3694				ret = validate_sibling_call(file, insn, &state);
3695				if (ret)
3696					return ret;
3697
3698			} else if (insn->jump_dest) {
3699				ret = validate_branch(file, func,
3700						      insn->jump_dest, state);
3701				if (ret) {
3702					BT_INSN(insn, "(branch)");
3703					return ret;
3704				}
3705			}
3706
3707			if (insn->type == INSN_JUMP_UNCONDITIONAL)
3708				return 0;
3709
3710			break;
3711
3712		case INSN_JUMP_DYNAMIC:
3713		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3714			if (is_sibling_call(insn)) {
3715				ret = validate_sibling_call(file, insn, &state);
3716				if (ret)
3717					return ret;
3718			}
3719
3720			if (insn->type == INSN_JUMP_DYNAMIC)
3721				return 0;
3722
3723			break;
3724
3725		case INSN_CONTEXT_SWITCH:
3726			if (func && (!next_insn || !next_insn->hint)) {
3727				WARN_INSN(insn, "unsupported instruction in callable function");
3728				return 1;
3729			}
3730			return 0;
3731
3732		case INSN_STAC:
3733			if (state.uaccess) {
3734				WARN_INSN(insn, "recursive UACCESS enable");
3735				return 1;
3736			}
3737
3738			state.uaccess = true;
3739			break;
3740
3741		case INSN_CLAC:
3742			if (!state.uaccess && func) {
3743				WARN_INSN(insn, "redundant UACCESS disable");
3744				return 1;
3745			}
3746
3747			if (func_uaccess_safe(func) && !state.uaccess_stack) {
3748				WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3749				return 1;
3750			}
3751
3752			state.uaccess = false;
3753			break;
3754
3755		case INSN_STD:
3756			if (state.df) {
3757				WARN_INSN(insn, "recursive STD");
3758				return 1;
3759			}
3760
3761			state.df = true;
3762			break;
3763
3764		case INSN_CLD:
3765			if (!state.df && func) {
3766				WARN_INSN(insn, "redundant CLD");
3767				return 1;
3768			}
3769
3770			state.df = false;
3771			break;
3772
3773		default:
3774			break;
3775		}
3776
3777		if (insn->dead_end)
3778			return 0;
3779
3780		if (!next_insn) {
3781			if (state.cfi.cfa.base == CFI_UNDEFINED)
3782				return 0;
3783			WARN("%s: unexpected end of section", sec->name);
3784			return 1;
3785		}
3786
3787		prev_insn = insn;
3788		insn = next_insn;
3789	}
3790
3791	return 0;
3792}
3793
3794static int validate_unwind_hint(struct objtool_file *file,
3795				  struct instruction *insn,
3796				  struct insn_state *state)
3797{
3798	if (insn->hint && !insn->visited && !insn->ignore) {
3799		int ret = validate_branch(file, insn_func(insn), insn, *state);
3800		if (ret)
3801			BT_INSN(insn, "<=== (hint)");
3802		return ret;
3803	}
3804
3805	return 0;
3806}
3807
3808static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3809{
3810	struct instruction *insn;
3811	struct insn_state state;
3812	int warnings = 0;
3813
3814	if (!file->hints)
3815		return 0;
3816
3817	init_insn_state(file, &state, sec);
3818
3819	if (sec) {
3820		sec_for_each_insn(file, sec, insn)
3821			warnings += validate_unwind_hint(file, insn, &state);
3822	} else {
3823		for_each_insn(file, insn)
3824			warnings += validate_unwind_hint(file, insn, &state);
3825	}
3826
3827	return warnings;
3828}
3829
3830/*
3831 * Validate rethunk entry constraint: must untrain RET before the first RET.
3832 *
3833 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3834 * before an actual RET instruction.
3835 */
3836static int validate_unret(struct objtool_file *file, struct instruction *insn)
3837{
3838	struct instruction *next, *dest;
3839	int ret;
3840
3841	for (;;) {
3842		next = next_insn_to_validate(file, insn);
3843
3844		if (insn->visited & VISITED_UNRET)
3845			return 0;
3846
3847		insn->visited |= VISITED_UNRET;
3848
3849		if (!insn->ignore_alts && insn->alts) {
3850			struct alternative *alt;
3851			bool skip_orig = false;
3852
3853			for (alt = insn->alts; alt; alt = alt->next) {
3854				if (alt->skip_orig)
3855					skip_orig = true;
3856
3857				ret = validate_unret(file, alt->insn);
3858				if (ret) {
3859					BT_INSN(insn, "(alt)");
3860					return ret;
3861				}
3862			}
3863
3864			if (skip_orig)
3865				return 0;
3866		}
3867
3868		switch (insn->type) {
3869
3870		case INSN_CALL_DYNAMIC:
3871		case INSN_JUMP_DYNAMIC:
3872		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3873			WARN_INSN(insn, "early indirect call");
3874			return 1;
3875
3876		case INSN_JUMP_UNCONDITIONAL:
3877		case INSN_JUMP_CONDITIONAL:
3878			if (!is_sibling_call(insn)) {
3879				if (!insn->jump_dest) {
3880					WARN_INSN(insn, "unresolved jump target after linking?!?");
3881					return -1;
3882				}
3883				ret = validate_unret(file, insn->jump_dest);
3884				if (ret) {
3885					BT_INSN(insn, "(branch%s)",
3886						insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3887					return ret;
3888				}
3889
3890				if (insn->type == INSN_JUMP_UNCONDITIONAL)
3891					return 0;
3892
3893				break;
3894			}
3895
3896			/* fallthrough */
3897		case INSN_CALL:
3898			dest = find_insn(file, insn_call_dest(insn)->sec,
3899					 insn_call_dest(insn)->offset);
3900			if (!dest) {
3901				WARN("Unresolved function after linking!?: %s",
3902				     insn_call_dest(insn)->name);
3903				return -1;
3904			}
3905
3906			ret = validate_unret(file, dest);
3907			if (ret) {
3908				BT_INSN(insn, "(call)");
3909				return ret;
3910			}
3911			/*
3912			 * If a call returns without error, it must have seen UNTRAIN_RET.
3913			 * Therefore any non-error return is a success.
3914			 */
3915			return 0;
3916
3917		case INSN_RETURN:
3918			WARN_INSN(insn, "RET before UNTRAIN");
3919			return 1;
3920
3921		case INSN_NOP:
3922			if (insn->retpoline_safe)
3923				return 0;
3924			break;
3925
3926		default:
3927			break;
3928		}
3929
3930		if (!next) {
3931			WARN_INSN(insn, "teh end!");
3932			return -1;
3933		}
3934		insn = next;
3935	}
3936
3937	return 0;
3938}
3939
3940/*
3941 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3942 * VALIDATE_UNRET_END before RET.
3943 */
3944static int validate_unrets(struct objtool_file *file)
3945{
3946	struct instruction *insn;
3947	int ret, warnings = 0;
3948
3949	for_each_insn(file, insn) {
3950		if (!insn->unret)
3951			continue;
3952
3953		ret = validate_unret(file, insn);
3954		if (ret < 0) {
3955			WARN_INSN(insn, "Failed UNRET validation");
3956			return ret;
3957		}
3958		warnings += ret;
3959	}
3960
3961	return warnings;
3962}
3963
3964static int validate_retpoline(struct objtool_file *file)
3965{
3966	struct instruction *insn;
3967	int warnings = 0;
3968
3969	for_each_insn(file, insn) {
3970		if (insn->type != INSN_JUMP_DYNAMIC &&
3971		    insn->type != INSN_CALL_DYNAMIC &&
3972		    insn->type != INSN_RETURN)
3973			continue;
3974
3975		if (insn->retpoline_safe)
3976			continue;
3977
3978		if (insn->sec->init)
3979			continue;
3980
3981		if (insn->type == INSN_RETURN) {
3982			if (opts.rethunk) {
3983				WARN_INSN(insn, "'naked' return found in RETHUNK build");
3984			} else
3985				continue;
3986		} else {
3987			WARN_INSN(insn, "indirect %s found in RETPOLINE build",
3988				  insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3989		}
3990
3991		warnings++;
3992	}
3993
3994	return warnings;
3995}
3996
3997static bool is_kasan_insn(struct instruction *insn)
3998{
3999	return (insn->type == INSN_CALL &&
4000		!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4001}
4002
4003static bool is_ubsan_insn(struct instruction *insn)
4004{
4005	return (insn->type == INSN_CALL &&
4006		!strcmp(insn_call_dest(insn)->name,
4007			"__ubsan_handle_builtin_unreachable"));
4008}
4009
4010static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4011{
4012	int i;
4013	struct instruction *prev_insn;
4014
4015	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4016		return true;
4017
4018	/*
4019	 * Ignore alternative replacement instructions.  This can happen
4020	 * when a whitelisted function uses one of the ALTERNATIVE macros.
4021	 */
4022	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4023	    !strcmp(insn->sec->name, ".altinstr_aux"))
4024		return true;
4025
4026	/*
4027	 * Whole archive runs might encounter dead code from weak symbols.
4028	 * This is where the linker will have dropped the weak symbol in
4029	 * favour of a regular symbol, but leaves the code in place.
4030	 *
4031	 * In this case we'll find a piece of code (whole function) that is not
4032	 * covered by a !section symbol. Ignore them.
4033	 */
4034	if (opts.link && !insn_func(insn)) {
4035		int size = find_symbol_hole_containing(insn->sec, insn->offset);
4036		unsigned long end = insn->offset + size;
4037
4038		if (!size) /* not a hole */
4039			return false;
4040
4041		if (size < 0) /* hole until the end */
4042			return true;
4043
4044		sec_for_each_insn_continue(file, insn) {
4045			/*
4046			 * If we reach a visited instruction at or before the
4047			 * end of the hole, ignore the unreachable.
4048			 */
4049			if (insn->visited)
4050				return true;
4051
4052			if (insn->offset >= end)
4053				break;
4054
4055			/*
4056			 * If this hole jumps to a .cold function, mark it ignore too.
4057			 */
4058			if (insn->jump_dest && insn_func(insn->jump_dest) &&
4059			    strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4060				struct instruction *dest = insn->jump_dest;
4061				func_for_each_insn(file, insn_func(dest), dest)
4062					dest->ignore = true;
4063			}
4064		}
4065
4066		return false;
4067	}
4068
4069	if (!insn_func(insn))
4070		return false;
4071
4072	if (insn_func(insn)->static_call_tramp)
4073		return true;
4074
4075	/*
4076	 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4077	 * __builtin_unreachable().  The BUG() macro has an unreachable() after
4078	 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4079	 * (or occasionally a JMP to UD2).
4080	 *
4081	 * It may also insert a UD2 after calling a __noreturn function.
4082	 */
4083	prev_insn = prev_insn_same_sec(file, insn);
4084	if (prev_insn->dead_end &&
4085	    (insn->type == INSN_BUG ||
4086	     (insn->type == INSN_JUMP_UNCONDITIONAL &&
4087	      insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4088		return true;
4089
4090	/*
4091	 * Check if this (or a subsequent) instruction is related to
4092	 * CONFIG_UBSAN or CONFIG_KASAN.
4093	 *
4094	 * End the search at 5 instructions to avoid going into the weeds.
4095	 */
4096	for (i = 0; i < 5; i++) {
4097
4098		if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4099			return true;
4100
4101		if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4102			if (insn->jump_dest &&
4103			    insn_func(insn->jump_dest) == insn_func(insn)) {
4104				insn = insn->jump_dest;
4105				continue;
4106			}
4107
4108			break;
4109		}
4110
4111		if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4112			break;
4113
4114		insn = next_insn_same_sec(file, insn);
4115	}
4116
4117	return false;
4118}
4119
4120static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4121{
4122	struct instruction *insn, *prev;
4123	struct cfi_state *cfi;
4124
4125	insn = find_insn(file, func->sec, func->offset);
4126	if (!insn)
4127		return -1;
4128
4129	for (prev = prev_insn_same_sec(file, insn);
4130	     prev;
4131	     prev = prev_insn_same_sec(file, prev)) {
4132		u64 offset;
4133
4134		if (prev->type != INSN_NOP)
4135			return -1;
4136
4137		offset = func->offset - prev->offset;
4138
4139		if (offset > opts.prefix)
4140			return -1;
4141
4142		if (offset < opts.prefix)
4143			continue;
4144
4145		elf_create_prefix_symbol(file->elf, func, opts.prefix);
4146		break;
4147	}
4148
4149	if (!prev)
4150		return -1;
4151
4152	if (!insn->cfi) {
4153		/*
4154		 * This can happen if stack validation isn't enabled or the
4155		 * function is annotated with STACK_FRAME_NON_STANDARD.
4156		 */
4157		return 0;
4158	}
4159
4160	/* Propagate insn->cfi to the prefix code */
4161	cfi = cfi_hash_find_or_add(insn->cfi);
4162	for (; prev != insn; prev = next_insn_same_sec(file, prev))
4163		prev->cfi = cfi;
4164
4165	return 0;
4166}
4167
4168static int add_prefix_symbols(struct objtool_file *file)
4169{
4170	struct section *sec;
4171	struct symbol *func;
4172
4173	for_each_sec(file, sec) {
4174		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4175			continue;
4176
4177		sec_for_each_sym(sec, func) {
4178			if (func->type != STT_FUNC)
4179				continue;
4180
4181			add_prefix_symbol(file, func);
4182		}
4183	}
4184
4185	return 0;
4186}
4187
4188static int validate_symbol(struct objtool_file *file, struct section *sec,
4189			   struct symbol *sym, struct insn_state *state)
4190{
4191	struct instruction *insn;
4192	int ret;
4193
4194	if (!sym->len) {
4195		WARN("%s() is missing an ELF size annotation", sym->name);
4196		return 1;
4197	}
4198
4199	if (sym->pfunc != sym || sym->alias != sym)
4200		return 0;
4201
4202	insn = find_insn(file, sec, sym->offset);
4203	if (!insn || insn->ignore || insn->visited)
4204		return 0;
4205
4206	state->uaccess = sym->uaccess_safe;
4207
4208	ret = validate_branch(file, insn_func(insn), insn, *state);
4209	if (ret)
4210		BT_INSN(insn, "<=== (sym)");
4211	return ret;
4212}
4213
4214static int validate_section(struct objtool_file *file, struct section *sec)
4215{
4216	struct insn_state state;
4217	struct symbol *func;
4218	int warnings = 0;
4219
4220	sec_for_each_sym(sec, func) {
4221		if (func->type != STT_FUNC)
4222			continue;
4223
4224		init_insn_state(file, &state, sec);
4225		set_func_state(&state.cfi);
4226
4227		warnings += validate_symbol(file, sec, func, &state);
4228	}
4229
4230	return warnings;
4231}
4232
4233static int validate_noinstr_sections(struct objtool_file *file)
4234{
4235	struct section *sec;
4236	int warnings = 0;
4237
4238	sec = find_section_by_name(file->elf, ".noinstr.text");
4239	if (sec) {
4240		warnings += validate_section(file, sec);
4241		warnings += validate_unwind_hints(file, sec);
4242	}
4243
4244	sec = find_section_by_name(file->elf, ".entry.text");
4245	if (sec) {
4246		warnings += validate_section(file, sec);
4247		warnings += validate_unwind_hints(file, sec);
4248	}
4249
4250	sec = find_section_by_name(file->elf, ".cpuidle.text");
4251	if (sec) {
4252		warnings += validate_section(file, sec);
4253		warnings += validate_unwind_hints(file, sec);
4254	}
4255
4256	return warnings;
4257}
4258
4259static int validate_functions(struct objtool_file *file)
4260{
4261	struct section *sec;
4262	int warnings = 0;
4263
4264	for_each_sec(file, sec) {
4265		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4266			continue;
4267
4268		warnings += validate_section(file, sec);
4269	}
4270
4271	return warnings;
4272}
4273
4274static void mark_endbr_used(struct instruction *insn)
4275{
4276	if (!list_empty(&insn->call_node))
4277		list_del_init(&insn->call_node);
4278}
4279
4280static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4281{
4282	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4283	struct instruction *first;
4284
4285	if (!sym)
4286		return false;
4287
4288	first = find_insn(file, sym->sec, sym->offset);
4289	if (!first)
4290		return false;
4291
4292	if (first->type != INSN_ENDBR && !first->noendbr)
4293		return false;
4294
4295	return insn->offset == sym->offset + sym->len;
4296}
4297
4298static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4299{
4300	struct instruction *dest;
4301	struct reloc *reloc;
4302	unsigned long off;
4303	int warnings = 0;
4304
4305	/*
4306	 * Looking for function pointer load relocations.  Ignore
4307	 * direct/indirect branches:
4308	 */
4309	switch (insn->type) {
4310	case INSN_CALL:
4311	case INSN_CALL_DYNAMIC:
4312	case INSN_JUMP_CONDITIONAL:
4313	case INSN_JUMP_UNCONDITIONAL:
4314	case INSN_JUMP_DYNAMIC:
4315	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4316	case INSN_RETURN:
4317	case INSN_NOP:
4318		return 0;
4319	default:
4320		break;
4321	}
4322
4323	for (reloc = insn_reloc(file, insn);
4324	     reloc;
4325	     reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4326					      reloc_offset(reloc) + 1,
4327					      (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4328
4329		/*
4330		 * static_call_update() references the trampoline, which
4331		 * doesn't have (or need) ENDBR.  Skip warning in that case.
4332		 */
4333		if (reloc->sym->static_call_tramp)
4334			continue;
4335
4336		off = reloc->sym->offset;
4337		if (reloc_type(reloc) == R_X86_64_PC32 ||
4338		    reloc_type(reloc) == R_X86_64_PLT32)
4339			off += arch_dest_reloc_offset(reloc_addend(reloc));
4340		else
4341			off += reloc_addend(reloc);
4342
4343		dest = find_insn(file, reloc->sym->sec, off);
4344		if (!dest)
4345			continue;
4346
4347		if (dest->type == INSN_ENDBR) {
4348			mark_endbr_used(dest);
4349			continue;
4350		}
4351
4352		if (insn_func(dest) && insn_func(insn) &&
4353		    insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4354			/*
4355			 * Anything from->to self is either _THIS_IP_ or
4356			 * IRET-to-self.
4357			 *
4358			 * There is no sane way to annotate _THIS_IP_ since the
4359			 * compiler treats the relocation as a constant and is
4360			 * happy to fold in offsets, skewing any annotation we
4361			 * do, leading to vast amounts of false-positives.
4362			 *
4363			 * There's also compiler generated _THIS_IP_ through
4364			 * KCOV and such which we have no hope of annotating.
4365			 *
4366			 * As such, blanket accept self-references without
4367			 * issue.
4368			 */
4369			continue;
4370		}
4371
4372		/*
4373		 * Accept anything ANNOTATE_NOENDBR.
4374		 */
4375		if (dest->noendbr)
4376			continue;
4377
4378		/*
4379		 * Accept if this is the instruction after a symbol
4380		 * that is (no)endbr -- typical code-range usage.
4381		 */
4382		if (noendbr_range(file, dest))
4383			continue;
4384
4385		WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4386
4387		warnings++;
4388	}
4389
4390	return warnings;
4391}
4392
4393static int validate_ibt_data_reloc(struct objtool_file *file,
4394				   struct reloc *reloc)
4395{
4396	struct instruction *dest;
4397
4398	dest = find_insn(file, reloc->sym->sec,
4399			 reloc->sym->offset + reloc_addend(reloc));
4400	if (!dest)
4401		return 0;
4402
4403	if (dest->type == INSN_ENDBR) {
4404		mark_endbr_used(dest);
4405		return 0;
4406	}
4407
4408	if (dest->noendbr)
4409		return 0;
4410
4411	WARN_FUNC("data relocation to !ENDBR: %s",
4412		  reloc->sec->base, reloc_offset(reloc),
4413		  offstr(dest->sec, dest->offset));
4414
4415	return 1;
4416}
4417
4418/*
4419 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4420 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4421 * NOPs) later, in create_ibt_endbr_seal_sections().
4422 */
4423static int validate_ibt(struct objtool_file *file)
4424{
4425	struct section *sec;
4426	struct reloc *reloc;
4427	struct instruction *insn;
4428	int warnings = 0;
4429
4430	for_each_insn(file, insn)
4431		warnings += validate_ibt_insn(file, insn);
4432
4433	for_each_sec(file, sec) {
4434
4435		/* Already done by validate_ibt_insn() */
4436		if (sec->sh.sh_flags & SHF_EXECINSTR)
4437			continue;
4438
4439		if (!sec->rsec)
4440			continue;
4441
4442		/*
4443		 * These sections can reference text addresses, but not with
4444		 * the intent to indirect branch to them.
4445		 */
4446		if ((!strncmp(sec->name, ".discard", 8) &&
4447		     strcmp(sec->name, ".discard.ibt_endbr_noseal"))	||
4448		    !strncmp(sec->name, ".debug", 6)			||
4449		    !strcmp(sec->name, ".altinstructions")		||
4450		    !strcmp(sec->name, ".ibt_endbr_seal")		||
4451		    !strcmp(sec->name, ".orc_unwind_ip")		||
4452		    !strcmp(sec->name, ".parainstructions")		||
4453		    !strcmp(sec->name, ".retpoline_sites")		||
4454		    !strcmp(sec->name, ".smp_locks")			||
4455		    !strcmp(sec->name, ".static_call_sites")		||
4456		    !strcmp(sec->name, "_error_injection_whitelist")	||
4457		    !strcmp(sec->name, "_kprobe_blacklist")		||
4458		    !strcmp(sec->name, "__bug_table")			||
4459		    !strcmp(sec->name, "__ex_table")			||
4460		    !strcmp(sec->name, "__jump_table")			||
4461		    !strcmp(sec->name, "__mcount_loc")			||
4462		    !strcmp(sec->name, ".kcfi_traps")			||
4463		    strstr(sec->name, "__patchable_function_entries"))
4464			continue;
4465
4466		for_each_reloc(sec->rsec, reloc)
4467			warnings += validate_ibt_data_reloc(file, reloc);
4468	}
4469
4470	return warnings;
4471}
4472
4473static int validate_sls(struct objtool_file *file)
4474{
4475	struct instruction *insn, *next_insn;
4476	int warnings = 0;
4477
4478	for_each_insn(file, insn) {
4479		next_insn = next_insn_same_sec(file, insn);
4480
4481		if (insn->retpoline_safe)
4482			continue;
4483
4484		switch (insn->type) {
4485		case INSN_RETURN:
4486			if (!next_insn || next_insn->type != INSN_TRAP) {
4487				WARN_INSN(insn, "missing int3 after ret");
4488				warnings++;
4489			}
4490
4491			break;
4492		case INSN_JUMP_DYNAMIC:
4493			if (!next_insn || next_insn->type != INSN_TRAP) {
4494				WARN_INSN(insn, "missing int3 after indirect jump");
4495				warnings++;
4496			}
4497			break;
4498		default:
4499			break;
4500		}
4501	}
4502
4503	return warnings;
4504}
4505
4506static bool ignore_noreturn_call(struct instruction *insn)
4507{
4508	struct symbol *call_dest = insn_call_dest(insn);
4509
4510	/*
4511	 * FIXME: hack, we need a real noreturn solution
4512	 *
4513	 * Problem is, exc_double_fault() may or may not return, depending on
4514	 * whether CONFIG_X86_ESPFIX64 is set.  But objtool has no visibility
4515	 * to the kernel config.
4516	 *
4517	 * Other potential ways to fix it:
4518	 *
4519	 *   - have compiler communicate __noreturn functions somehow
4520	 *   - remove CONFIG_X86_ESPFIX64
4521	 *   - read the .config file
4522	 *   - add a cmdline option
4523	 *   - create a generic objtool annotation format (vs a bunch of custom
4524	 *     formats) and annotate it
4525	 */
4526	if (!strcmp(call_dest->name, "exc_double_fault")) {
4527		/* prevent further unreachable warnings for the caller */
4528		insn->sym->warned = 1;
4529		return true;
4530	}
4531
4532	return false;
4533}
4534
4535static int validate_reachable_instructions(struct objtool_file *file)
4536{
4537	struct instruction *insn, *prev_insn;
4538	struct symbol *call_dest;
4539	int warnings = 0;
4540
4541	if (file->ignore_unreachables)
4542		return 0;
4543
4544	for_each_insn(file, insn) {
4545		if (insn->visited || ignore_unreachable_insn(file, insn))
4546			continue;
4547
4548		prev_insn = prev_insn_same_sec(file, insn);
4549		if (prev_insn && prev_insn->dead_end) {
4550			call_dest = insn_call_dest(prev_insn);
4551			if (call_dest && !ignore_noreturn_call(prev_insn)) {
4552				WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4553					  call_dest->name);
4554				warnings++;
4555				continue;
4556			}
4557		}
4558
4559		WARN_INSN(insn, "unreachable instruction");
4560		warnings++;
4561	}
4562
4563	return warnings;
4564}
4565
4566/* 'funcs' is a space-separated list of function names */
4567static int disas_funcs(const char *funcs)
4568{
4569	const char *objdump_str, *cross_compile;
4570	int size, ret;
4571	char *cmd;
4572
4573	cross_compile = getenv("CROSS_COMPILE");
4574
4575	objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4576			"BEGIN { split(_funcs, funcs); }"
4577			"/^$/ { func_match = 0; }"
4578			"/<.*>:/ { "
4579				"f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4580				"for (i in funcs) {"
4581					"if (funcs[i] == f) {"
4582						"func_match = 1;"
4583						"base = strtonum(\"0x\" $1);"
4584						"break;"
4585					"}"
4586				"}"
4587			"}"
4588			"{"
4589				"if (func_match) {"
4590					"addr = strtonum(\"0x\" $1);"
4591					"printf(\"%%04x \", addr - base);"
4592					"print;"
4593				"}"
4594			"}' 1>&2";
4595
4596	/* fake snprintf() to calculate the size */
4597	size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4598	if (size <= 0) {
4599		WARN("objdump string size calculation failed");
4600		return -1;
4601	}
4602
4603	cmd = malloc(size);
4604
4605	/* real snprintf() */
4606	snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4607	ret = system(cmd);
4608	if (ret) {
4609		WARN("disassembly failed: %d", ret);
4610		return -1;
4611	}
4612
4613	return 0;
4614}
4615
4616static int disas_warned_funcs(struct objtool_file *file)
4617{
4618	struct symbol *sym;
4619	char *funcs = NULL, *tmp;
4620
4621	for_each_sym(file, sym) {
4622		if (sym->warned) {
4623			if (!funcs) {
4624				funcs = malloc(strlen(sym->name) + 1);
4625				strcpy(funcs, sym->name);
4626			} else {
4627				tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4628				sprintf(tmp, "%s %s", funcs, sym->name);
4629				free(funcs);
4630				funcs = tmp;
4631			}
4632		}
4633	}
4634
4635	if (funcs)
4636		disas_funcs(funcs);
4637
4638	return 0;
4639}
4640
4641struct insn_chunk {
4642	void *addr;
4643	struct insn_chunk *next;
4644};
4645
4646/*
4647 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4648 * which can trigger more allocations for .debug_* sections whose data hasn't
4649 * been read yet.
4650 */
4651static void free_insns(struct objtool_file *file)
4652{
4653	struct instruction *insn;
4654	struct insn_chunk *chunks = NULL, *chunk;
4655
4656	for_each_insn(file, insn) {
4657		if (!insn->idx) {
4658			chunk = malloc(sizeof(*chunk));
4659			chunk->addr = insn;
4660			chunk->next = chunks;
4661			chunks = chunk;
4662		}
4663	}
4664
4665	for (chunk = chunks; chunk; chunk = chunk->next)
4666		free(chunk->addr);
4667}
4668
4669int check(struct objtool_file *file)
4670{
4671	int ret, warnings = 0;
4672
4673	arch_initial_func_cfi_state(&initial_func_cfi);
4674	init_cfi_state(&init_cfi);
4675	init_cfi_state(&func_cfi);
4676	set_func_state(&func_cfi);
4677	init_cfi_state(&force_undefined_cfi);
4678	force_undefined_cfi.force_undefined = true;
4679
4680	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4681		goto out;
4682
4683	cfi_hash_add(&init_cfi);
4684	cfi_hash_add(&func_cfi);
4685
4686	ret = decode_sections(file);
4687	if (ret < 0)
4688		goto out;
4689
4690	warnings += ret;
4691
4692	if (!nr_insns)
4693		goto out;
4694
4695	if (opts.retpoline) {
4696		ret = validate_retpoline(file);
4697		if (ret < 0)
4698			return ret;
4699		warnings += ret;
4700	}
4701
4702	if (opts.stackval || opts.orc || opts.uaccess) {
4703		ret = validate_functions(file);
4704		if (ret < 0)
4705			goto out;
4706		warnings += ret;
4707
4708		ret = validate_unwind_hints(file, NULL);
4709		if (ret < 0)
4710			goto out;
4711		warnings += ret;
4712
4713		if (!warnings) {
4714			ret = validate_reachable_instructions(file);
4715			if (ret < 0)
4716				goto out;
4717			warnings += ret;
4718		}
4719
4720	} else if (opts.noinstr) {
4721		ret = validate_noinstr_sections(file);
4722		if (ret < 0)
4723			goto out;
4724		warnings += ret;
4725	}
4726
4727	if (opts.unret) {
4728		/*
4729		 * Must be after validate_branch() and friends, it plays
4730		 * further games with insn->visited.
4731		 */
4732		ret = validate_unrets(file);
4733		if (ret < 0)
4734			return ret;
4735		warnings += ret;
4736	}
4737
4738	if (opts.ibt) {
4739		ret = validate_ibt(file);
4740		if (ret < 0)
4741			goto out;
4742		warnings += ret;
4743	}
4744
4745	if (opts.sls) {
4746		ret = validate_sls(file);
4747		if (ret < 0)
4748			goto out;
4749		warnings += ret;
4750	}
4751
4752	if (opts.static_call) {
4753		ret = create_static_call_sections(file);
4754		if (ret < 0)
4755			goto out;
4756		warnings += ret;
4757	}
4758
4759	if (opts.retpoline) {
4760		ret = create_retpoline_sites_sections(file);
4761		if (ret < 0)
4762			goto out;
4763		warnings += ret;
4764	}
4765
4766	if (opts.cfi) {
4767		ret = create_cfi_sections(file);
4768		if (ret < 0)
4769			goto out;
4770		warnings += ret;
4771	}
4772
4773	if (opts.rethunk) {
4774		ret = create_return_sites_sections(file);
4775		if (ret < 0)
4776			goto out;
4777		warnings += ret;
4778
4779		if (opts.hack_skylake) {
4780			ret = create_direct_call_sections(file);
4781			if (ret < 0)
4782				goto out;
4783			warnings += ret;
4784		}
4785	}
4786
4787	if (opts.mcount) {
4788		ret = create_mcount_loc_sections(file);
4789		if (ret < 0)
4790			goto out;
4791		warnings += ret;
4792	}
4793
4794	if (opts.prefix) {
4795		ret = add_prefix_symbols(file);
4796		if (ret < 0)
4797			return ret;
4798		warnings += ret;
4799	}
4800
4801	if (opts.ibt) {
4802		ret = create_ibt_endbr_seal_sections(file);
4803		if (ret < 0)
4804			goto out;
4805		warnings += ret;
4806	}
4807
4808	if (opts.orc && nr_insns) {
4809		ret = orc_create(file);
4810		if (ret < 0)
4811			goto out;
4812		warnings += ret;
4813	}
4814
4815	free_insns(file);
4816
4817	if (opts.verbose)
4818		disas_warned_funcs(file);
4819
4820	if (opts.stats) {
4821		printf("nr_insns_visited: %ld\n", nr_insns_visited);
4822		printf("nr_cfi: %ld\n", nr_cfi);
4823		printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4824		printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4825	}
4826
4827out:
4828	/*
4829	 *  For now, don't fail the kernel build on fatal warnings.  These
4830	 *  errors are still fairly common due to the growing matrix of
4831	 *  supported toolchains and their recent pace of change.
4832	 */
4833	return 0;
4834}