Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6#include <string.h>
7#include <stdlib.h>
8
9#include <arch/elf.h>
10#include <objtool/builtin.h>
11#include <objtool/cfi.h>
12#include <objtool/arch.h>
13#include <objtool/check.h>
14#include <objtool/special.h>
15#include <objtool/warn.h>
16#include <objtool/endianness.h>
17
18#include <linux/objtool.h>
19#include <linux/hashtable.h>
20#include <linux/kernel.h>
21#include <linux/static_call_types.h>
22
23struct alternative {
24 struct list_head list;
25 struct instruction *insn;
26 bool skip_orig;
27};
28
29struct cfi_init_state initial_func_cfi;
30
31struct instruction *find_insn(struct objtool_file *file,
32 struct section *sec, unsigned long offset)
33{
34 struct instruction *insn;
35
36 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
37 if (insn->sec == sec && insn->offset == offset)
38 return insn;
39 }
40
41 return NULL;
42}
43
44static struct instruction *next_insn_same_sec(struct objtool_file *file,
45 struct instruction *insn)
46{
47 struct instruction *next = list_next_entry(insn, list);
48
49 if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
50 return NULL;
51
52 return next;
53}
54
55static struct instruction *next_insn_same_func(struct objtool_file *file,
56 struct instruction *insn)
57{
58 struct instruction *next = list_next_entry(insn, list);
59 struct symbol *func = insn->func;
60
61 if (!func)
62 return NULL;
63
64 if (&next->list != &file->insn_list && next->func == func)
65 return next;
66
67 /* Check if we're already in the subfunction: */
68 if (func == func->cfunc)
69 return NULL;
70
71 /* Move to the subfunction: */
72 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
73}
74
75static struct instruction *prev_insn_same_sym(struct objtool_file *file,
76 struct instruction *insn)
77{
78 struct instruction *prev = list_prev_entry(insn, list);
79
80 if (&prev->list != &file->insn_list && prev->func == insn->func)
81 return prev;
82
83 return NULL;
84}
85
86#define func_for_each_insn(file, func, insn) \
87 for (insn = find_insn(file, func->sec, func->offset); \
88 insn; \
89 insn = next_insn_same_func(file, insn))
90
91#define sym_for_each_insn(file, sym, insn) \
92 for (insn = find_insn(file, sym->sec, sym->offset); \
93 insn && &insn->list != &file->insn_list && \
94 insn->sec == sym->sec && \
95 insn->offset < sym->offset + sym->len; \
96 insn = list_next_entry(insn, list))
97
98#define sym_for_each_insn_continue_reverse(file, sym, insn) \
99 for (insn = list_prev_entry(insn, list); \
100 &insn->list != &file->insn_list && \
101 insn->sec == sym->sec && insn->offset >= sym->offset; \
102 insn = list_prev_entry(insn, list))
103
104#define sec_for_each_insn_from(file, insn) \
105 for (; insn; insn = next_insn_same_sec(file, insn))
106
107#define sec_for_each_insn_continue(file, insn) \
108 for (insn = next_insn_same_sec(file, insn); insn; \
109 insn = next_insn_same_sec(file, insn))
110
111static bool is_jump_table_jump(struct instruction *insn)
112{
113 struct alt_group *alt_group = insn->alt_group;
114
115 if (insn->jump_table)
116 return true;
117
118 /* Retpoline alternative for a jump table? */
119 return alt_group && alt_group->orig_group &&
120 alt_group->orig_group->first_insn->jump_table;
121}
122
123static bool is_sibling_call(struct instruction *insn)
124{
125 /*
126 * Assume only ELF functions can make sibling calls. This ensures
127 * sibling call detection consistency between vmlinux.o and individual
128 * objects.
129 */
130 if (!insn->func)
131 return false;
132
133 /* An indirect jump is either a sibling call or a jump to a table. */
134 if (insn->type == INSN_JUMP_DYNAMIC)
135 return !is_jump_table_jump(insn);
136
137 /* add_jump_destinations() sets insn->call_dest for sibling calls. */
138 return (is_static_jump(insn) && insn->call_dest);
139}
140
141/*
142 * This checks to see if the given function is a "noreturn" function.
143 *
144 * For global functions which are outside the scope of this object file, we
145 * have to keep a manual list of them.
146 *
147 * For local functions, we have to detect them manually by simply looking for
148 * the lack of a return instruction.
149 */
150static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
151 int recursion)
152{
153 int i;
154 struct instruction *insn;
155 bool empty = true;
156
157 /*
158 * Unfortunately these have to be hard coded because the noreturn
159 * attribute isn't provided in ELF data.
160 */
161 static const char * const global_noreturns[] = {
162 "__stack_chk_fail",
163 "panic",
164 "do_exit",
165 "do_task_dead",
166 "__module_put_and_exit",
167 "complete_and_exit",
168 "__reiserfs_panic",
169 "lbug_with_loc",
170 "fortify_panic",
171 "usercopy_abort",
172 "machine_real_restart",
173 "rewind_stack_do_exit",
174 "kunit_try_catch_throw",
175 "xen_start_kernel",
176 };
177
178 if (!func)
179 return false;
180
181 if (func->bind == STB_WEAK)
182 return false;
183
184 if (func->bind == STB_GLOBAL)
185 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
186 if (!strcmp(func->name, global_noreturns[i]))
187 return true;
188
189 if (!func->len)
190 return false;
191
192 insn = find_insn(file, func->sec, func->offset);
193 if (!insn->func)
194 return false;
195
196 func_for_each_insn(file, func, insn) {
197 empty = false;
198
199 if (insn->type == INSN_RETURN)
200 return false;
201 }
202
203 if (empty)
204 return false;
205
206 /*
207 * A function can have a sibling call instead of a return. In that
208 * case, the function's dead-end status depends on whether the target
209 * of the sibling call returns.
210 */
211 func_for_each_insn(file, func, insn) {
212 if (is_sibling_call(insn)) {
213 struct instruction *dest = insn->jump_dest;
214
215 if (!dest)
216 /* sibling call to another file */
217 return false;
218
219 /* local sibling call */
220 if (recursion == 5) {
221 /*
222 * Infinite recursion: two functions have
223 * sibling calls to each other. This is a very
224 * rare case. It means they aren't dead ends.
225 */
226 return false;
227 }
228
229 return __dead_end_function(file, dest->func, recursion+1);
230 }
231 }
232
233 return true;
234}
235
236static bool dead_end_function(struct objtool_file *file, struct symbol *func)
237{
238 return __dead_end_function(file, func, 0);
239}
240
241static void init_cfi_state(struct cfi_state *cfi)
242{
243 int i;
244
245 for (i = 0; i < CFI_NUM_REGS; i++) {
246 cfi->regs[i].base = CFI_UNDEFINED;
247 cfi->vals[i].base = CFI_UNDEFINED;
248 }
249 cfi->cfa.base = CFI_UNDEFINED;
250 cfi->drap_reg = CFI_UNDEFINED;
251 cfi->drap_offset = -1;
252}
253
254static void init_insn_state(struct insn_state *state, struct section *sec)
255{
256 memset(state, 0, sizeof(*state));
257 init_cfi_state(&state->cfi);
258
259 /*
260 * We need the full vmlinux for noinstr validation, otherwise we can
261 * not correctly determine insn->call_dest->sec (external symbols do
262 * not have a section).
263 */
264 if (vmlinux && noinstr && sec)
265 state->noinstr = sec->noinstr;
266}
267
268/*
269 * Call the arch-specific instruction decoder for all the instructions and add
270 * them to the global instruction list.
271 */
272static int decode_instructions(struct objtool_file *file)
273{
274 struct section *sec;
275 struct symbol *func;
276 unsigned long offset;
277 struct instruction *insn;
278 unsigned long nr_insns = 0;
279 int ret;
280
281 for_each_sec(file, sec) {
282
283 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
284 continue;
285
286 if (strcmp(sec->name, ".altinstr_replacement") &&
287 strcmp(sec->name, ".altinstr_aux") &&
288 strncmp(sec->name, ".discard.", 9))
289 sec->text = true;
290
291 if (!strcmp(sec->name, ".noinstr.text") ||
292 !strcmp(sec->name, ".entry.text"))
293 sec->noinstr = true;
294
295 for (offset = 0; offset < sec->len; offset += insn->len) {
296 insn = malloc(sizeof(*insn));
297 if (!insn) {
298 WARN("malloc failed");
299 return -1;
300 }
301 memset(insn, 0, sizeof(*insn));
302 INIT_LIST_HEAD(&insn->alts);
303 INIT_LIST_HEAD(&insn->stack_ops);
304 init_cfi_state(&insn->cfi);
305
306 insn->sec = sec;
307 insn->offset = offset;
308
309 ret = arch_decode_instruction(file->elf, sec, offset,
310 sec->len - offset,
311 &insn->len, &insn->type,
312 &insn->immediate,
313 &insn->stack_ops);
314 if (ret)
315 goto err;
316
317 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
318 list_add_tail(&insn->list, &file->insn_list);
319 nr_insns++;
320 }
321
322 list_for_each_entry(func, &sec->symbol_list, list) {
323 if (func->type != STT_FUNC || func->alias != func)
324 continue;
325
326 if (!find_insn(file, sec, func->offset)) {
327 WARN("%s(): can't find starting instruction",
328 func->name);
329 return -1;
330 }
331
332 sym_for_each_insn(file, func, insn)
333 insn->func = func;
334 }
335 }
336
337 if (stats)
338 printf("nr_insns: %lu\n", nr_insns);
339
340 return 0;
341
342err:
343 free(insn);
344 return ret;
345}
346
347static struct instruction *find_last_insn(struct objtool_file *file,
348 struct section *sec)
349{
350 struct instruction *insn = NULL;
351 unsigned int offset;
352 unsigned int end = (sec->len > 10) ? sec->len - 10 : 0;
353
354 for (offset = sec->len - 1; offset >= end && !insn; offset--)
355 insn = find_insn(file, sec, offset);
356
357 return insn;
358}
359
360/*
361 * Mark "ud2" instructions and manually annotated dead ends.
362 */
363static int add_dead_ends(struct objtool_file *file)
364{
365 struct section *sec;
366 struct reloc *reloc;
367 struct instruction *insn;
368
369 /*
370 * By default, "ud2" is a dead end unless otherwise annotated, because
371 * GCC 7 inserts it for certain divide-by-zero cases.
372 */
373 for_each_insn(file, insn)
374 if (insn->type == INSN_BUG)
375 insn->dead_end = true;
376
377 /*
378 * Check for manually annotated dead ends.
379 */
380 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
381 if (!sec)
382 goto reachable;
383
384 list_for_each_entry(reloc, &sec->reloc_list, list) {
385 if (reloc->sym->type != STT_SECTION) {
386 WARN("unexpected relocation symbol type in %s", sec->name);
387 return -1;
388 }
389 insn = find_insn(file, reloc->sym->sec, reloc->addend);
390 if (insn)
391 insn = list_prev_entry(insn, list);
392 else if (reloc->addend == reloc->sym->sec->len) {
393 insn = find_last_insn(file, reloc->sym->sec);
394 if (!insn) {
395 WARN("can't find unreachable insn at %s+0x%x",
396 reloc->sym->sec->name, reloc->addend);
397 return -1;
398 }
399 } else {
400 WARN("can't find unreachable insn at %s+0x%x",
401 reloc->sym->sec->name, reloc->addend);
402 return -1;
403 }
404
405 insn->dead_end = true;
406 }
407
408reachable:
409 /*
410 * These manually annotated reachable checks are needed for GCC 4.4,
411 * where the Linux unreachable() macro isn't supported. In that case
412 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
413 * not a dead end.
414 */
415 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
416 if (!sec)
417 return 0;
418
419 list_for_each_entry(reloc, &sec->reloc_list, list) {
420 if (reloc->sym->type != STT_SECTION) {
421 WARN("unexpected relocation symbol type in %s", sec->name);
422 return -1;
423 }
424 insn = find_insn(file, reloc->sym->sec, reloc->addend);
425 if (insn)
426 insn = list_prev_entry(insn, list);
427 else if (reloc->addend == reloc->sym->sec->len) {
428 insn = find_last_insn(file, reloc->sym->sec);
429 if (!insn) {
430 WARN("can't find reachable insn at %s+0x%x",
431 reloc->sym->sec->name, reloc->addend);
432 return -1;
433 }
434 } else {
435 WARN("can't find reachable insn at %s+0x%x",
436 reloc->sym->sec->name, reloc->addend);
437 return -1;
438 }
439
440 insn->dead_end = false;
441 }
442
443 return 0;
444}
445
446static int create_static_call_sections(struct objtool_file *file)
447{
448 struct section *sec;
449 struct static_call_site *site;
450 struct instruction *insn;
451 struct symbol *key_sym;
452 char *key_name, *tmp;
453 int idx;
454
455 sec = find_section_by_name(file->elf, ".static_call_sites");
456 if (sec) {
457 INIT_LIST_HEAD(&file->static_call_list);
458 WARN("file already has .static_call_sites section, skipping");
459 return 0;
460 }
461
462 if (list_empty(&file->static_call_list))
463 return 0;
464
465 idx = 0;
466 list_for_each_entry(insn, &file->static_call_list, call_node)
467 idx++;
468
469 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
470 sizeof(struct static_call_site), idx);
471 if (!sec)
472 return -1;
473
474 idx = 0;
475 list_for_each_entry(insn, &file->static_call_list, call_node) {
476
477 site = (struct static_call_site *)sec->data->d_buf + idx;
478 memset(site, 0, sizeof(struct static_call_site));
479
480 /* populate reloc for 'addr' */
481 if (elf_add_reloc_to_insn(file->elf, sec,
482 idx * sizeof(struct static_call_site),
483 R_X86_64_PC32,
484 insn->sec, insn->offset))
485 return -1;
486
487 /* find key symbol */
488 key_name = strdup(insn->call_dest->name);
489 if (!key_name) {
490 perror("strdup");
491 return -1;
492 }
493 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
494 STATIC_CALL_TRAMP_PREFIX_LEN)) {
495 WARN("static_call: trampoline name malformed: %s", key_name);
496 return -1;
497 }
498 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
499 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
500
501 key_sym = find_symbol_by_name(file->elf, tmp);
502 if (!key_sym) {
503 if (!module) {
504 WARN("static_call: can't find static_call_key symbol: %s", tmp);
505 return -1;
506 }
507
508 /*
509 * For modules(), the key might not be exported, which
510 * means the module can make static calls but isn't
511 * allowed to change them.
512 *
513 * In that case we temporarily set the key to be the
514 * trampoline address. This is fixed up in
515 * static_call_add_module().
516 */
517 key_sym = insn->call_dest;
518 }
519 free(key_name);
520
521 /* populate reloc for 'key' */
522 if (elf_add_reloc(file->elf, sec,
523 idx * sizeof(struct static_call_site) + 4,
524 R_X86_64_PC32, key_sym,
525 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
526 return -1;
527
528 idx++;
529 }
530
531 return 0;
532}
533
534static int create_mcount_loc_sections(struct objtool_file *file)
535{
536 struct section *sec;
537 unsigned long *loc;
538 struct instruction *insn;
539 int idx;
540
541 sec = find_section_by_name(file->elf, "__mcount_loc");
542 if (sec) {
543 INIT_LIST_HEAD(&file->mcount_loc_list);
544 WARN("file already has __mcount_loc section, skipping");
545 return 0;
546 }
547
548 if (list_empty(&file->mcount_loc_list))
549 return 0;
550
551 idx = 0;
552 list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node)
553 idx++;
554
555 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
556 if (!sec)
557 return -1;
558
559 idx = 0;
560 list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node) {
561
562 loc = (unsigned long *)sec->data->d_buf + idx;
563 memset(loc, 0, sizeof(unsigned long));
564
565 if (elf_add_reloc_to_insn(file->elf, sec,
566 idx * sizeof(unsigned long),
567 R_X86_64_64,
568 insn->sec, insn->offset))
569 return -1;
570
571 idx++;
572 }
573
574 return 0;
575}
576
577/*
578 * Warnings shouldn't be reported for ignored functions.
579 */
580static void add_ignores(struct objtool_file *file)
581{
582 struct instruction *insn;
583 struct section *sec;
584 struct symbol *func;
585 struct reloc *reloc;
586
587 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
588 if (!sec)
589 return;
590
591 list_for_each_entry(reloc, &sec->reloc_list, list) {
592 switch (reloc->sym->type) {
593 case STT_FUNC:
594 func = reloc->sym;
595 break;
596
597 case STT_SECTION:
598 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
599 if (!func)
600 continue;
601 break;
602
603 default:
604 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
605 continue;
606 }
607
608 func_for_each_insn(file, func, insn)
609 insn->ignore = true;
610 }
611}
612
613/*
614 * This is a whitelist of functions that is allowed to be called with AC set.
615 * The list is meant to be minimal and only contains compiler instrumentation
616 * ABI and a few functions used to implement *_{to,from}_user() functions.
617 *
618 * These functions must not directly change AC, but may PUSHF/POPF.
619 */
620static const char *uaccess_safe_builtin[] = {
621 /* KASAN */
622 "kasan_report",
623 "kasan_check_range",
624 /* KASAN out-of-line */
625 "__asan_loadN_noabort",
626 "__asan_load1_noabort",
627 "__asan_load2_noabort",
628 "__asan_load4_noabort",
629 "__asan_load8_noabort",
630 "__asan_load16_noabort",
631 "__asan_storeN_noabort",
632 "__asan_store1_noabort",
633 "__asan_store2_noabort",
634 "__asan_store4_noabort",
635 "__asan_store8_noabort",
636 "__asan_store16_noabort",
637 "__kasan_check_read",
638 "__kasan_check_write",
639 /* KASAN in-line */
640 "__asan_report_load_n_noabort",
641 "__asan_report_load1_noabort",
642 "__asan_report_load2_noabort",
643 "__asan_report_load4_noabort",
644 "__asan_report_load8_noabort",
645 "__asan_report_load16_noabort",
646 "__asan_report_store_n_noabort",
647 "__asan_report_store1_noabort",
648 "__asan_report_store2_noabort",
649 "__asan_report_store4_noabort",
650 "__asan_report_store8_noabort",
651 "__asan_report_store16_noabort",
652 /* KCSAN */
653 "__kcsan_check_access",
654 "kcsan_found_watchpoint",
655 "kcsan_setup_watchpoint",
656 "kcsan_check_scoped_accesses",
657 "kcsan_disable_current",
658 "kcsan_enable_current_nowarn",
659 /* KCSAN/TSAN */
660 "__tsan_func_entry",
661 "__tsan_func_exit",
662 "__tsan_read_range",
663 "__tsan_write_range",
664 "__tsan_read1",
665 "__tsan_read2",
666 "__tsan_read4",
667 "__tsan_read8",
668 "__tsan_read16",
669 "__tsan_write1",
670 "__tsan_write2",
671 "__tsan_write4",
672 "__tsan_write8",
673 "__tsan_write16",
674 "__tsan_read_write1",
675 "__tsan_read_write2",
676 "__tsan_read_write4",
677 "__tsan_read_write8",
678 "__tsan_read_write16",
679 "__tsan_atomic8_load",
680 "__tsan_atomic16_load",
681 "__tsan_atomic32_load",
682 "__tsan_atomic64_load",
683 "__tsan_atomic8_store",
684 "__tsan_atomic16_store",
685 "__tsan_atomic32_store",
686 "__tsan_atomic64_store",
687 "__tsan_atomic8_exchange",
688 "__tsan_atomic16_exchange",
689 "__tsan_atomic32_exchange",
690 "__tsan_atomic64_exchange",
691 "__tsan_atomic8_fetch_add",
692 "__tsan_atomic16_fetch_add",
693 "__tsan_atomic32_fetch_add",
694 "__tsan_atomic64_fetch_add",
695 "__tsan_atomic8_fetch_sub",
696 "__tsan_atomic16_fetch_sub",
697 "__tsan_atomic32_fetch_sub",
698 "__tsan_atomic64_fetch_sub",
699 "__tsan_atomic8_fetch_and",
700 "__tsan_atomic16_fetch_and",
701 "__tsan_atomic32_fetch_and",
702 "__tsan_atomic64_fetch_and",
703 "__tsan_atomic8_fetch_or",
704 "__tsan_atomic16_fetch_or",
705 "__tsan_atomic32_fetch_or",
706 "__tsan_atomic64_fetch_or",
707 "__tsan_atomic8_fetch_xor",
708 "__tsan_atomic16_fetch_xor",
709 "__tsan_atomic32_fetch_xor",
710 "__tsan_atomic64_fetch_xor",
711 "__tsan_atomic8_fetch_nand",
712 "__tsan_atomic16_fetch_nand",
713 "__tsan_atomic32_fetch_nand",
714 "__tsan_atomic64_fetch_nand",
715 "__tsan_atomic8_compare_exchange_strong",
716 "__tsan_atomic16_compare_exchange_strong",
717 "__tsan_atomic32_compare_exchange_strong",
718 "__tsan_atomic64_compare_exchange_strong",
719 "__tsan_atomic8_compare_exchange_weak",
720 "__tsan_atomic16_compare_exchange_weak",
721 "__tsan_atomic32_compare_exchange_weak",
722 "__tsan_atomic64_compare_exchange_weak",
723 "__tsan_atomic8_compare_exchange_val",
724 "__tsan_atomic16_compare_exchange_val",
725 "__tsan_atomic32_compare_exchange_val",
726 "__tsan_atomic64_compare_exchange_val",
727 "__tsan_atomic_thread_fence",
728 "__tsan_atomic_signal_fence",
729 /* KCOV */
730 "write_comp_data",
731 "check_kcov_mode",
732 "__sanitizer_cov_trace_pc",
733 "__sanitizer_cov_trace_const_cmp1",
734 "__sanitizer_cov_trace_const_cmp2",
735 "__sanitizer_cov_trace_const_cmp4",
736 "__sanitizer_cov_trace_const_cmp8",
737 "__sanitizer_cov_trace_cmp1",
738 "__sanitizer_cov_trace_cmp2",
739 "__sanitizer_cov_trace_cmp4",
740 "__sanitizer_cov_trace_cmp8",
741 "__sanitizer_cov_trace_switch",
742 /* UBSAN */
743 "ubsan_type_mismatch_common",
744 "__ubsan_handle_type_mismatch",
745 "__ubsan_handle_type_mismatch_v1",
746 "__ubsan_handle_shift_out_of_bounds",
747 /* misc */
748 "csum_partial_copy_generic",
749 "copy_mc_fragile",
750 "copy_mc_fragile_handle_tail",
751 "copy_mc_enhanced_fast_string",
752 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
753 NULL
754};
755
756static void add_uaccess_safe(struct objtool_file *file)
757{
758 struct symbol *func;
759 const char **name;
760
761 if (!uaccess)
762 return;
763
764 for (name = uaccess_safe_builtin; *name; name++) {
765 func = find_symbol_by_name(file->elf, *name);
766 if (!func)
767 continue;
768
769 func->uaccess_safe = true;
770 }
771}
772
773/*
774 * FIXME: For now, just ignore any alternatives which add retpolines. This is
775 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
776 * But it at least allows objtool to understand the control flow *around* the
777 * retpoline.
778 */
779static int add_ignore_alternatives(struct objtool_file *file)
780{
781 struct section *sec;
782 struct reloc *reloc;
783 struct instruction *insn;
784
785 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
786 if (!sec)
787 return 0;
788
789 list_for_each_entry(reloc, &sec->reloc_list, list) {
790 if (reloc->sym->type != STT_SECTION) {
791 WARN("unexpected relocation symbol type in %s", sec->name);
792 return -1;
793 }
794
795 insn = find_insn(file, reloc->sym->sec, reloc->addend);
796 if (!insn) {
797 WARN("bad .discard.ignore_alts entry");
798 return -1;
799 }
800
801 insn->ignore_alts = true;
802 }
803
804 return 0;
805}
806
807__weak bool arch_is_retpoline(struct symbol *sym)
808{
809 return false;
810}
811
812#define NEGATIVE_RELOC ((void *)-1L)
813
814static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
815{
816 if (insn->reloc == NEGATIVE_RELOC)
817 return NULL;
818
819 if (!insn->reloc) {
820 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
821 insn->offset, insn->len);
822 if (!insn->reloc) {
823 insn->reloc = NEGATIVE_RELOC;
824 return NULL;
825 }
826 }
827
828 return insn->reloc;
829}
830
831/*
832 * Find the destination instructions for all jumps.
833 */
834static int add_jump_destinations(struct objtool_file *file)
835{
836 struct instruction *insn;
837 struct reloc *reloc;
838 struct section *dest_sec;
839 unsigned long dest_off;
840
841 for_each_insn(file, insn) {
842 if (!is_static_jump(insn))
843 continue;
844
845 reloc = insn_reloc(file, insn);
846 if (!reloc) {
847 dest_sec = insn->sec;
848 dest_off = arch_jump_destination(insn);
849 } else if (reloc->sym->type == STT_SECTION) {
850 dest_sec = reloc->sym->sec;
851 dest_off = arch_dest_reloc_offset(reloc->addend);
852 } else if (arch_is_retpoline(reloc->sym)) {
853 /*
854 * Retpoline jumps are really dynamic jumps in
855 * disguise, so convert them accordingly.
856 */
857 if (insn->type == INSN_JUMP_UNCONDITIONAL)
858 insn->type = INSN_JUMP_DYNAMIC;
859 else
860 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
861
862 list_add_tail(&insn->call_node,
863 &file->retpoline_call_list);
864
865 insn->retpoline_safe = true;
866 continue;
867 } else if (insn->func) {
868 /* internal or external sibling call (with reloc) */
869 insn->call_dest = reloc->sym;
870 if (insn->call_dest->static_call_tramp) {
871 list_add_tail(&insn->call_node,
872 &file->static_call_list);
873 }
874 continue;
875 } else if (reloc->sym->sec->idx) {
876 dest_sec = reloc->sym->sec;
877 dest_off = reloc->sym->sym.st_value +
878 arch_dest_reloc_offset(reloc->addend);
879 } else {
880 /* non-func asm code jumping to another file */
881 continue;
882 }
883
884 insn->jump_dest = find_insn(file, dest_sec, dest_off);
885 if (!insn->jump_dest) {
886
887 /*
888 * This is a special case where an alt instruction
889 * jumps past the end of the section. These are
890 * handled later in handle_group_alt().
891 */
892 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
893 continue;
894
895 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
896 insn->sec, insn->offset, dest_sec->name,
897 dest_off);
898 return -1;
899 }
900
901 /*
902 * Cross-function jump.
903 */
904 if (insn->func && insn->jump_dest->func &&
905 insn->func != insn->jump_dest->func) {
906
907 /*
908 * For GCC 8+, create parent/child links for any cold
909 * subfunctions. This is _mostly_ redundant with a
910 * similar initialization in read_symbols().
911 *
912 * If a function has aliases, we want the *first* such
913 * function in the symbol table to be the subfunction's
914 * parent. In that case we overwrite the
915 * initialization done in read_symbols().
916 *
917 * However this code can't completely replace the
918 * read_symbols() code because this doesn't detect the
919 * case where the parent function's only reference to a
920 * subfunction is through a jump table.
921 */
922 if (!strstr(insn->func->name, ".cold") &&
923 strstr(insn->jump_dest->func->name, ".cold")) {
924 insn->func->cfunc = insn->jump_dest->func;
925 insn->jump_dest->func->pfunc = insn->func;
926
927 } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
928 insn->jump_dest->offset == insn->jump_dest->func->offset) {
929
930 /* internal sibling call (without reloc) */
931 insn->call_dest = insn->jump_dest->func;
932 if (insn->call_dest->static_call_tramp) {
933 list_add_tail(&insn->call_node,
934 &file->static_call_list);
935 }
936 }
937 }
938 }
939
940 return 0;
941}
942
943static void remove_insn_ops(struct instruction *insn)
944{
945 struct stack_op *op, *tmp;
946
947 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
948 list_del(&op->list);
949 free(op);
950 }
951}
952
953static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
954{
955 struct symbol *call_dest;
956
957 call_dest = find_func_by_offset(sec, offset);
958 if (!call_dest)
959 call_dest = find_symbol_by_offset(sec, offset);
960
961 return call_dest;
962}
963
964/*
965 * Find the destination instructions for all calls.
966 */
967static int add_call_destinations(struct objtool_file *file)
968{
969 struct instruction *insn;
970 unsigned long dest_off;
971 struct reloc *reloc;
972
973 for_each_insn(file, insn) {
974 if (insn->type != INSN_CALL)
975 continue;
976
977 reloc = insn_reloc(file, insn);
978 if (!reloc) {
979 dest_off = arch_jump_destination(insn);
980 insn->call_dest = find_call_destination(insn->sec, dest_off);
981
982 if (insn->ignore)
983 continue;
984
985 if (!insn->call_dest) {
986 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
987 return -1;
988 }
989
990 if (insn->func && insn->call_dest->type != STT_FUNC) {
991 WARN_FUNC("unsupported call to non-function",
992 insn->sec, insn->offset);
993 return -1;
994 }
995
996 } else if (reloc->sym->type == STT_SECTION) {
997 dest_off = arch_dest_reloc_offset(reloc->addend);
998 insn->call_dest = find_call_destination(reloc->sym->sec,
999 dest_off);
1000 if (!insn->call_dest) {
1001 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1002 insn->sec, insn->offset,
1003 reloc->sym->sec->name,
1004 dest_off);
1005 return -1;
1006 }
1007
1008 } else if (arch_is_retpoline(reloc->sym)) {
1009 /*
1010 * Retpoline calls are really dynamic calls in
1011 * disguise, so convert them accordingly.
1012 */
1013 insn->type = INSN_CALL_DYNAMIC;
1014 insn->retpoline_safe = true;
1015
1016 list_add_tail(&insn->call_node,
1017 &file->retpoline_call_list);
1018
1019 remove_insn_ops(insn);
1020 continue;
1021
1022 } else
1023 insn->call_dest = reloc->sym;
1024
1025 if (insn->call_dest && insn->call_dest->static_call_tramp) {
1026 list_add_tail(&insn->call_node,
1027 &file->static_call_list);
1028 }
1029
1030 /*
1031 * Many compilers cannot disable KCOV with a function attribute
1032 * so they need a little help, NOP out any KCOV calls from noinstr
1033 * text.
1034 */
1035 if (insn->sec->noinstr &&
1036 !strncmp(insn->call_dest->name, "__sanitizer_cov_", 16)) {
1037 if (reloc) {
1038 reloc->type = R_NONE;
1039 elf_write_reloc(file->elf, reloc);
1040 }
1041
1042 elf_write_insn(file->elf, insn->sec,
1043 insn->offset, insn->len,
1044 arch_nop_insn(insn->len));
1045 insn->type = INSN_NOP;
1046 }
1047
1048 if (mcount && !strcmp(insn->call_dest->name, "__fentry__")) {
1049 if (reloc) {
1050 reloc->type = R_NONE;
1051 elf_write_reloc(file->elf, reloc);
1052 }
1053
1054 elf_write_insn(file->elf, insn->sec,
1055 insn->offset, insn->len,
1056 arch_nop_insn(insn->len));
1057
1058 insn->type = INSN_NOP;
1059
1060 list_add_tail(&insn->mcount_loc_node,
1061 &file->mcount_loc_list);
1062 }
1063
1064 /*
1065 * Whatever stack impact regular CALLs have, should be undone
1066 * by the RETURN of the called function.
1067 *
1068 * Annotated intra-function calls retain the stack_ops but
1069 * are converted to JUMP, see read_intra_function_calls().
1070 */
1071 remove_insn_ops(insn);
1072 }
1073
1074 return 0;
1075}
1076
1077/*
1078 * The .alternatives section requires some extra special care over and above
1079 * other special sections because alternatives are patched in place.
1080 */
1081static int handle_group_alt(struct objtool_file *file,
1082 struct special_alt *special_alt,
1083 struct instruction *orig_insn,
1084 struct instruction **new_insn)
1085{
1086 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1087 struct alt_group *orig_alt_group, *new_alt_group;
1088 unsigned long dest_off;
1089
1090
1091 orig_alt_group = malloc(sizeof(*orig_alt_group));
1092 if (!orig_alt_group) {
1093 WARN("malloc failed");
1094 return -1;
1095 }
1096 orig_alt_group->cfi = calloc(special_alt->orig_len,
1097 sizeof(struct cfi_state *));
1098 if (!orig_alt_group->cfi) {
1099 WARN("calloc failed");
1100 return -1;
1101 }
1102
1103 last_orig_insn = NULL;
1104 insn = orig_insn;
1105 sec_for_each_insn_from(file, insn) {
1106 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1107 break;
1108
1109 insn->alt_group = orig_alt_group;
1110 last_orig_insn = insn;
1111 }
1112 orig_alt_group->orig_group = NULL;
1113 orig_alt_group->first_insn = orig_insn;
1114 orig_alt_group->last_insn = last_orig_insn;
1115
1116
1117 new_alt_group = malloc(sizeof(*new_alt_group));
1118 if (!new_alt_group) {
1119 WARN("malloc failed");
1120 return -1;
1121 }
1122
1123 if (special_alt->new_len < special_alt->orig_len) {
1124 /*
1125 * Insert a fake nop at the end to make the replacement
1126 * alt_group the same size as the original. This is needed to
1127 * allow propagate_alt_cfi() to do its magic. When the last
1128 * instruction affects the stack, the instruction after it (the
1129 * nop) will propagate the new state to the shared CFI array.
1130 */
1131 nop = malloc(sizeof(*nop));
1132 if (!nop) {
1133 WARN("malloc failed");
1134 return -1;
1135 }
1136 memset(nop, 0, sizeof(*nop));
1137 INIT_LIST_HEAD(&nop->alts);
1138 INIT_LIST_HEAD(&nop->stack_ops);
1139 init_cfi_state(&nop->cfi);
1140
1141 nop->sec = special_alt->new_sec;
1142 nop->offset = special_alt->new_off + special_alt->new_len;
1143 nop->len = special_alt->orig_len - special_alt->new_len;
1144 nop->type = INSN_NOP;
1145 nop->func = orig_insn->func;
1146 nop->alt_group = new_alt_group;
1147 nop->ignore = orig_insn->ignore_alts;
1148 }
1149
1150 if (!special_alt->new_len) {
1151 *new_insn = nop;
1152 goto end;
1153 }
1154
1155 insn = *new_insn;
1156 sec_for_each_insn_from(file, insn) {
1157 struct reloc *alt_reloc;
1158
1159 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1160 break;
1161
1162 last_new_insn = insn;
1163
1164 insn->ignore = orig_insn->ignore_alts;
1165 insn->func = orig_insn->func;
1166 insn->alt_group = new_alt_group;
1167
1168 /*
1169 * Since alternative replacement code is copy/pasted by the
1170 * kernel after applying relocations, generally such code can't
1171 * have relative-address relocation references to outside the
1172 * .altinstr_replacement section, unless the arch's
1173 * alternatives code can adjust the relative offsets
1174 * accordingly.
1175 */
1176 alt_reloc = insn_reloc(file, insn);
1177 if (alt_reloc &&
1178 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1179
1180 WARN_FUNC("unsupported relocation in alternatives section",
1181 insn->sec, insn->offset);
1182 return -1;
1183 }
1184
1185 if (!is_static_jump(insn))
1186 continue;
1187
1188 if (!insn->immediate)
1189 continue;
1190
1191 dest_off = arch_jump_destination(insn);
1192 if (dest_off == special_alt->new_off + special_alt->new_len)
1193 insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1194
1195 if (!insn->jump_dest) {
1196 WARN_FUNC("can't find alternative jump destination",
1197 insn->sec, insn->offset);
1198 return -1;
1199 }
1200 }
1201
1202 if (!last_new_insn) {
1203 WARN_FUNC("can't find last new alternative instruction",
1204 special_alt->new_sec, special_alt->new_off);
1205 return -1;
1206 }
1207
1208 if (nop)
1209 list_add(&nop->list, &last_new_insn->list);
1210end:
1211 new_alt_group->orig_group = orig_alt_group;
1212 new_alt_group->first_insn = *new_insn;
1213 new_alt_group->last_insn = nop ? : last_new_insn;
1214 new_alt_group->cfi = orig_alt_group->cfi;
1215 return 0;
1216}
1217
1218/*
1219 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1220 * If the original instruction is a jump, make the alt entry an effective nop
1221 * by just skipping the original instruction.
1222 */
1223static int handle_jump_alt(struct objtool_file *file,
1224 struct special_alt *special_alt,
1225 struct instruction *orig_insn,
1226 struct instruction **new_insn)
1227{
1228 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1229 orig_insn->type != INSN_NOP) {
1230
1231 WARN_FUNC("unsupported instruction at jump label",
1232 orig_insn->sec, orig_insn->offset);
1233 return -1;
1234 }
1235
1236 if (special_alt->key_addend & 2) {
1237 struct reloc *reloc = insn_reloc(file, orig_insn);
1238
1239 if (reloc) {
1240 reloc->type = R_NONE;
1241 elf_write_reloc(file->elf, reloc);
1242 }
1243 elf_write_insn(file->elf, orig_insn->sec,
1244 orig_insn->offset, orig_insn->len,
1245 arch_nop_insn(orig_insn->len));
1246 orig_insn->type = INSN_NOP;
1247 }
1248
1249 if (orig_insn->type == INSN_NOP) {
1250 if (orig_insn->len == 2)
1251 file->jl_nop_short++;
1252 else
1253 file->jl_nop_long++;
1254
1255 return 0;
1256 }
1257
1258 if (orig_insn->len == 2)
1259 file->jl_short++;
1260 else
1261 file->jl_long++;
1262
1263 *new_insn = list_next_entry(orig_insn, list);
1264 return 0;
1265}
1266
1267/*
1268 * Read all the special sections which have alternate instructions which can be
1269 * patched in or redirected to at runtime. Each instruction having alternate
1270 * instruction(s) has them added to its insn->alts list, which will be
1271 * traversed in validate_branch().
1272 */
1273static int add_special_section_alts(struct objtool_file *file)
1274{
1275 struct list_head special_alts;
1276 struct instruction *orig_insn, *new_insn;
1277 struct special_alt *special_alt, *tmp;
1278 struct alternative *alt;
1279 int ret;
1280
1281 ret = special_get_alts(file->elf, &special_alts);
1282 if (ret)
1283 return ret;
1284
1285 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1286
1287 orig_insn = find_insn(file, special_alt->orig_sec,
1288 special_alt->orig_off);
1289 if (!orig_insn) {
1290 WARN_FUNC("special: can't find orig instruction",
1291 special_alt->orig_sec, special_alt->orig_off);
1292 ret = -1;
1293 goto out;
1294 }
1295
1296 new_insn = NULL;
1297 if (!special_alt->group || special_alt->new_len) {
1298 new_insn = find_insn(file, special_alt->new_sec,
1299 special_alt->new_off);
1300 if (!new_insn) {
1301 WARN_FUNC("special: can't find new instruction",
1302 special_alt->new_sec,
1303 special_alt->new_off);
1304 ret = -1;
1305 goto out;
1306 }
1307 }
1308
1309 if (special_alt->group) {
1310 if (!special_alt->orig_len) {
1311 WARN_FUNC("empty alternative entry",
1312 orig_insn->sec, orig_insn->offset);
1313 continue;
1314 }
1315
1316 ret = handle_group_alt(file, special_alt, orig_insn,
1317 &new_insn);
1318 if (ret)
1319 goto out;
1320 } else if (special_alt->jump_or_nop) {
1321 ret = handle_jump_alt(file, special_alt, orig_insn,
1322 &new_insn);
1323 if (ret)
1324 goto out;
1325 }
1326
1327 alt = malloc(sizeof(*alt));
1328 if (!alt) {
1329 WARN("malloc failed");
1330 ret = -1;
1331 goto out;
1332 }
1333
1334 alt->insn = new_insn;
1335 alt->skip_orig = special_alt->skip_orig;
1336 orig_insn->ignore_alts |= special_alt->skip_alt;
1337 list_add_tail(&alt->list, &orig_insn->alts);
1338
1339 list_del(&special_alt->list);
1340 free(special_alt);
1341 }
1342
1343 if (stats) {
1344 printf("jl\\\tNOP\tJMP\n");
1345 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1346 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1347 }
1348
1349out:
1350 return ret;
1351}
1352
1353static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1354 struct reloc *table)
1355{
1356 struct reloc *reloc = table;
1357 struct instruction *dest_insn;
1358 struct alternative *alt;
1359 struct symbol *pfunc = insn->func->pfunc;
1360 unsigned int prev_offset = 0;
1361
1362 /*
1363 * Each @reloc is a switch table relocation which points to the target
1364 * instruction.
1365 */
1366 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1367
1368 /* Check for the end of the table: */
1369 if (reloc != table && reloc->jump_table_start)
1370 break;
1371
1372 /* Make sure the table entries are consecutive: */
1373 if (prev_offset && reloc->offset != prev_offset + 8)
1374 break;
1375
1376 /* Detect function pointers from contiguous objects: */
1377 if (reloc->sym->sec == pfunc->sec &&
1378 reloc->addend == pfunc->offset)
1379 break;
1380
1381 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1382 if (!dest_insn)
1383 break;
1384
1385 /* Make sure the destination is in the same function: */
1386 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1387 break;
1388
1389 alt = malloc(sizeof(*alt));
1390 if (!alt) {
1391 WARN("malloc failed");
1392 return -1;
1393 }
1394
1395 alt->insn = dest_insn;
1396 list_add_tail(&alt->list, &insn->alts);
1397 prev_offset = reloc->offset;
1398 }
1399
1400 if (!prev_offset) {
1401 WARN_FUNC("can't find switch jump table",
1402 insn->sec, insn->offset);
1403 return -1;
1404 }
1405
1406 return 0;
1407}
1408
1409/*
1410 * find_jump_table() - Given a dynamic jump, find the switch jump table
1411 * associated with it.
1412 */
1413static struct reloc *find_jump_table(struct objtool_file *file,
1414 struct symbol *func,
1415 struct instruction *insn)
1416{
1417 struct reloc *table_reloc;
1418 struct instruction *dest_insn, *orig_insn = insn;
1419
1420 /*
1421 * Backward search using the @first_jump_src links, these help avoid
1422 * much of the 'in between' code. Which avoids us getting confused by
1423 * it.
1424 */
1425 for (;
1426 insn && insn->func && insn->func->pfunc == func;
1427 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1428
1429 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1430 break;
1431
1432 /* allow small jumps within the range */
1433 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1434 insn->jump_dest &&
1435 (insn->jump_dest->offset <= insn->offset ||
1436 insn->jump_dest->offset > orig_insn->offset))
1437 break;
1438
1439 table_reloc = arch_find_switch_table(file, insn);
1440 if (!table_reloc)
1441 continue;
1442 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1443 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1444 continue;
1445
1446 return table_reloc;
1447 }
1448
1449 return NULL;
1450}
1451
1452/*
1453 * First pass: Mark the head of each jump table so that in the next pass,
1454 * we know when a given jump table ends and the next one starts.
1455 */
1456static void mark_func_jump_tables(struct objtool_file *file,
1457 struct symbol *func)
1458{
1459 struct instruction *insn, *last = NULL;
1460 struct reloc *reloc;
1461
1462 func_for_each_insn(file, func, insn) {
1463 if (!last)
1464 last = insn;
1465
1466 /*
1467 * Store back-pointers for unconditional forward jumps such
1468 * that find_jump_table() can back-track using those and
1469 * avoid some potentially confusing code.
1470 */
1471 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1472 insn->offset > last->offset &&
1473 insn->jump_dest->offset > insn->offset &&
1474 !insn->jump_dest->first_jump_src) {
1475
1476 insn->jump_dest->first_jump_src = insn;
1477 last = insn->jump_dest;
1478 }
1479
1480 if (insn->type != INSN_JUMP_DYNAMIC)
1481 continue;
1482
1483 reloc = find_jump_table(file, func, insn);
1484 if (reloc) {
1485 reloc->jump_table_start = true;
1486 insn->jump_table = reloc;
1487 }
1488 }
1489}
1490
1491static int add_func_jump_tables(struct objtool_file *file,
1492 struct symbol *func)
1493{
1494 struct instruction *insn;
1495 int ret;
1496
1497 func_for_each_insn(file, func, insn) {
1498 if (!insn->jump_table)
1499 continue;
1500
1501 ret = add_jump_table(file, insn, insn->jump_table);
1502 if (ret)
1503 return ret;
1504 }
1505
1506 return 0;
1507}
1508
1509/*
1510 * For some switch statements, gcc generates a jump table in the .rodata
1511 * section which contains a list of addresses within the function to jump to.
1512 * This finds these jump tables and adds them to the insn->alts lists.
1513 */
1514static int add_jump_table_alts(struct objtool_file *file)
1515{
1516 struct section *sec;
1517 struct symbol *func;
1518 int ret;
1519
1520 if (!file->rodata)
1521 return 0;
1522
1523 for_each_sec(file, sec) {
1524 list_for_each_entry(func, &sec->symbol_list, list) {
1525 if (func->type != STT_FUNC)
1526 continue;
1527
1528 mark_func_jump_tables(file, func);
1529 ret = add_func_jump_tables(file, func);
1530 if (ret)
1531 return ret;
1532 }
1533 }
1534
1535 return 0;
1536}
1537
1538static void set_func_state(struct cfi_state *state)
1539{
1540 state->cfa = initial_func_cfi.cfa;
1541 memcpy(&state->regs, &initial_func_cfi.regs,
1542 CFI_NUM_REGS * sizeof(struct cfi_reg));
1543 state->stack_size = initial_func_cfi.cfa.offset;
1544}
1545
1546static int read_unwind_hints(struct objtool_file *file)
1547{
1548 struct section *sec, *relocsec;
1549 struct reloc *reloc;
1550 struct unwind_hint *hint;
1551 struct instruction *insn;
1552 int i;
1553
1554 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1555 if (!sec)
1556 return 0;
1557
1558 relocsec = sec->reloc;
1559 if (!relocsec) {
1560 WARN("missing .rela.discard.unwind_hints section");
1561 return -1;
1562 }
1563
1564 if (sec->len % sizeof(struct unwind_hint)) {
1565 WARN("struct unwind_hint size mismatch");
1566 return -1;
1567 }
1568
1569 file->hints = true;
1570
1571 for (i = 0; i < sec->len / sizeof(struct unwind_hint); i++) {
1572 hint = (struct unwind_hint *)sec->data->d_buf + i;
1573
1574 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1575 if (!reloc) {
1576 WARN("can't find reloc for unwind_hints[%d]", i);
1577 return -1;
1578 }
1579
1580 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1581 if (!insn) {
1582 WARN("can't find insn for unwind_hints[%d]", i);
1583 return -1;
1584 }
1585
1586 insn->hint = true;
1587
1588 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1589 set_func_state(&insn->cfi);
1590 continue;
1591 }
1592
1593 if (arch_decode_hint_reg(insn, hint->sp_reg)) {
1594 WARN_FUNC("unsupported unwind_hint sp base reg %d",
1595 insn->sec, insn->offset, hint->sp_reg);
1596 return -1;
1597 }
1598
1599 insn->cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1600 insn->cfi.type = hint->type;
1601 insn->cfi.end = hint->end;
1602 }
1603
1604 return 0;
1605}
1606
1607static int read_retpoline_hints(struct objtool_file *file)
1608{
1609 struct section *sec;
1610 struct instruction *insn;
1611 struct reloc *reloc;
1612
1613 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
1614 if (!sec)
1615 return 0;
1616
1617 list_for_each_entry(reloc, &sec->reloc_list, list) {
1618 if (reloc->sym->type != STT_SECTION) {
1619 WARN("unexpected relocation symbol type in %s", sec->name);
1620 return -1;
1621 }
1622
1623 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1624 if (!insn) {
1625 WARN("bad .discard.retpoline_safe entry");
1626 return -1;
1627 }
1628
1629 if (insn->type != INSN_JUMP_DYNAMIC &&
1630 insn->type != INSN_CALL_DYNAMIC) {
1631 WARN_FUNC("retpoline_safe hint not an indirect jump/call",
1632 insn->sec, insn->offset);
1633 return -1;
1634 }
1635
1636 insn->retpoline_safe = true;
1637 }
1638
1639 return 0;
1640}
1641
1642static int read_instr_hints(struct objtool_file *file)
1643{
1644 struct section *sec;
1645 struct instruction *insn;
1646 struct reloc *reloc;
1647
1648 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1649 if (!sec)
1650 return 0;
1651
1652 list_for_each_entry(reloc, &sec->reloc_list, list) {
1653 if (reloc->sym->type != STT_SECTION) {
1654 WARN("unexpected relocation symbol type in %s", sec->name);
1655 return -1;
1656 }
1657
1658 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1659 if (!insn) {
1660 WARN("bad .discard.instr_end entry");
1661 return -1;
1662 }
1663
1664 insn->instr--;
1665 }
1666
1667 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1668 if (!sec)
1669 return 0;
1670
1671 list_for_each_entry(reloc, &sec->reloc_list, list) {
1672 if (reloc->sym->type != STT_SECTION) {
1673 WARN("unexpected relocation symbol type in %s", sec->name);
1674 return -1;
1675 }
1676
1677 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1678 if (!insn) {
1679 WARN("bad .discard.instr_begin entry");
1680 return -1;
1681 }
1682
1683 insn->instr++;
1684 }
1685
1686 return 0;
1687}
1688
1689static int read_intra_function_calls(struct objtool_file *file)
1690{
1691 struct instruction *insn;
1692 struct section *sec;
1693 struct reloc *reloc;
1694
1695 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1696 if (!sec)
1697 return 0;
1698
1699 list_for_each_entry(reloc, &sec->reloc_list, list) {
1700 unsigned long dest_off;
1701
1702 if (reloc->sym->type != STT_SECTION) {
1703 WARN("unexpected relocation symbol type in %s",
1704 sec->name);
1705 return -1;
1706 }
1707
1708 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1709 if (!insn) {
1710 WARN("bad .discard.intra_function_call entry");
1711 return -1;
1712 }
1713
1714 if (insn->type != INSN_CALL) {
1715 WARN_FUNC("intra_function_call not a direct call",
1716 insn->sec, insn->offset);
1717 return -1;
1718 }
1719
1720 /*
1721 * Treat intra-function CALLs as JMPs, but with a stack_op.
1722 * See add_call_destinations(), which strips stack_ops from
1723 * normal CALLs.
1724 */
1725 insn->type = INSN_JUMP_UNCONDITIONAL;
1726
1727 dest_off = insn->offset + insn->len + insn->immediate;
1728 insn->jump_dest = find_insn(file, insn->sec, dest_off);
1729 if (!insn->jump_dest) {
1730 WARN_FUNC("can't find call dest at %s+0x%lx",
1731 insn->sec, insn->offset,
1732 insn->sec->name, dest_off);
1733 return -1;
1734 }
1735 }
1736
1737 return 0;
1738}
1739
1740static int read_static_call_tramps(struct objtool_file *file)
1741{
1742 struct section *sec;
1743 struct symbol *func;
1744
1745 for_each_sec(file, sec) {
1746 list_for_each_entry(func, &sec->symbol_list, list) {
1747 if (func->bind == STB_GLOBAL &&
1748 !strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
1749 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
1750 func->static_call_tramp = true;
1751 }
1752 }
1753
1754 return 0;
1755}
1756
1757static void mark_rodata(struct objtool_file *file)
1758{
1759 struct section *sec;
1760 bool found = false;
1761
1762 /*
1763 * Search for the following rodata sections, each of which can
1764 * potentially contain jump tables:
1765 *
1766 * - .rodata: can contain GCC switch tables
1767 * - .rodata.<func>: same, if -fdata-sections is being used
1768 * - .rodata..c_jump_table: contains C annotated jump tables
1769 *
1770 * .rodata.str1.* sections are ignored; they don't contain jump tables.
1771 */
1772 for_each_sec(file, sec) {
1773 if (!strncmp(sec->name, ".rodata", 7) &&
1774 !strstr(sec->name, ".str1.")) {
1775 sec->rodata = true;
1776 found = true;
1777 }
1778 }
1779
1780 file->rodata = found;
1781}
1782
1783__weak int arch_rewrite_retpolines(struct objtool_file *file)
1784{
1785 return 0;
1786}
1787
1788static int decode_sections(struct objtool_file *file)
1789{
1790 int ret;
1791
1792 mark_rodata(file);
1793
1794 ret = decode_instructions(file);
1795 if (ret)
1796 return ret;
1797
1798 ret = add_dead_ends(file);
1799 if (ret)
1800 return ret;
1801
1802 add_ignores(file);
1803 add_uaccess_safe(file);
1804
1805 ret = add_ignore_alternatives(file);
1806 if (ret)
1807 return ret;
1808
1809 /*
1810 * Must be before add_{jump_call}_destination.
1811 */
1812 ret = read_static_call_tramps(file);
1813 if (ret)
1814 return ret;
1815
1816 /*
1817 * Must be before add_special_section_alts() as that depends on
1818 * jump_dest being set.
1819 */
1820 ret = add_jump_destinations(file);
1821 if (ret)
1822 return ret;
1823
1824 ret = add_special_section_alts(file);
1825 if (ret)
1826 return ret;
1827
1828 /*
1829 * Must be before add_call_destination(); it changes INSN_CALL to
1830 * INSN_JUMP.
1831 */
1832 ret = read_intra_function_calls(file);
1833 if (ret)
1834 return ret;
1835
1836 ret = add_call_destinations(file);
1837 if (ret)
1838 return ret;
1839
1840 ret = add_jump_table_alts(file);
1841 if (ret)
1842 return ret;
1843
1844 ret = read_unwind_hints(file);
1845 if (ret)
1846 return ret;
1847
1848 ret = read_retpoline_hints(file);
1849 if (ret)
1850 return ret;
1851
1852 ret = read_instr_hints(file);
1853 if (ret)
1854 return ret;
1855
1856 /*
1857 * Must be after add_special_section_alts(), since this will emit
1858 * alternatives. Must be after add_{jump,call}_destination(), since
1859 * those create the call insn lists.
1860 */
1861 ret = arch_rewrite_retpolines(file);
1862 if (ret)
1863 return ret;
1864
1865 return 0;
1866}
1867
1868static bool is_fentry_call(struct instruction *insn)
1869{
1870 if (insn->type == INSN_CALL && insn->call_dest &&
1871 insn->call_dest->type == STT_NOTYPE &&
1872 !strcmp(insn->call_dest->name, "__fentry__"))
1873 return true;
1874
1875 return false;
1876}
1877
1878static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
1879{
1880 struct cfi_state *cfi = &state->cfi;
1881 int i;
1882
1883 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
1884 return true;
1885
1886 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
1887 return true;
1888
1889 if (cfi->stack_size != initial_func_cfi.cfa.offset)
1890 return true;
1891
1892 for (i = 0; i < CFI_NUM_REGS; i++) {
1893 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
1894 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
1895 return true;
1896 }
1897
1898 return false;
1899}
1900
1901static bool check_reg_frame_pos(const struct cfi_reg *reg,
1902 int expected_offset)
1903{
1904 return reg->base == CFI_CFA &&
1905 reg->offset == expected_offset;
1906}
1907
1908static bool has_valid_stack_frame(struct insn_state *state)
1909{
1910 struct cfi_state *cfi = &state->cfi;
1911
1912 if (cfi->cfa.base == CFI_BP &&
1913 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
1914 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
1915 return true;
1916
1917 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
1918 return true;
1919
1920 return false;
1921}
1922
1923static int update_cfi_state_regs(struct instruction *insn,
1924 struct cfi_state *cfi,
1925 struct stack_op *op)
1926{
1927 struct cfi_reg *cfa = &cfi->cfa;
1928
1929 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
1930 return 0;
1931
1932 /* push */
1933 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
1934 cfa->offset += 8;
1935
1936 /* pop */
1937 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
1938 cfa->offset -= 8;
1939
1940 /* add immediate to sp */
1941 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
1942 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
1943 cfa->offset -= op->src.offset;
1944
1945 return 0;
1946}
1947
1948static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
1949{
1950 if (arch_callee_saved_reg(reg) &&
1951 cfi->regs[reg].base == CFI_UNDEFINED) {
1952 cfi->regs[reg].base = base;
1953 cfi->regs[reg].offset = offset;
1954 }
1955}
1956
1957static void restore_reg(struct cfi_state *cfi, unsigned char reg)
1958{
1959 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
1960 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
1961}
1962
1963/*
1964 * A note about DRAP stack alignment:
1965 *
1966 * GCC has the concept of a DRAP register, which is used to help keep track of
1967 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
1968 * register. The typical DRAP pattern is:
1969 *
1970 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
1971 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
1972 * 41 ff 72 f8 pushq -0x8(%r10)
1973 * 55 push %rbp
1974 * 48 89 e5 mov %rsp,%rbp
1975 * (more pushes)
1976 * 41 52 push %r10
1977 * ...
1978 * 41 5a pop %r10
1979 * (more pops)
1980 * 5d pop %rbp
1981 * 49 8d 62 f8 lea -0x8(%r10),%rsp
1982 * c3 retq
1983 *
1984 * There are some variations in the epilogues, like:
1985 *
1986 * 5b pop %rbx
1987 * 41 5a pop %r10
1988 * 41 5c pop %r12
1989 * 41 5d pop %r13
1990 * 41 5e pop %r14
1991 * c9 leaveq
1992 * 49 8d 62 f8 lea -0x8(%r10),%rsp
1993 * c3 retq
1994 *
1995 * and:
1996 *
1997 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
1998 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
1999 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2000 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2001 * c9 leaveq
2002 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2003 * c3 retq
2004 *
2005 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2006 * restored beforehand:
2007 *
2008 * 41 55 push %r13
2009 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2010 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2011 * ...
2012 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2013 * 41 5d pop %r13
2014 * c3 retq
2015 */
2016static int update_cfi_state(struct instruction *insn,
2017 struct instruction *next_insn,
2018 struct cfi_state *cfi, struct stack_op *op)
2019{
2020 struct cfi_reg *cfa = &cfi->cfa;
2021 struct cfi_reg *regs = cfi->regs;
2022
2023 /* stack operations don't make sense with an undefined CFA */
2024 if (cfa->base == CFI_UNDEFINED) {
2025 if (insn->func) {
2026 WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2027 return -1;
2028 }
2029 return 0;
2030 }
2031
2032 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2033 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2034 return update_cfi_state_regs(insn, cfi, op);
2035
2036 switch (op->dest.type) {
2037
2038 case OP_DEST_REG:
2039 switch (op->src.type) {
2040
2041 case OP_SRC_REG:
2042 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2043 cfa->base == CFI_SP &&
2044 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2045
2046 /* mov %rsp, %rbp */
2047 cfa->base = op->dest.reg;
2048 cfi->bp_scratch = false;
2049 }
2050
2051 else if (op->src.reg == CFI_SP &&
2052 op->dest.reg == CFI_BP && cfi->drap) {
2053
2054 /* drap: mov %rsp, %rbp */
2055 regs[CFI_BP].base = CFI_BP;
2056 regs[CFI_BP].offset = -cfi->stack_size;
2057 cfi->bp_scratch = false;
2058 }
2059
2060 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2061
2062 /*
2063 * mov %rsp, %reg
2064 *
2065 * This is needed for the rare case where GCC
2066 * does:
2067 *
2068 * mov %rsp, %rax
2069 * ...
2070 * mov %rax, %rsp
2071 */
2072 cfi->vals[op->dest.reg].base = CFI_CFA;
2073 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2074 }
2075
2076 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2077 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2078
2079 /*
2080 * mov %rbp, %rsp
2081 *
2082 * Restore the original stack pointer (Clang).
2083 */
2084 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2085 }
2086
2087 else if (op->dest.reg == cfa->base) {
2088
2089 /* mov %reg, %rsp */
2090 if (cfa->base == CFI_SP &&
2091 cfi->vals[op->src.reg].base == CFI_CFA) {
2092
2093 /*
2094 * This is needed for the rare case
2095 * where GCC does something dumb like:
2096 *
2097 * lea 0x8(%rsp), %rcx
2098 * ...
2099 * mov %rcx, %rsp
2100 */
2101 cfa->offset = -cfi->vals[op->src.reg].offset;
2102 cfi->stack_size = cfa->offset;
2103
2104 } else if (cfa->base == CFI_SP &&
2105 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2106 cfi->vals[op->src.reg].offset == cfa->offset) {
2107
2108 /*
2109 * Stack swizzle:
2110 *
2111 * 1: mov %rsp, (%[tos])
2112 * 2: mov %[tos], %rsp
2113 * ...
2114 * 3: pop %rsp
2115 *
2116 * Where:
2117 *
2118 * 1 - places a pointer to the previous
2119 * stack at the Top-of-Stack of the
2120 * new stack.
2121 *
2122 * 2 - switches to the new stack.
2123 *
2124 * 3 - pops the Top-of-Stack to restore
2125 * the original stack.
2126 *
2127 * Note: we set base to SP_INDIRECT
2128 * here and preserve offset. Therefore
2129 * when the unwinder reaches ToS it
2130 * will dereference SP and then add the
2131 * offset to find the next frame, IOW:
2132 * (%rsp) + offset.
2133 */
2134 cfa->base = CFI_SP_INDIRECT;
2135
2136 } else {
2137 cfa->base = CFI_UNDEFINED;
2138 cfa->offset = 0;
2139 }
2140 }
2141
2142 else if (op->dest.reg == CFI_SP &&
2143 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2144 cfi->vals[op->src.reg].offset == cfa->offset) {
2145
2146 /*
2147 * The same stack swizzle case 2) as above. But
2148 * because we can't change cfa->base, case 3)
2149 * will become a regular POP. Pretend we're a
2150 * PUSH so things don't go unbalanced.
2151 */
2152 cfi->stack_size += 8;
2153 }
2154
2155
2156 break;
2157
2158 case OP_SRC_ADD:
2159 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2160
2161 /* add imm, %rsp */
2162 cfi->stack_size -= op->src.offset;
2163 if (cfa->base == CFI_SP)
2164 cfa->offset -= op->src.offset;
2165 break;
2166 }
2167
2168 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2169
2170 /* lea disp(%rbp), %rsp */
2171 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2172 break;
2173 }
2174
2175 if (!cfi->drap && op->src.reg == CFI_SP &&
2176 op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2177 check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) {
2178
2179 /* lea disp(%rsp), %rbp */
2180 cfa->base = CFI_BP;
2181 cfa->offset -= op->src.offset;
2182 cfi->bp_scratch = false;
2183 break;
2184 }
2185
2186 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2187
2188 /* drap: lea disp(%rsp), %drap */
2189 cfi->drap_reg = op->dest.reg;
2190
2191 /*
2192 * lea disp(%rsp), %reg
2193 *
2194 * This is needed for the rare case where GCC
2195 * does something dumb like:
2196 *
2197 * lea 0x8(%rsp), %rcx
2198 * ...
2199 * mov %rcx, %rsp
2200 */
2201 cfi->vals[op->dest.reg].base = CFI_CFA;
2202 cfi->vals[op->dest.reg].offset = \
2203 -cfi->stack_size + op->src.offset;
2204
2205 break;
2206 }
2207
2208 if (cfi->drap && op->dest.reg == CFI_SP &&
2209 op->src.reg == cfi->drap_reg) {
2210
2211 /* drap: lea disp(%drap), %rsp */
2212 cfa->base = CFI_SP;
2213 cfa->offset = cfi->stack_size = -op->src.offset;
2214 cfi->drap_reg = CFI_UNDEFINED;
2215 cfi->drap = false;
2216 break;
2217 }
2218
2219 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2220 WARN_FUNC("unsupported stack register modification",
2221 insn->sec, insn->offset);
2222 return -1;
2223 }
2224
2225 break;
2226
2227 case OP_SRC_AND:
2228 if (op->dest.reg != CFI_SP ||
2229 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2230 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2231 WARN_FUNC("unsupported stack pointer realignment",
2232 insn->sec, insn->offset);
2233 return -1;
2234 }
2235
2236 if (cfi->drap_reg != CFI_UNDEFINED) {
2237 /* drap: and imm, %rsp */
2238 cfa->base = cfi->drap_reg;
2239 cfa->offset = cfi->stack_size = 0;
2240 cfi->drap = true;
2241 }
2242
2243 /*
2244 * Older versions of GCC (4.8ish) realign the stack
2245 * without DRAP, with a frame pointer.
2246 */
2247
2248 break;
2249
2250 case OP_SRC_POP:
2251 case OP_SRC_POPF:
2252 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2253
2254 /* pop %rsp; # restore from a stack swizzle */
2255 cfa->base = CFI_SP;
2256 break;
2257 }
2258
2259 if (!cfi->drap && op->dest.reg == cfa->base) {
2260
2261 /* pop %rbp */
2262 cfa->base = CFI_SP;
2263 }
2264
2265 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2266 op->dest.reg == cfi->drap_reg &&
2267 cfi->drap_offset == -cfi->stack_size) {
2268
2269 /* drap: pop %drap */
2270 cfa->base = cfi->drap_reg;
2271 cfa->offset = 0;
2272 cfi->drap_offset = -1;
2273
2274 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2275
2276 /* pop %reg */
2277 restore_reg(cfi, op->dest.reg);
2278 }
2279
2280 cfi->stack_size -= 8;
2281 if (cfa->base == CFI_SP)
2282 cfa->offset -= 8;
2283
2284 break;
2285
2286 case OP_SRC_REG_INDIRECT:
2287 if (!cfi->drap && op->dest.reg == cfa->base &&
2288 op->dest.reg == CFI_BP) {
2289
2290 /* mov disp(%rsp), %rbp */
2291 cfa->base = CFI_SP;
2292 cfa->offset = cfi->stack_size;
2293 }
2294
2295 if (cfi->drap && op->src.reg == CFI_BP &&
2296 op->src.offset == cfi->drap_offset) {
2297
2298 /* drap: mov disp(%rbp), %drap */
2299 cfa->base = cfi->drap_reg;
2300 cfa->offset = 0;
2301 cfi->drap_offset = -1;
2302 }
2303
2304 if (cfi->drap && op->src.reg == CFI_BP &&
2305 op->src.offset == regs[op->dest.reg].offset) {
2306
2307 /* drap: mov disp(%rbp), %reg */
2308 restore_reg(cfi, op->dest.reg);
2309
2310 } else if (op->src.reg == cfa->base &&
2311 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2312
2313 /* mov disp(%rbp), %reg */
2314 /* mov disp(%rsp), %reg */
2315 restore_reg(cfi, op->dest.reg);
2316
2317 } else if (op->src.reg == CFI_SP &&
2318 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2319
2320 /* mov disp(%rsp), %reg */
2321 restore_reg(cfi, op->dest.reg);
2322 }
2323
2324 break;
2325
2326 default:
2327 WARN_FUNC("unknown stack-related instruction",
2328 insn->sec, insn->offset);
2329 return -1;
2330 }
2331
2332 break;
2333
2334 case OP_DEST_PUSH:
2335 case OP_DEST_PUSHF:
2336 cfi->stack_size += 8;
2337 if (cfa->base == CFI_SP)
2338 cfa->offset += 8;
2339
2340 if (op->src.type != OP_SRC_REG)
2341 break;
2342
2343 if (cfi->drap) {
2344 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2345
2346 /* drap: push %drap */
2347 cfa->base = CFI_BP_INDIRECT;
2348 cfa->offset = -cfi->stack_size;
2349
2350 /* save drap so we know when to restore it */
2351 cfi->drap_offset = -cfi->stack_size;
2352
2353 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2354
2355 /* drap: push %rbp */
2356 cfi->stack_size = 0;
2357
2358 } else {
2359
2360 /* drap: push %reg */
2361 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2362 }
2363
2364 } else {
2365
2366 /* push %reg */
2367 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2368 }
2369
2370 /* detect when asm code uses rbp as a scratch register */
2371 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
2372 cfa->base != CFI_BP)
2373 cfi->bp_scratch = true;
2374 break;
2375
2376 case OP_DEST_REG_INDIRECT:
2377
2378 if (cfi->drap) {
2379 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2380
2381 /* drap: mov %drap, disp(%rbp) */
2382 cfa->base = CFI_BP_INDIRECT;
2383 cfa->offset = op->dest.offset;
2384
2385 /* save drap offset so we know when to restore it */
2386 cfi->drap_offset = op->dest.offset;
2387 } else {
2388
2389 /* drap: mov reg, disp(%rbp) */
2390 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2391 }
2392
2393 } else if (op->dest.reg == cfa->base) {
2394
2395 /* mov reg, disp(%rbp) */
2396 /* mov reg, disp(%rsp) */
2397 save_reg(cfi, op->src.reg, CFI_CFA,
2398 op->dest.offset - cfi->cfa.offset);
2399
2400 } else if (op->dest.reg == CFI_SP) {
2401
2402 /* mov reg, disp(%rsp) */
2403 save_reg(cfi, op->src.reg, CFI_CFA,
2404 op->dest.offset - cfi->stack_size);
2405
2406 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2407
2408 /* mov %rsp, (%reg); # setup a stack swizzle. */
2409 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2410 cfi->vals[op->dest.reg].offset = cfa->offset;
2411 }
2412
2413 break;
2414
2415 case OP_DEST_MEM:
2416 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2417 WARN_FUNC("unknown stack-related memory operation",
2418 insn->sec, insn->offset);
2419 return -1;
2420 }
2421
2422 /* pop mem */
2423 cfi->stack_size -= 8;
2424 if (cfa->base == CFI_SP)
2425 cfa->offset -= 8;
2426
2427 break;
2428
2429 default:
2430 WARN_FUNC("unknown stack-related instruction",
2431 insn->sec, insn->offset);
2432 return -1;
2433 }
2434
2435 return 0;
2436}
2437
2438/*
2439 * The stack layouts of alternatives instructions can sometimes diverge when
2440 * they have stack modifications. That's fine as long as the potential stack
2441 * layouts don't conflict at any given potential instruction boundary.
2442 *
2443 * Flatten the CFIs of the different alternative code streams (both original
2444 * and replacement) into a single shared CFI array which can be used to detect
2445 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2446 */
2447static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2448{
2449 struct cfi_state **alt_cfi;
2450 int group_off;
2451
2452 if (!insn->alt_group)
2453 return 0;
2454
2455 alt_cfi = insn->alt_group->cfi;
2456 group_off = insn->offset - insn->alt_group->first_insn->offset;
2457
2458 if (!alt_cfi[group_off]) {
2459 alt_cfi[group_off] = &insn->cfi;
2460 } else {
2461 if (memcmp(alt_cfi[group_off], &insn->cfi, sizeof(struct cfi_state))) {
2462 WARN_FUNC("stack layout conflict in alternatives",
2463 insn->sec, insn->offset);
2464 return -1;
2465 }
2466 }
2467
2468 return 0;
2469}
2470
2471static int handle_insn_ops(struct instruction *insn,
2472 struct instruction *next_insn,
2473 struct insn_state *state)
2474{
2475 struct stack_op *op;
2476
2477 list_for_each_entry(op, &insn->stack_ops, list) {
2478
2479 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2480 return 1;
2481
2482 if (!insn->alt_group)
2483 continue;
2484
2485 if (op->dest.type == OP_DEST_PUSHF) {
2486 if (!state->uaccess_stack) {
2487 state->uaccess_stack = 1;
2488 } else if (state->uaccess_stack >> 31) {
2489 WARN_FUNC("PUSHF stack exhausted",
2490 insn->sec, insn->offset);
2491 return 1;
2492 }
2493 state->uaccess_stack <<= 1;
2494 state->uaccess_stack |= state->uaccess;
2495 }
2496
2497 if (op->src.type == OP_SRC_POPF) {
2498 if (state->uaccess_stack) {
2499 state->uaccess = state->uaccess_stack & 1;
2500 state->uaccess_stack >>= 1;
2501 if (state->uaccess_stack == 1)
2502 state->uaccess_stack = 0;
2503 }
2504 }
2505 }
2506
2507 return 0;
2508}
2509
2510static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
2511{
2512 struct cfi_state *cfi1 = &insn->cfi;
2513 int i;
2514
2515 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2516
2517 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2518 insn->sec, insn->offset,
2519 cfi1->cfa.base, cfi1->cfa.offset,
2520 cfi2->cfa.base, cfi2->cfa.offset);
2521
2522 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
2523 for (i = 0; i < CFI_NUM_REGS; i++) {
2524 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
2525 sizeof(struct cfi_reg)))
2526 continue;
2527
2528 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2529 insn->sec, insn->offset,
2530 i, cfi1->regs[i].base, cfi1->regs[i].offset,
2531 i, cfi2->regs[i].base, cfi2->regs[i].offset);
2532 break;
2533 }
2534
2535 } else if (cfi1->type != cfi2->type) {
2536
2537 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
2538 insn->sec, insn->offset, cfi1->type, cfi2->type);
2539
2540 } else if (cfi1->drap != cfi2->drap ||
2541 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2542 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
2543
2544 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
2545 insn->sec, insn->offset,
2546 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2547 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
2548
2549 } else
2550 return true;
2551
2552 return false;
2553}
2554
2555static inline bool func_uaccess_safe(struct symbol *func)
2556{
2557 if (func)
2558 return func->uaccess_safe;
2559
2560 return false;
2561}
2562
2563static inline const char *call_dest_name(struct instruction *insn)
2564{
2565 if (insn->call_dest)
2566 return insn->call_dest->name;
2567
2568 return "{dynamic}";
2569}
2570
2571static inline bool noinstr_call_dest(struct symbol *func)
2572{
2573 /*
2574 * We can't deal with indirect function calls at present;
2575 * assume they're instrumented.
2576 */
2577 if (!func)
2578 return false;
2579
2580 /*
2581 * If the symbol is from a noinstr section; we good.
2582 */
2583 if (func->sec->noinstr)
2584 return true;
2585
2586 /*
2587 * The __ubsan_handle_*() calls are like WARN(), they only happen when
2588 * something 'BAD' happened. At the risk of taking the machine down,
2589 * let them proceed to get the message out.
2590 */
2591 if (!strncmp(func->name, "__ubsan_handle_", 15))
2592 return true;
2593
2594 return false;
2595}
2596
2597static int validate_call(struct instruction *insn, struct insn_state *state)
2598{
2599 if (state->noinstr && state->instr <= 0 &&
2600 !noinstr_call_dest(insn->call_dest)) {
2601 WARN_FUNC("call to %s() leaves .noinstr.text section",
2602 insn->sec, insn->offset, call_dest_name(insn));
2603 return 1;
2604 }
2605
2606 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2607 WARN_FUNC("call to %s() with UACCESS enabled",
2608 insn->sec, insn->offset, call_dest_name(insn));
2609 return 1;
2610 }
2611
2612 if (state->df) {
2613 WARN_FUNC("call to %s() with DF set",
2614 insn->sec, insn->offset, call_dest_name(insn));
2615 return 1;
2616 }
2617
2618 return 0;
2619}
2620
2621static int validate_sibling_call(struct instruction *insn, struct insn_state *state)
2622{
2623 if (has_modified_stack_frame(insn, state)) {
2624 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2625 insn->sec, insn->offset);
2626 return 1;
2627 }
2628
2629 return validate_call(insn, state);
2630}
2631
2632static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2633{
2634 if (state->noinstr && state->instr > 0) {
2635 WARN_FUNC("return with instrumentation enabled",
2636 insn->sec, insn->offset);
2637 return 1;
2638 }
2639
2640 if (state->uaccess && !func_uaccess_safe(func)) {
2641 WARN_FUNC("return with UACCESS enabled",
2642 insn->sec, insn->offset);
2643 return 1;
2644 }
2645
2646 if (!state->uaccess && func_uaccess_safe(func)) {
2647 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2648 insn->sec, insn->offset);
2649 return 1;
2650 }
2651
2652 if (state->df) {
2653 WARN_FUNC("return with DF set",
2654 insn->sec, insn->offset);
2655 return 1;
2656 }
2657
2658 if (func && has_modified_stack_frame(insn, state)) {
2659 WARN_FUNC("return with modified stack frame",
2660 insn->sec, insn->offset);
2661 return 1;
2662 }
2663
2664 if (state->cfi.bp_scratch) {
2665 WARN_FUNC("BP used as a scratch register",
2666 insn->sec, insn->offset);
2667 return 1;
2668 }
2669
2670 return 0;
2671}
2672
2673static struct instruction *next_insn_to_validate(struct objtool_file *file,
2674 struct instruction *insn)
2675{
2676 struct alt_group *alt_group = insn->alt_group;
2677
2678 /*
2679 * Simulate the fact that alternatives are patched in-place. When the
2680 * end of a replacement alt_group is reached, redirect objtool flow to
2681 * the end of the original alt_group.
2682 */
2683 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2684 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2685
2686 return next_insn_same_sec(file, insn);
2687}
2688
2689/*
2690 * Follow the branch starting at the given instruction, and recursively follow
2691 * any other branches (jumps). Meanwhile, track the frame pointer state at
2692 * each instruction and validate all the rules described in
2693 * tools/objtool/Documentation/stack-validation.txt.
2694 */
2695static int validate_branch(struct objtool_file *file, struct symbol *func,
2696 struct instruction *insn, struct insn_state state)
2697{
2698 struct alternative *alt;
2699 struct instruction *next_insn;
2700 struct section *sec;
2701 u8 visited;
2702 int ret;
2703
2704 sec = insn->sec;
2705
2706 while (1) {
2707 next_insn = next_insn_to_validate(file, insn);
2708
2709 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
2710 WARN("%s() falls through to next function %s()",
2711 func->name, insn->func->name);
2712 return 1;
2713 }
2714
2715 if (func && insn->ignore) {
2716 WARN_FUNC("BUG: why am I validating an ignored function?",
2717 sec, insn->offset);
2718 return 1;
2719 }
2720
2721 visited = 1 << state.uaccess;
2722 if (insn->visited) {
2723 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
2724 return 1;
2725
2726 if (insn->visited & visited)
2727 return 0;
2728 }
2729
2730 if (state.noinstr)
2731 state.instr += insn->instr;
2732
2733 if (insn->hint)
2734 state.cfi = insn->cfi;
2735 else
2736 insn->cfi = state.cfi;
2737
2738 insn->visited |= visited;
2739
2740 if (propagate_alt_cfi(file, insn))
2741 return 1;
2742
2743 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
2744 bool skip_orig = false;
2745
2746 list_for_each_entry(alt, &insn->alts, list) {
2747 if (alt->skip_orig)
2748 skip_orig = true;
2749
2750 ret = validate_branch(file, func, alt->insn, state);
2751 if (ret) {
2752 if (backtrace)
2753 BT_FUNC("(alt)", insn);
2754 return ret;
2755 }
2756 }
2757
2758 if (skip_orig)
2759 return 0;
2760 }
2761
2762 if (handle_insn_ops(insn, next_insn, &state))
2763 return 1;
2764
2765 switch (insn->type) {
2766
2767 case INSN_RETURN:
2768 return validate_return(func, insn, &state);
2769
2770 case INSN_CALL:
2771 case INSN_CALL_DYNAMIC:
2772 ret = validate_call(insn, &state);
2773 if (ret)
2774 return ret;
2775
2776 if (!no_fp && func && !is_fentry_call(insn) &&
2777 !has_valid_stack_frame(&state)) {
2778 WARN_FUNC("call without frame pointer save/setup",
2779 sec, insn->offset);
2780 return 1;
2781 }
2782
2783 if (dead_end_function(file, insn->call_dest))
2784 return 0;
2785
2786 break;
2787
2788 case INSN_JUMP_CONDITIONAL:
2789 case INSN_JUMP_UNCONDITIONAL:
2790 if (is_sibling_call(insn)) {
2791 ret = validate_sibling_call(insn, &state);
2792 if (ret)
2793 return ret;
2794
2795 } else if (insn->jump_dest) {
2796 ret = validate_branch(file, func,
2797 insn->jump_dest, state);
2798 if (ret) {
2799 if (backtrace)
2800 BT_FUNC("(branch)", insn);
2801 return ret;
2802 }
2803 }
2804
2805 if (insn->type == INSN_JUMP_UNCONDITIONAL)
2806 return 0;
2807
2808 break;
2809
2810 case INSN_JUMP_DYNAMIC:
2811 case INSN_JUMP_DYNAMIC_CONDITIONAL:
2812 if (is_sibling_call(insn)) {
2813 ret = validate_sibling_call(insn, &state);
2814 if (ret)
2815 return ret;
2816 }
2817
2818 if (insn->type == INSN_JUMP_DYNAMIC)
2819 return 0;
2820
2821 break;
2822
2823 case INSN_CONTEXT_SWITCH:
2824 if (func && (!next_insn || !next_insn->hint)) {
2825 WARN_FUNC("unsupported instruction in callable function",
2826 sec, insn->offset);
2827 return 1;
2828 }
2829 return 0;
2830
2831 case INSN_STAC:
2832 if (state.uaccess) {
2833 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
2834 return 1;
2835 }
2836
2837 state.uaccess = true;
2838 break;
2839
2840 case INSN_CLAC:
2841 if (!state.uaccess && func) {
2842 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
2843 return 1;
2844 }
2845
2846 if (func_uaccess_safe(func) && !state.uaccess_stack) {
2847 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
2848 return 1;
2849 }
2850
2851 state.uaccess = false;
2852 break;
2853
2854 case INSN_STD:
2855 if (state.df) {
2856 WARN_FUNC("recursive STD", sec, insn->offset);
2857 return 1;
2858 }
2859
2860 state.df = true;
2861 break;
2862
2863 case INSN_CLD:
2864 if (!state.df && func) {
2865 WARN_FUNC("redundant CLD", sec, insn->offset);
2866 return 1;
2867 }
2868
2869 state.df = false;
2870 break;
2871
2872 default:
2873 break;
2874 }
2875
2876 if (insn->dead_end)
2877 return 0;
2878
2879 if (!next_insn) {
2880 if (state.cfi.cfa.base == CFI_UNDEFINED)
2881 return 0;
2882 WARN("%s: unexpected end of section", sec->name);
2883 return 1;
2884 }
2885
2886 insn = next_insn;
2887 }
2888
2889 return 0;
2890}
2891
2892static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
2893{
2894 struct instruction *insn;
2895 struct insn_state state;
2896 int ret, warnings = 0;
2897
2898 if (!file->hints)
2899 return 0;
2900
2901 init_insn_state(&state, sec);
2902
2903 if (sec) {
2904 insn = find_insn(file, sec, 0);
2905 if (!insn)
2906 return 0;
2907 } else {
2908 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
2909 }
2910
2911 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
2912 if (insn->hint && !insn->visited) {
2913 ret = validate_branch(file, insn->func, insn, state);
2914 if (ret && backtrace)
2915 BT_FUNC("<=== (hint)", insn);
2916 warnings += ret;
2917 }
2918
2919 insn = list_next_entry(insn, list);
2920 }
2921
2922 return warnings;
2923}
2924
2925static int validate_retpoline(struct objtool_file *file)
2926{
2927 struct instruction *insn;
2928 int warnings = 0;
2929
2930 for_each_insn(file, insn) {
2931 if (insn->type != INSN_JUMP_DYNAMIC &&
2932 insn->type != INSN_CALL_DYNAMIC)
2933 continue;
2934
2935 if (insn->retpoline_safe)
2936 continue;
2937
2938 /*
2939 * .init.text code is ran before userspace and thus doesn't
2940 * strictly need retpolines, except for modules which are
2941 * loaded late, they very much do need retpoline in their
2942 * .init.text
2943 */
2944 if (!strcmp(insn->sec->name, ".init.text") && !module)
2945 continue;
2946
2947 WARN_FUNC("indirect %s found in RETPOLINE build",
2948 insn->sec, insn->offset,
2949 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
2950
2951 warnings++;
2952 }
2953
2954 return warnings;
2955}
2956
2957static bool is_kasan_insn(struct instruction *insn)
2958{
2959 return (insn->type == INSN_CALL &&
2960 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
2961}
2962
2963static bool is_ubsan_insn(struct instruction *insn)
2964{
2965 return (insn->type == INSN_CALL &&
2966 !strcmp(insn->call_dest->name,
2967 "__ubsan_handle_builtin_unreachable"));
2968}
2969
2970static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
2971{
2972 int i;
2973 struct instruction *prev_insn;
2974
2975 if (insn->ignore || insn->type == INSN_NOP)
2976 return true;
2977
2978 /*
2979 * Ignore any unused exceptions. This can happen when a whitelisted
2980 * function has an exception table entry.
2981 *
2982 * Also ignore alternative replacement instructions. This can happen
2983 * when a whitelisted function uses one of the ALTERNATIVE macros.
2984 */
2985 if (!strcmp(insn->sec->name, ".fixup") ||
2986 !strcmp(insn->sec->name, ".altinstr_replacement") ||
2987 !strcmp(insn->sec->name, ".altinstr_aux"))
2988 return true;
2989
2990 if (!insn->func)
2991 return false;
2992
2993 /*
2994 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
2995 * __builtin_unreachable(). The BUG() macro has an unreachable() after
2996 * the UD2, which causes GCC's undefined trap logic to emit another UD2
2997 * (or occasionally a JMP to UD2).
2998 *
2999 * It may also insert a UD2 after calling a __noreturn function.
3000 */
3001 prev_insn = list_prev_entry(insn, list);
3002 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3003 (insn->type == INSN_BUG ||
3004 (insn->type == INSN_JUMP_UNCONDITIONAL &&
3005 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3006 return true;
3007
3008 /*
3009 * Check if this (or a subsequent) instruction is related to
3010 * CONFIG_UBSAN or CONFIG_KASAN.
3011 *
3012 * End the search at 5 instructions to avoid going into the weeds.
3013 */
3014 for (i = 0; i < 5; i++) {
3015
3016 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3017 return true;
3018
3019 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3020 if (insn->jump_dest &&
3021 insn->jump_dest->func == insn->func) {
3022 insn = insn->jump_dest;
3023 continue;
3024 }
3025
3026 break;
3027 }
3028
3029 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3030 break;
3031
3032 insn = list_next_entry(insn, list);
3033 }
3034
3035 return false;
3036}
3037
3038static int validate_symbol(struct objtool_file *file, struct section *sec,
3039 struct symbol *sym, struct insn_state *state)
3040{
3041 struct instruction *insn;
3042 int ret;
3043
3044 if (!sym->len) {
3045 WARN("%s() is missing an ELF size annotation", sym->name);
3046 return 1;
3047 }
3048
3049 if (sym->pfunc != sym || sym->alias != sym)
3050 return 0;
3051
3052 insn = find_insn(file, sec, sym->offset);
3053 if (!insn || insn->ignore || insn->visited)
3054 return 0;
3055
3056 state->uaccess = sym->uaccess_safe;
3057
3058 ret = validate_branch(file, insn->func, insn, *state);
3059 if (ret && backtrace)
3060 BT_FUNC("<=== (sym)", insn);
3061 return ret;
3062}
3063
3064static int validate_section(struct objtool_file *file, struct section *sec)
3065{
3066 struct insn_state state;
3067 struct symbol *func;
3068 int warnings = 0;
3069
3070 list_for_each_entry(func, &sec->symbol_list, list) {
3071 if (func->type != STT_FUNC)
3072 continue;
3073
3074 init_insn_state(&state, sec);
3075 set_func_state(&state.cfi);
3076
3077 warnings += validate_symbol(file, sec, func, &state);
3078 }
3079
3080 return warnings;
3081}
3082
3083static int validate_vmlinux_functions(struct objtool_file *file)
3084{
3085 struct section *sec;
3086 int warnings = 0;
3087
3088 sec = find_section_by_name(file->elf, ".noinstr.text");
3089 if (sec) {
3090 warnings += validate_section(file, sec);
3091 warnings += validate_unwind_hints(file, sec);
3092 }
3093
3094 sec = find_section_by_name(file->elf, ".entry.text");
3095 if (sec) {
3096 warnings += validate_section(file, sec);
3097 warnings += validate_unwind_hints(file, sec);
3098 }
3099
3100 return warnings;
3101}
3102
3103static int validate_functions(struct objtool_file *file)
3104{
3105 struct section *sec;
3106 int warnings = 0;
3107
3108 for_each_sec(file, sec) {
3109 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3110 continue;
3111
3112 warnings += validate_section(file, sec);
3113 }
3114
3115 return warnings;
3116}
3117
3118static int validate_reachable_instructions(struct objtool_file *file)
3119{
3120 struct instruction *insn;
3121
3122 if (file->ignore_unreachables)
3123 return 0;
3124
3125 for_each_insn(file, insn) {
3126 if (insn->visited || ignore_unreachable_insn(file, insn))
3127 continue;
3128
3129 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3130 return 1;
3131 }
3132
3133 return 0;
3134}
3135
3136int check(struct objtool_file *file)
3137{
3138 int ret, warnings = 0;
3139
3140 arch_initial_func_cfi_state(&initial_func_cfi);
3141
3142 ret = decode_sections(file);
3143 if (ret < 0)
3144 goto out;
3145 warnings += ret;
3146
3147 if (list_empty(&file->insn_list))
3148 goto out;
3149
3150 if (vmlinux && !validate_dup) {
3151 ret = validate_vmlinux_functions(file);
3152 if (ret < 0)
3153 goto out;
3154
3155 warnings += ret;
3156 goto out;
3157 }
3158
3159 if (retpoline) {
3160 ret = validate_retpoline(file);
3161 if (ret < 0)
3162 return ret;
3163 warnings += ret;
3164 }
3165
3166 ret = validate_functions(file);
3167 if (ret < 0)
3168 goto out;
3169 warnings += ret;
3170
3171 ret = validate_unwind_hints(file, NULL);
3172 if (ret < 0)
3173 goto out;
3174 warnings += ret;
3175
3176 if (!warnings) {
3177 ret = validate_reachable_instructions(file);
3178 if (ret < 0)
3179 goto out;
3180 warnings += ret;
3181 }
3182
3183 ret = create_static_call_sections(file);
3184 if (ret < 0)
3185 goto out;
3186 warnings += ret;
3187
3188 if (mcount) {
3189 ret = create_mcount_loc_sections(file);
3190 if (ret < 0)
3191 goto out;
3192 warnings += ret;
3193 }
3194
3195out:
3196 /*
3197 * For now, don't fail the kernel build on fatal warnings. These
3198 * errors are still fairly common due to the growing matrix of
3199 * supported toolchains and their recent pace of change.
3200 */
3201 return 0;
3202}
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6#include <string.h>
7#include <stdlib.h>
8#include <inttypes.h>
9#include <sys/mman.h>
10
11#include <objtool/builtin.h>
12#include <objtool/cfi.h>
13#include <objtool/arch.h>
14#include <objtool/check.h>
15#include <objtool/special.h>
16#include <objtool/warn.h>
17#include <objtool/endianness.h>
18
19#include <linux/objtool_types.h>
20#include <linux/hashtable.h>
21#include <linux/kernel.h>
22#include <linux/static_call_types.h>
23
24struct alternative {
25 struct alternative *next;
26 struct instruction *insn;
27 bool skip_orig;
28};
29
30static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31
32static struct cfi_init_state initial_func_cfi;
33static struct cfi_state init_cfi;
34static struct cfi_state func_cfi;
35static struct cfi_state force_undefined_cfi;
36
37struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
39{
40 struct instruction *insn;
41
42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
45 }
46
47 return NULL;
48}
49
50struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
52{
53 if (insn->idx == INSN_CHUNK_MAX)
54 return find_insn(file, insn->sec, insn->offset + insn->len);
55
56 insn++;
57 if (!insn->len)
58 return NULL;
59
60 return insn;
61}
62
63static struct instruction *next_insn_same_func(struct objtool_file *file,
64 struct instruction *insn)
65{
66 struct instruction *next = next_insn_same_sec(file, insn);
67 struct symbol *func = insn_func(insn);
68
69 if (!func)
70 return NULL;
71
72 if (next && insn_func(next) == func)
73 return next;
74
75 /* Check if we're already in the subfunction: */
76 if (func == func->cfunc)
77 return NULL;
78
79 /* Move to the subfunction: */
80 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
81}
82
83static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84 struct instruction *insn)
85{
86 if (insn->idx == 0) {
87 if (insn->prev_len)
88 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
89 return NULL;
90 }
91
92 return insn - 1;
93}
94
95static struct instruction *prev_insn_same_sym(struct objtool_file *file,
96 struct instruction *insn)
97{
98 struct instruction *prev = prev_insn_same_sec(file, insn);
99
100 if (prev && insn_func(prev) == insn_func(insn))
101 return prev;
102
103 return NULL;
104}
105
106#define for_each_insn(file, insn) \
107 for (struct section *__sec, *__fake = (struct section *)1; \
108 __fake; __fake = NULL) \
109 for_each_sec(file, __sec) \
110 sec_for_each_insn(file, __sec, insn)
111
112#define func_for_each_insn(file, func, insn) \
113 for (insn = find_insn(file, func->sec, func->offset); \
114 insn; \
115 insn = next_insn_same_func(file, insn))
116
117#define sym_for_each_insn(file, sym, insn) \
118 for (insn = find_insn(file, sym->sec, sym->offset); \
119 insn && insn->offset < sym->offset + sym->len; \
120 insn = next_insn_same_sec(file, insn))
121
122#define sym_for_each_insn_continue_reverse(file, sym, insn) \
123 for (insn = prev_insn_same_sec(file, insn); \
124 insn && insn->offset >= sym->offset; \
125 insn = prev_insn_same_sec(file, insn))
126
127#define sec_for_each_insn_from(file, insn) \
128 for (; insn; insn = next_insn_same_sec(file, insn))
129
130#define sec_for_each_insn_continue(file, insn) \
131 for (insn = next_insn_same_sec(file, insn); insn; \
132 insn = next_insn_same_sec(file, insn))
133
134static inline struct symbol *insn_call_dest(struct instruction *insn)
135{
136 if (insn->type == INSN_JUMP_DYNAMIC ||
137 insn->type == INSN_CALL_DYNAMIC)
138 return NULL;
139
140 return insn->_call_dest;
141}
142
143static inline struct reloc *insn_jump_table(struct instruction *insn)
144{
145 if (insn->type == INSN_JUMP_DYNAMIC ||
146 insn->type == INSN_CALL_DYNAMIC)
147 return insn->_jump_table;
148
149 return NULL;
150}
151
152static bool is_jump_table_jump(struct instruction *insn)
153{
154 struct alt_group *alt_group = insn->alt_group;
155
156 if (insn_jump_table(insn))
157 return true;
158
159 /* Retpoline alternative for a jump table? */
160 return alt_group && alt_group->orig_group &&
161 insn_jump_table(alt_group->orig_group->first_insn);
162}
163
164static bool is_sibling_call(struct instruction *insn)
165{
166 /*
167 * Assume only STT_FUNC calls have jump-tables.
168 */
169 if (insn_func(insn)) {
170 /* An indirect jump is either a sibling call or a jump to a table. */
171 if (insn->type == INSN_JUMP_DYNAMIC)
172 return !is_jump_table_jump(insn);
173 }
174
175 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176 return (is_static_jump(insn) && insn_call_dest(insn));
177}
178
179/*
180 * This checks to see if the given function is a "noreturn" function.
181 *
182 * For global functions which are outside the scope of this object file, we
183 * have to keep a manual list of them.
184 *
185 * For local functions, we have to detect them manually by simply looking for
186 * the lack of a return instruction.
187 */
188static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
189 int recursion)
190{
191 int i;
192 struct instruction *insn;
193 bool empty = true;
194
195#define NORETURN(func) __stringify(func),
196 static const char * const global_noreturns[] = {
197#include "noreturns.h"
198 };
199#undef NORETURN
200
201 if (!func)
202 return false;
203
204 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
205 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
206 if (!strcmp(func->name, global_noreturns[i]))
207 return true;
208
209 if (func->bind == STB_WEAK)
210 return false;
211
212 if (!func->len)
213 return false;
214
215 insn = find_insn(file, func->sec, func->offset);
216 if (!insn || !insn_func(insn))
217 return false;
218
219 func_for_each_insn(file, func, insn) {
220 empty = false;
221
222 if (insn->type == INSN_RETURN)
223 return false;
224 }
225
226 if (empty)
227 return false;
228
229 /*
230 * A function can have a sibling call instead of a return. In that
231 * case, the function's dead-end status depends on whether the target
232 * of the sibling call returns.
233 */
234 func_for_each_insn(file, func, insn) {
235 if (is_sibling_call(insn)) {
236 struct instruction *dest = insn->jump_dest;
237
238 if (!dest)
239 /* sibling call to another file */
240 return false;
241
242 /* local sibling call */
243 if (recursion == 5) {
244 /*
245 * Infinite recursion: two functions have
246 * sibling calls to each other. This is a very
247 * rare case. It means they aren't dead ends.
248 */
249 return false;
250 }
251
252 return __dead_end_function(file, insn_func(dest), recursion+1);
253 }
254 }
255
256 return true;
257}
258
259static bool dead_end_function(struct objtool_file *file, struct symbol *func)
260{
261 return __dead_end_function(file, func, 0);
262}
263
264static void init_cfi_state(struct cfi_state *cfi)
265{
266 int i;
267
268 for (i = 0; i < CFI_NUM_REGS; i++) {
269 cfi->regs[i].base = CFI_UNDEFINED;
270 cfi->vals[i].base = CFI_UNDEFINED;
271 }
272 cfi->cfa.base = CFI_UNDEFINED;
273 cfi->drap_reg = CFI_UNDEFINED;
274 cfi->drap_offset = -1;
275}
276
277static void init_insn_state(struct objtool_file *file, struct insn_state *state,
278 struct section *sec)
279{
280 memset(state, 0, sizeof(*state));
281 init_cfi_state(&state->cfi);
282
283 /*
284 * We need the full vmlinux for noinstr validation, otherwise we can
285 * not correctly determine insn_call_dest(insn)->sec (external symbols
286 * do not have a section).
287 */
288 if (opts.link && opts.noinstr && sec)
289 state->noinstr = sec->noinstr;
290}
291
292static struct cfi_state *cfi_alloc(void)
293{
294 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
295 if (!cfi) {
296 WARN("calloc failed");
297 exit(1);
298 }
299 nr_cfi++;
300 return cfi;
301}
302
303static int cfi_bits;
304static struct hlist_head *cfi_hash;
305
306static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
307{
308 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
309 (void *)cfi2 + sizeof(cfi2->hash),
310 sizeof(struct cfi_state) - sizeof(struct hlist_node));
311}
312
313static inline u32 cfi_key(struct cfi_state *cfi)
314{
315 return jhash((void *)cfi + sizeof(cfi->hash),
316 sizeof(*cfi) - sizeof(cfi->hash), 0);
317}
318
319static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
320{
321 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
322 struct cfi_state *obj;
323
324 hlist_for_each_entry(obj, head, hash) {
325 if (!cficmp(cfi, obj)) {
326 nr_cfi_cache++;
327 return obj;
328 }
329 }
330
331 obj = cfi_alloc();
332 *obj = *cfi;
333 hlist_add_head(&obj->hash, head);
334
335 return obj;
336}
337
338static void cfi_hash_add(struct cfi_state *cfi)
339{
340 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
341
342 hlist_add_head(&cfi->hash, head);
343}
344
345static void *cfi_hash_alloc(unsigned long size)
346{
347 cfi_bits = max(10, ilog2(size));
348 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
349 PROT_READ|PROT_WRITE,
350 MAP_PRIVATE|MAP_ANON, -1, 0);
351 if (cfi_hash == (void *)-1L) {
352 WARN("mmap fail cfi_hash");
353 cfi_hash = NULL;
354 } else if (opts.stats) {
355 printf("cfi_bits: %d\n", cfi_bits);
356 }
357
358 return cfi_hash;
359}
360
361static unsigned long nr_insns;
362static unsigned long nr_insns_visited;
363
364/*
365 * Call the arch-specific instruction decoder for all the instructions and add
366 * them to the global instruction list.
367 */
368static int decode_instructions(struct objtool_file *file)
369{
370 struct section *sec;
371 struct symbol *func;
372 unsigned long offset;
373 struct instruction *insn;
374 int ret;
375
376 for_each_sec(file, sec) {
377 struct instruction *insns = NULL;
378 u8 prev_len = 0;
379 u8 idx = 0;
380
381 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
382 continue;
383
384 if (strcmp(sec->name, ".altinstr_replacement") &&
385 strcmp(sec->name, ".altinstr_aux") &&
386 strncmp(sec->name, ".discard.", 9))
387 sec->text = true;
388
389 if (!strcmp(sec->name, ".noinstr.text") ||
390 !strcmp(sec->name, ".entry.text") ||
391 !strcmp(sec->name, ".cpuidle.text") ||
392 !strncmp(sec->name, ".text..__x86.", 13))
393 sec->noinstr = true;
394
395 /*
396 * .init.text code is ran before userspace and thus doesn't
397 * strictly need retpolines, except for modules which are
398 * loaded late, they very much do need retpoline in their
399 * .init.text
400 */
401 if (!strcmp(sec->name, ".init.text") && !opts.module)
402 sec->init = true;
403
404 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
405 if (!insns || idx == INSN_CHUNK_MAX) {
406 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
407 if (!insns) {
408 WARN("malloc failed");
409 return -1;
410 }
411 idx = 0;
412 } else {
413 idx++;
414 }
415 insn = &insns[idx];
416 insn->idx = idx;
417
418 INIT_LIST_HEAD(&insn->call_node);
419 insn->sec = sec;
420 insn->offset = offset;
421 insn->prev_len = prev_len;
422
423 ret = arch_decode_instruction(file, sec, offset,
424 sec->sh.sh_size - offset,
425 insn);
426 if (ret)
427 return ret;
428
429 prev_len = insn->len;
430
431 /*
432 * By default, "ud2" is a dead end unless otherwise
433 * annotated, because GCC 7 inserts it for certain
434 * divide-by-zero cases.
435 */
436 if (insn->type == INSN_BUG)
437 insn->dead_end = true;
438
439 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
440 nr_insns++;
441 }
442
443// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
444
445 sec_for_each_sym(sec, func) {
446 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
447 continue;
448
449 if (func->offset == sec->sh.sh_size) {
450 /* Heuristic: likely an "end" symbol */
451 if (func->type == STT_NOTYPE)
452 continue;
453 WARN("%s(): STT_FUNC at end of section",
454 func->name);
455 return -1;
456 }
457
458 if (func->embedded_insn || func->alias != func)
459 continue;
460
461 if (!find_insn(file, sec, func->offset)) {
462 WARN("%s(): can't find starting instruction",
463 func->name);
464 return -1;
465 }
466
467 sym_for_each_insn(file, func, insn) {
468 insn->sym = func;
469 if (func->type == STT_FUNC &&
470 insn->type == INSN_ENDBR &&
471 list_empty(&insn->call_node)) {
472 if (insn->offset == func->offset) {
473 list_add_tail(&insn->call_node, &file->endbr_list);
474 file->nr_endbr++;
475 } else {
476 file->nr_endbr_int++;
477 }
478 }
479 }
480 }
481 }
482
483 if (opts.stats)
484 printf("nr_insns: %lu\n", nr_insns);
485
486 return 0;
487}
488
489/*
490 * Read the pv_ops[] .data table to find the static initialized values.
491 */
492static int add_pv_ops(struct objtool_file *file, const char *symname)
493{
494 struct symbol *sym, *func;
495 unsigned long off, end;
496 struct reloc *reloc;
497 int idx;
498
499 sym = find_symbol_by_name(file->elf, symname);
500 if (!sym)
501 return 0;
502
503 off = sym->offset;
504 end = off + sym->len;
505 for (;;) {
506 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
507 if (!reloc)
508 break;
509
510 func = reloc->sym;
511 if (func->type == STT_SECTION)
512 func = find_symbol_by_offset(reloc->sym->sec,
513 reloc_addend(reloc));
514
515 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
516
517 objtool_pv_add(file, idx, func);
518
519 off = reloc_offset(reloc) + 1;
520 if (off > end)
521 break;
522 }
523
524 return 0;
525}
526
527/*
528 * Allocate and initialize file->pv_ops[].
529 */
530static int init_pv_ops(struct objtool_file *file)
531{
532 static const char *pv_ops_tables[] = {
533 "pv_ops",
534 "xen_cpu_ops",
535 "xen_irq_ops",
536 "xen_mmu_ops",
537 NULL,
538 };
539 const char *pv_ops;
540 struct symbol *sym;
541 int idx, nr;
542
543 if (!opts.noinstr)
544 return 0;
545
546 file->pv_ops = NULL;
547
548 sym = find_symbol_by_name(file->elf, "pv_ops");
549 if (!sym)
550 return 0;
551
552 nr = sym->len / sizeof(unsigned long);
553 file->pv_ops = calloc(sizeof(struct pv_state), nr);
554 if (!file->pv_ops)
555 return -1;
556
557 for (idx = 0; idx < nr; idx++)
558 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
559
560 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
561 add_pv_ops(file, pv_ops);
562
563 return 0;
564}
565
566static struct instruction *find_last_insn(struct objtool_file *file,
567 struct section *sec)
568{
569 struct instruction *insn = NULL;
570 unsigned int offset;
571 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
572
573 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
574 insn = find_insn(file, sec, offset);
575
576 return insn;
577}
578
579/*
580 * Mark "ud2" instructions and manually annotated dead ends.
581 */
582static int add_dead_ends(struct objtool_file *file)
583{
584 struct section *rsec;
585 struct reloc *reloc;
586 struct instruction *insn;
587 s64 addend;
588
589 /*
590 * Check for manually annotated dead ends.
591 */
592 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
593 if (!rsec)
594 goto reachable;
595
596 for_each_reloc(rsec, reloc) {
597
598 if (reloc->sym->type != STT_SECTION) {
599 WARN("unexpected relocation symbol type in %s", rsec->name);
600 return -1;
601 }
602
603 addend = reloc_addend(reloc);
604
605 insn = find_insn(file, reloc->sym->sec, addend);
606 if (insn)
607 insn = prev_insn_same_sec(file, insn);
608 else if (addend == reloc->sym->sec->sh.sh_size) {
609 insn = find_last_insn(file, reloc->sym->sec);
610 if (!insn) {
611 WARN("can't find unreachable insn at %s+0x%" PRIx64,
612 reloc->sym->sec->name, addend);
613 return -1;
614 }
615 } else {
616 WARN("can't find unreachable insn at %s+0x%" PRIx64,
617 reloc->sym->sec->name, addend);
618 return -1;
619 }
620
621 insn->dead_end = true;
622 }
623
624reachable:
625 /*
626 * These manually annotated reachable checks are needed for GCC 4.4,
627 * where the Linux unreachable() macro isn't supported. In that case
628 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
629 * not a dead end.
630 */
631 rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
632 if (!rsec)
633 return 0;
634
635 for_each_reloc(rsec, reloc) {
636
637 if (reloc->sym->type != STT_SECTION) {
638 WARN("unexpected relocation symbol type in %s", rsec->name);
639 return -1;
640 }
641
642 addend = reloc_addend(reloc);
643
644 insn = find_insn(file, reloc->sym->sec, addend);
645 if (insn)
646 insn = prev_insn_same_sec(file, insn);
647 else if (addend == reloc->sym->sec->sh.sh_size) {
648 insn = find_last_insn(file, reloc->sym->sec);
649 if (!insn) {
650 WARN("can't find reachable insn at %s+0x%" PRIx64,
651 reloc->sym->sec->name, addend);
652 return -1;
653 }
654 } else {
655 WARN("can't find reachable insn at %s+0x%" PRIx64,
656 reloc->sym->sec->name, addend);
657 return -1;
658 }
659
660 insn->dead_end = false;
661 }
662
663 return 0;
664}
665
666static int create_static_call_sections(struct objtool_file *file)
667{
668 struct static_call_site *site;
669 struct section *sec;
670 struct instruction *insn;
671 struct symbol *key_sym;
672 char *key_name, *tmp;
673 int idx;
674
675 sec = find_section_by_name(file->elf, ".static_call_sites");
676 if (sec) {
677 INIT_LIST_HEAD(&file->static_call_list);
678 WARN("file already has .static_call_sites section, skipping");
679 return 0;
680 }
681
682 if (list_empty(&file->static_call_list))
683 return 0;
684
685 idx = 0;
686 list_for_each_entry(insn, &file->static_call_list, call_node)
687 idx++;
688
689 sec = elf_create_section_pair(file->elf, ".static_call_sites",
690 sizeof(*site), idx, idx * 2);
691 if (!sec)
692 return -1;
693
694 /* Allow modules to modify the low bits of static_call_site::key */
695 sec->sh.sh_flags |= SHF_WRITE;
696
697 idx = 0;
698 list_for_each_entry(insn, &file->static_call_list, call_node) {
699
700 /* populate reloc for 'addr' */
701 if (!elf_init_reloc_text_sym(file->elf, sec,
702 idx * sizeof(*site), idx * 2,
703 insn->sec, insn->offset))
704 return -1;
705
706 /* find key symbol */
707 key_name = strdup(insn_call_dest(insn)->name);
708 if (!key_name) {
709 perror("strdup");
710 return -1;
711 }
712 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
713 STATIC_CALL_TRAMP_PREFIX_LEN)) {
714 WARN("static_call: trampoline name malformed: %s", key_name);
715 free(key_name);
716 return -1;
717 }
718 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
719 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
720
721 key_sym = find_symbol_by_name(file->elf, tmp);
722 if (!key_sym) {
723 if (!opts.module) {
724 WARN("static_call: can't find static_call_key symbol: %s", tmp);
725 free(key_name);
726 return -1;
727 }
728
729 /*
730 * For modules(), the key might not be exported, which
731 * means the module can make static calls but isn't
732 * allowed to change them.
733 *
734 * In that case we temporarily set the key to be the
735 * trampoline address. This is fixed up in
736 * static_call_add_module().
737 */
738 key_sym = insn_call_dest(insn);
739 }
740 free(key_name);
741
742 /* populate reloc for 'key' */
743 if (!elf_init_reloc_data_sym(file->elf, sec,
744 idx * sizeof(*site) + 4,
745 (idx * 2) + 1, key_sym,
746 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
747 return -1;
748
749 idx++;
750 }
751
752 return 0;
753}
754
755static int create_retpoline_sites_sections(struct objtool_file *file)
756{
757 struct instruction *insn;
758 struct section *sec;
759 int idx;
760
761 sec = find_section_by_name(file->elf, ".retpoline_sites");
762 if (sec) {
763 WARN("file already has .retpoline_sites, skipping");
764 return 0;
765 }
766
767 idx = 0;
768 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
769 idx++;
770
771 if (!idx)
772 return 0;
773
774 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
775 sizeof(int), idx, idx);
776 if (!sec)
777 return -1;
778
779 idx = 0;
780 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
781
782 if (!elf_init_reloc_text_sym(file->elf, sec,
783 idx * sizeof(int), idx,
784 insn->sec, insn->offset))
785 return -1;
786
787 idx++;
788 }
789
790 return 0;
791}
792
793static int create_return_sites_sections(struct objtool_file *file)
794{
795 struct instruction *insn;
796 struct section *sec;
797 int idx;
798
799 sec = find_section_by_name(file->elf, ".return_sites");
800 if (sec) {
801 WARN("file already has .return_sites, skipping");
802 return 0;
803 }
804
805 idx = 0;
806 list_for_each_entry(insn, &file->return_thunk_list, call_node)
807 idx++;
808
809 if (!idx)
810 return 0;
811
812 sec = elf_create_section_pair(file->elf, ".return_sites",
813 sizeof(int), idx, idx);
814 if (!sec)
815 return -1;
816
817 idx = 0;
818 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
819
820 if (!elf_init_reloc_text_sym(file->elf, sec,
821 idx * sizeof(int), idx,
822 insn->sec, insn->offset))
823 return -1;
824
825 idx++;
826 }
827
828 return 0;
829}
830
831static int create_ibt_endbr_seal_sections(struct objtool_file *file)
832{
833 struct instruction *insn;
834 struct section *sec;
835 int idx;
836
837 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
838 if (sec) {
839 WARN("file already has .ibt_endbr_seal, skipping");
840 return 0;
841 }
842
843 idx = 0;
844 list_for_each_entry(insn, &file->endbr_list, call_node)
845 idx++;
846
847 if (opts.stats) {
848 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
849 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
850 printf("ibt: superfluous ENDBR: %d\n", idx);
851 }
852
853 if (!idx)
854 return 0;
855
856 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
857 sizeof(int), idx, idx);
858 if (!sec)
859 return -1;
860
861 idx = 0;
862 list_for_each_entry(insn, &file->endbr_list, call_node) {
863
864 int *site = (int *)sec->data->d_buf + idx;
865 struct symbol *sym = insn->sym;
866 *site = 0;
867
868 if (opts.module && sym && sym->type == STT_FUNC &&
869 insn->offset == sym->offset &&
870 (!strcmp(sym->name, "init_module") ||
871 !strcmp(sym->name, "cleanup_module")))
872 WARN("%s(): not an indirect call target", sym->name);
873
874 if (!elf_init_reloc_text_sym(file->elf, sec,
875 idx * sizeof(int), idx,
876 insn->sec, insn->offset))
877 return -1;
878
879 idx++;
880 }
881
882 return 0;
883}
884
885static int create_cfi_sections(struct objtool_file *file)
886{
887 struct section *sec;
888 struct symbol *sym;
889 int idx;
890
891 sec = find_section_by_name(file->elf, ".cfi_sites");
892 if (sec) {
893 INIT_LIST_HEAD(&file->call_list);
894 WARN("file already has .cfi_sites section, skipping");
895 return 0;
896 }
897
898 idx = 0;
899 for_each_sym(file, sym) {
900 if (sym->type != STT_FUNC)
901 continue;
902
903 if (strncmp(sym->name, "__cfi_", 6))
904 continue;
905
906 idx++;
907 }
908
909 sec = elf_create_section_pair(file->elf, ".cfi_sites",
910 sizeof(unsigned int), idx, idx);
911 if (!sec)
912 return -1;
913
914 idx = 0;
915 for_each_sym(file, sym) {
916 if (sym->type != STT_FUNC)
917 continue;
918
919 if (strncmp(sym->name, "__cfi_", 6))
920 continue;
921
922 if (!elf_init_reloc_text_sym(file->elf, sec,
923 idx * sizeof(unsigned int), idx,
924 sym->sec, sym->offset))
925 return -1;
926
927 idx++;
928 }
929
930 return 0;
931}
932
933static int create_mcount_loc_sections(struct objtool_file *file)
934{
935 size_t addr_size = elf_addr_size(file->elf);
936 struct instruction *insn;
937 struct section *sec;
938 int idx;
939
940 sec = find_section_by_name(file->elf, "__mcount_loc");
941 if (sec) {
942 INIT_LIST_HEAD(&file->mcount_loc_list);
943 WARN("file already has __mcount_loc section, skipping");
944 return 0;
945 }
946
947 if (list_empty(&file->mcount_loc_list))
948 return 0;
949
950 idx = 0;
951 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
952 idx++;
953
954 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
955 idx, idx);
956 if (!sec)
957 return -1;
958
959 sec->sh.sh_addralign = addr_size;
960
961 idx = 0;
962 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
963
964 struct reloc *reloc;
965
966 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
967 insn->sec, insn->offset);
968 if (!reloc)
969 return -1;
970
971 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
972
973 idx++;
974 }
975
976 return 0;
977}
978
979static int create_direct_call_sections(struct objtool_file *file)
980{
981 struct instruction *insn;
982 struct section *sec;
983 int idx;
984
985 sec = find_section_by_name(file->elf, ".call_sites");
986 if (sec) {
987 INIT_LIST_HEAD(&file->call_list);
988 WARN("file already has .call_sites section, skipping");
989 return 0;
990 }
991
992 if (list_empty(&file->call_list))
993 return 0;
994
995 idx = 0;
996 list_for_each_entry(insn, &file->call_list, call_node)
997 idx++;
998
999 sec = elf_create_section_pair(file->elf, ".call_sites",
1000 sizeof(unsigned int), idx, idx);
1001 if (!sec)
1002 return -1;
1003
1004 idx = 0;
1005 list_for_each_entry(insn, &file->call_list, call_node) {
1006
1007 if (!elf_init_reloc_text_sym(file->elf, sec,
1008 idx * sizeof(unsigned int), idx,
1009 insn->sec, insn->offset))
1010 return -1;
1011
1012 idx++;
1013 }
1014
1015 return 0;
1016}
1017
1018/*
1019 * Warnings shouldn't be reported for ignored functions.
1020 */
1021static void add_ignores(struct objtool_file *file)
1022{
1023 struct instruction *insn;
1024 struct section *rsec;
1025 struct symbol *func;
1026 struct reloc *reloc;
1027
1028 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1029 if (!rsec)
1030 return;
1031
1032 for_each_reloc(rsec, reloc) {
1033 switch (reloc->sym->type) {
1034 case STT_FUNC:
1035 func = reloc->sym;
1036 break;
1037
1038 case STT_SECTION:
1039 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1040 if (!func)
1041 continue;
1042 break;
1043
1044 default:
1045 WARN("unexpected relocation symbol type in %s: %d",
1046 rsec->name, reloc->sym->type);
1047 continue;
1048 }
1049
1050 func_for_each_insn(file, func, insn)
1051 insn->ignore = true;
1052 }
1053}
1054
1055/*
1056 * This is a whitelist of functions that is allowed to be called with AC set.
1057 * The list is meant to be minimal and only contains compiler instrumentation
1058 * ABI and a few functions used to implement *_{to,from}_user() functions.
1059 *
1060 * These functions must not directly change AC, but may PUSHF/POPF.
1061 */
1062static const char *uaccess_safe_builtin[] = {
1063 /* KASAN */
1064 "kasan_report",
1065 "kasan_check_range",
1066 /* KASAN out-of-line */
1067 "__asan_loadN_noabort",
1068 "__asan_load1_noabort",
1069 "__asan_load2_noabort",
1070 "__asan_load4_noabort",
1071 "__asan_load8_noabort",
1072 "__asan_load16_noabort",
1073 "__asan_storeN_noabort",
1074 "__asan_store1_noabort",
1075 "__asan_store2_noabort",
1076 "__asan_store4_noabort",
1077 "__asan_store8_noabort",
1078 "__asan_store16_noabort",
1079 "__kasan_check_read",
1080 "__kasan_check_write",
1081 /* KASAN in-line */
1082 "__asan_report_load_n_noabort",
1083 "__asan_report_load1_noabort",
1084 "__asan_report_load2_noabort",
1085 "__asan_report_load4_noabort",
1086 "__asan_report_load8_noabort",
1087 "__asan_report_load16_noabort",
1088 "__asan_report_store_n_noabort",
1089 "__asan_report_store1_noabort",
1090 "__asan_report_store2_noabort",
1091 "__asan_report_store4_noabort",
1092 "__asan_report_store8_noabort",
1093 "__asan_report_store16_noabort",
1094 /* KCSAN */
1095 "__kcsan_check_access",
1096 "__kcsan_mb",
1097 "__kcsan_wmb",
1098 "__kcsan_rmb",
1099 "__kcsan_release",
1100 "kcsan_found_watchpoint",
1101 "kcsan_setup_watchpoint",
1102 "kcsan_check_scoped_accesses",
1103 "kcsan_disable_current",
1104 "kcsan_enable_current_nowarn",
1105 /* KCSAN/TSAN */
1106 "__tsan_func_entry",
1107 "__tsan_func_exit",
1108 "__tsan_read_range",
1109 "__tsan_write_range",
1110 "__tsan_read1",
1111 "__tsan_read2",
1112 "__tsan_read4",
1113 "__tsan_read8",
1114 "__tsan_read16",
1115 "__tsan_write1",
1116 "__tsan_write2",
1117 "__tsan_write4",
1118 "__tsan_write8",
1119 "__tsan_write16",
1120 "__tsan_read_write1",
1121 "__tsan_read_write2",
1122 "__tsan_read_write4",
1123 "__tsan_read_write8",
1124 "__tsan_read_write16",
1125 "__tsan_volatile_read1",
1126 "__tsan_volatile_read2",
1127 "__tsan_volatile_read4",
1128 "__tsan_volatile_read8",
1129 "__tsan_volatile_read16",
1130 "__tsan_volatile_write1",
1131 "__tsan_volatile_write2",
1132 "__tsan_volatile_write4",
1133 "__tsan_volatile_write8",
1134 "__tsan_volatile_write16",
1135 "__tsan_atomic8_load",
1136 "__tsan_atomic16_load",
1137 "__tsan_atomic32_load",
1138 "__tsan_atomic64_load",
1139 "__tsan_atomic8_store",
1140 "__tsan_atomic16_store",
1141 "__tsan_atomic32_store",
1142 "__tsan_atomic64_store",
1143 "__tsan_atomic8_exchange",
1144 "__tsan_atomic16_exchange",
1145 "__tsan_atomic32_exchange",
1146 "__tsan_atomic64_exchange",
1147 "__tsan_atomic8_fetch_add",
1148 "__tsan_atomic16_fetch_add",
1149 "__tsan_atomic32_fetch_add",
1150 "__tsan_atomic64_fetch_add",
1151 "__tsan_atomic8_fetch_sub",
1152 "__tsan_atomic16_fetch_sub",
1153 "__tsan_atomic32_fetch_sub",
1154 "__tsan_atomic64_fetch_sub",
1155 "__tsan_atomic8_fetch_and",
1156 "__tsan_atomic16_fetch_and",
1157 "__tsan_atomic32_fetch_and",
1158 "__tsan_atomic64_fetch_and",
1159 "__tsan_atomic8_fetch_or",
1160 "__tsan_atomic16_fetch_or",
1161 "__tsan_atomic32_fetch_or",
1162 "__tsan_atomic64_fetch_or",
1163 "__tsan_atomic8_fetch_xor",
1164 "__tsan_atomic16_fetch_xor",
1165 "__tsan_atomic32_fetch_xor",
1166 "__tsan_atomic64_fetch_xor",
1167 "__tsan_atomic8_fetch_nand",
1168 "__tsan_atomic16_fetch_nand",
1169 "__tsan_atomic32_fetch_nand",
1170 "__tsan_atomic64_fetch_nand",
1171 "__tsan_atomic8_compare_exchange_strong",
1172 "__tsan_atomic16_compare_exchange_strong",
1173 "__tsan_atomic32_compare_exchange_strong",
1174 "__tsan_atomic64_compare_exchange_strong",
1175 "__tsan_atomic8_compare_exchange_weak",
1176 "__tsan_atomic16_compare_exchange_weak",
1177 "__tsan_atomic32_compare_exchange_weak",
1178 "__tsan_atomic64_compare_exchange_weak",
1179 "__tsan_atomic8_compare_exchange_val",
1180 "__tsan_atomic16_compare_exchange_val",
1181 "__tsan_atomic32_compare_exchange_val",
1182 "__tsan_atomic64_compare_exchange_val",
1183 "__tsan_atomic_thread_fence",
1184 "__tsan_atomic_signal_fence",
1185 "__tsan_unaligned_read16",
1186 "__tsan_unaligned_write16",
1187 /* KCOV */
1188 "write_comp_data",
1189 "check_kcov_mode",
1190 "__sanitizer_cov_trace_pc",
1191 "__sanitizer_cov_trace_const_cmp1",
1192 "__sanitizer_cov_trace_const_cmp2",
1193 "__sanitizer_cov_trace_const_cmp4",
1194 "__sanitizer_cov_trace_const_cmp8",
1195 "__sanitizer_cov_trace_cmp1",
1196 "__sanitizer_cov_trace_cmp2",
1197 "__sanitizer_cov_trace_cmp4",
1198 "__sanitizer_cov_trace_cmp8",
1199 "__sanitizer_cov_trace_switch",
1200 /* KMSAN */
1201 "kmsan_copy_to_user",
1202 "kmsan_report",
1203 "kmsan_unpoison_entry_regs",
1204 "kmsan_unpoison_memory",
1205 "__msan_chain_origin",
1206 "__msan_get_context_state",
1207 "__msan_instrument_asm_store",
1208 "__msan_metadata_ptr_for_load_1",
1209 "__msan_metadata_ptr_for_load_2",
1210 "__msan_metadata_ptr_for_load_4",
1211 "__msan_metadata_ptr_for_load_8",
1212 "__msan_metadata_ptr_for_load_n",
1213 "__msan_metadata_ptr_for_store_1",
1214 "__msan_metadata_ptr_for_store_2",
1215 "__msan_metadata_ptr_for_store_4",
1216 "__msan_metadata_ptr_for_store_8",
1217 "__msan_metadata_ptr_for_store_n",
1218 "__msan_poison_alloca",
1219 "__msan_warning",
1220 /* UBSAN */
1221 "ubsan_type_mismatch_common",
1222 "__ubsan_handle_type_mismatch",
1223 "__ubsan_handle_type_mismatch_v1",
1224 "__ubsan_handle_shift_out_of_bounds",
1225 "__ubsan_handle_load_invalid_value",
1226 /* STACKLEAK */
1227 "stackleak_track_stack",
1228 /* misc */
1229 "csum_partial_copy_generic",
1230 "copy_mc_fragile",
1231 "copy_mc_fragile_handle_tail",
1232 "copy_mc_enhanced_fast_string",
1233 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1234 "rep_stos_alternative",
1235 "rep_movs_alternative",
1236 "__copy_user_nocache",
1237 NULL
1238};
1239
1240static void add_uaccess_safe(struct objtool_file *file)
1241{
1242 struct symbol *func;
1243 const char **name;
1244
1245 if (!opts.uaccess)
1246 return;
1247
1248 for (name = uaccess_safe_builtin; *name; name++) {
1249 func = find_symbol_by_name(file->elf, *name);
1250 if (!func)
1251 continue;
1252
1253 func->uaccess_safe = true;
1254 }
1255}
1256
1257/*
1258 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1259 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1260 * But it at least allows objtool to understand the control flow *around* the
1261 * retpoline.
1262 */
1263static int add_ignore_alternatives(struct objtool_file *file)
1264{
1265 struct section *rsec;
1266 struct reloc *reloc;
1267 struct instruction *insn;
1268
1269 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1270 if (!rsec)
1271 return 0;
1272
1273 for_each_reloc(rsec, reloc) {
1274 if (reloc->sym->type != STT_SECTION) {
1275 WARN("unexpected relocation symbol type in %s", rsec->name);
1276 return -1;
1277 }
1278
1279 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1280 if (!insn) {
1281 WARN("bad .discard.ignore_alts entry");
1282 return -1;
1283 }
1284
1285 insn->ignore_alts = true;
1286 }
1287
1288 return 0;
1289}
1290
1291/*
1292 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1293 * will be added to the .retpoline_sites section.
1294 */
1295__weak bool arch_is_retpoline(struct symbol *sym)
1296{
1297 return false;
1298}
1299
1300/*
1301 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1302 * will be added to the .return_sites section.
1303 */
1304__weak bool arch_is_rethunk(struct symbol *sym)
1305{
1306 return false;
1307}
1308
1309/*
1310 * Symbols that are embedded inside other instructions, because sometimes crazy
1311 * code exists. These are mostly ignored for validation purposes.
1312 */
1313__weak bool arch_is_embedded_insn(struct symbol *sym)
1314{
1315 return false;
1316}
1317
1318static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1319{
1320 struct reloc *reloc;
1321
1322 if (insn->no_reloc)
1323 return NULL;
1324
1325 if (!file)
1326 return NULL;
1327
1328 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1329 insn->offset, insn->len);
1330 if (!reloc) {
1331 insn->no_reloc = 1;
1332 return NULL;
1333 }
1334
1335 return reloc;
1336}
1337
1338static void remove_insn_ops(struct instruction *insn)
1339{
1340 struct stack_op *op, *next;
1341
1342 for (op = insn->stack_ops; op; op = next) {
1343 next = op->next;
1344 free(op);
1345 }
1346 insn->stack_ops = NULL;
1347}
1348
1349static void annotate_call_site(struct objtool_file *file,
1350 struct instruction *insn, bool sibling)
1351{
1352 struct reloc *reloc = insn_reloc(file, insn);
1353 struct symbol *sym = insn_call_dest(insn);
1354
1355 if (!sym)
1356 sym = reloc->sym;
1357
1358 /*
1359 * Alternative replacement code is just template code which is
1360 * sometimes copied to the original instruction. For now, don't
1361 * annotate it. (In the future we might consider annotating the
1362 * original instruction if/when it ever makes sense to do so.)
1363 */
1364 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1365 return;
1366
1367 if (sym->static_call_tramp) {
1368 list_add_tail(&insn->call_node, &file->static_call_list);
1369 return;
1370 }
1371
1372 if (sym->retpoline_thunk) {
1373 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1374 return;
1375 }
1376
1377 /*
1378 * Many compilers cannot disable KCOV or sanitizer calls with a function
1379 * attribute so they need a little help, NOP out any such calls from
1380 * noinstr text.
1381 */
1382 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1383 if (reloc)
1384 set_reloc_type(file->elf, reloc, R_NONE);
1385
1386 elf_write_insn(file->elf, insn->sec,
1387 insn->offset, insn->len,
1388 sibling ? arch_ret_insn(insn->len)
1389 : arch_nop_insn(insn->len));
1390
1391 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1392
1393 if (sibling) {
1394 /*
1395 * We've replaced the tail-call JMP insn by two new
1396 * insn: RET; INT3, except we only have a single struct
1397 * insn here. Mark it retpoline_safe to avoid the SLS
1398 * warning, instead of adding another insn.
1399 */
1400 insn->retpoline_safe = true;
1401 }
1402
1403 return;
1404 }
1405
1406 if (opts.mcount && sym->fentry) {
1407 if (sibling)
1408 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1409 if (opts.mnop) {
1410 if (reloc)
1411 set_reloc_type(file->elf, reloc, R_NONE);
1412
1413 elf_write_insn(file->elf, insn->sec,
1414 insn->offset, insn->len,
1415 arch_nop_insn(insn->len));
1416
1417 insn->type = INSN_NOP;
1418 }
1419
1420 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1421 return;
1422 }
1423
1424 if (insn->type == INSN_CALL && !insn->sec->init)
1425 list_add_tail(&insn->call_node, &file->call_list);
1426
1427 if (!sibling && dead_end_function(file, sym))
1428 insn->dead_end = true;
1429}
1430
1431static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1432 struct symbol *dest, bool sibling)
1433{
1434 insn->_call_dest = dest;
1435 if (!dest)
1436 return;
1437
1438 /*
1439 * Whatever stack impact regular CALLs have, should be undone
1440 * by the RETURN of the called function.
1441 *
1442 * Annotated intra-function calls retain the stack_ops but
1443 * are converted to JUMP, see read_intra_function_calls().
1444 */
1445 remove_insn_ops(insn);
1446
1447 annotate_call_site(file, insn, sibling);
1448}
1449
1450static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1451{
1452 /*
1453 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1454 * so convert them accordingly.
1455 */
1456 switch (insn->type) {
1457 case INSN_CALL:
1458 insn->type = INSN_CALL_DYNAMIC;
1459 break;
1460 case INSN_JUMP_UNCONDITIONAL:
1461 insn->type = INSN_JUMP_DYNAMIC;
1462 break;
1463 case INSN_JUMP_CONDITIONAL:
1464 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1465 break;
1466 default:
1467 return;
1468 }
1469
1470 insn->retpoline_safe = true;
1471
1472 /*
1473 * Whatever stack impact regular CALLs have, should be undone
1474 * by the RETURN of the called function.
1475 *
1476 * Annotated intra-function calls retain the stack_ops but
1477 * are converted to JUMP, see read_intra_function_calls().
1478 */
1479 remove_insn_ops(insn);
1480
1481 annotate_call_site(file, insn, false);
1482}
1483
1484static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1485{
1486 /*
1487 * Return thunk tail calls are really just returns in disguise,
1488 * so convert them accordingly.
1489 */
1490 insn->type = INSN_RETURN;
1491 insn->retpoline_safe = true;
1492
1493 if (add)
1494 list_add_tail(&insn->call_node, &file->return_thunk_list);
1495}
1496
1497static bool is_first_func_insn(struct objtool_file *file,
1498 struct instruction *insn, struct symbol *sym)
1499{
1500 if (insn->offset == sym->offset)
1501 return true;
1502
1503 /* Allow direct CALL/JMP past ENDBR */
1504 if (opts.ibt) {
1505 struct instruction *prev = prev_insn_same_sym(file, insn);
1506
1507 if (prev && prev->type == INSN_ENDBR &&
1508 insn->offset == sym->offset + prev->len)
1509 return true;
1510 }
1511
1512 return false;
1513}
1514
1515/*
1516 * A sibling call is a tail-call to another symbol -- to differentiate from a
1517 * recursive tail-call which is to the same symbol.
1518 */
1519static bool jump_is_sibling_call(struct objtool_file *file,
1520 struct instruction *from, struct instruction *to)
1521{
1522 struct symbol *fs = from->sym;
1523 struct symbol *ts = to->sym;
1524
1525 /* Not a sibling call if from/to a symbol hole */
1526 if (!fs || !ts)
1527 return false;
1528
1529 /* Not a sibling call if not targeting the start of a symbol. */
1530 if (!is_first_func_insn(file, to, ts))
1531 return false;
1532
1533 /* Disallow sibling calls into STT_NOTYPE */
1534 if (ts->type == STT_NOTYPE)
1535 return false;
1536
1537 /* Must not be self to be a sibling */
1538 return fs->pfunc != ts->pfunc;
1539}
1540
1541/*
1542 * Find the destination instructions for all jumps.
1543 */
1544static int add_jump_destinations(struct objtool_file *file)
1545{
1546 struct instruction *insn, *jump_dest;
1547 struct reloc *reloc;
1548 struct section *dest_sec;
1549 unsigned long dest_off;
1550
1551 for_each_insn(file, insn) {
1552 if (insn->jump_dest) {
1553 /*
1554 * handle_group_alt() may have previously set
1555 * 'jump_dest' for some alternatives.
1556 */
1557 continue;
1558 }
1559 if (!is_static_jump(insn))
1560 continue;
1561
1562 reloc = insn_reloc(file, insn);
1563 if (!reloc) {
1564 dest_sec = insn->sec;
1565 dest_off = arch_jump_destination(insn);
1566 } else if (reloc->sym->type == STT_SECTION) {
1567 dest_sec = reloc->sym->sec;
1568 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1569 } else if (reloc->sym->retpoline_thunk) {
1570 add_retpoline_call(file, insn);
1571 continue;
1572 } else if (reloc->sym->return_thunk) {
1573 add_return_call(file, insn, true);
1574 continue;
1575 } else if (insn_func(insn)) {
1576 /*
1577 * External sibling call or internal sibling call with
1578 * STT_FUNC reloc.
1579 */
1580 add_call_dest(file, insn, reloc->sym, true);
1581 continue;
1582 } else if (reloc->sym->sec->idx) {
1583 dest_sec = reloc->sym->sec;
1584 dest_off = reloc->sym->sym.st_value +
1585 arch_dest_reloc_offset(reloc_addend(reloc));
1586 } else {
1587 /* non-func asm code jumping to another file */
1588 continue;
1589 }
1590
1591 jump_dest = find_insn(file, dest_sec, dest_off);
1592 if (!jump_dest) {
1593 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1594
1595 /*
1596 * This is a special case for retbleed_untrain_ret().
1597 * It jumps to __x86_return_thunk(), but objtool
1598 * can't find the thunk's starting RET
1599 * instruction, because the RET is also in the
1600 * middle of another instruction. Objtool only
1601 * knows about the outer instruction.
1602 */
1603 if (sym && sym->embedded_insn) {
1604 add_return_call(file, insn, false);
1605 continue;
1606 }
1607
1608 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1609 dest_sec->name, dest_off);
1610 return -1;
1611 }
1612
1613 /*
1614 * An intra-TU jump in retpoline.o might not have a relocation
1615 * for its jump dest, in which case the above
1616 * add_{retpoline,return}_call() didn't happen.
1617 */
1618 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1619 if (jump_dest->sym->retpoline_thunk) {
1620 add_retpoline_call(file, insn);
1621 continue;
1622 }
1623 if (jump_dest->sym->return_thunk) {
1624 add_return_call(file, insn, true);
1625 continue;
1626 }
1627 }
1628
1629 /*
1630 * Cross-function jump.
1631 */
1632 if (insn_func(insn) && insn_func(jump_dest) &&
1633 insn_func(insn) != insn_func(jump_dest)) {
1634
1635 /*
1636 * For GCC 8+, create parent/child links for any cold
1637 * subfunctions. This is _mostly_ redundant with a
1638 * similar initialization in read_symbols().
1639 *
1640 * If a function has aliases, we want the *first* such
1641 * function in the symbol table to be the subfunction's
1642 * parent. In that case we overwrite the
1643 * initialization done in read_symbols().
1644 *
1645 * However this code can't completely replace the
1646 * read_symbols() code because this doesn't detect the
1647 * case where the parent function's only reference to a
1648 * subfunction is through a jump table.
1649 */
1650 if (!strstr(insn_func(insn)->name, ".cold") &&
1651 strstr(insn_func(jump_dest)->name, ".cold")) {
1652 insn_func(insn)->cfunc = insn_func(jump_dest);
1653 insn_func(jump_dest)->pfunc = insn_func(insn);
1654 }
1655 }
1656
1657 if (jump_is_sibling_call(file, insn, jump_dest)) {
1658 /*
1659 * Internal sibling call without reloc or with
1660 * STT_SECTION reloc.
1661 */
1662 add_call_dest(file, insn, insn_func(jump_dest), true);
1663 continue;
1664 }
1665
1666 insn->jump_dest = jump_dest;
1667 }
1668
1669 return 0;
1670}
1671
1672static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1673{
1674 struct symbol *call_dest;
1675
1676 call_dest = find_func_by_offset(sec, offset);
1677 if (!call_dest)
1678 call_dest = find_symbol_by_offset(sec, offset);
1679
1680 return call_dest;
1681}
1682
1683/*
1684 * Find the destination instructions for all calls.
1685 */
1686static int add_call_destinations(struct objtool_file *file)
1687{
1688 struct instruction *insn;
1689 unsigned long dest_off;
1690 struct symbol *dest;
1691 struct reloc *reloc;
1692
1693 for_each_insn(file, insn) {
1694 if (insn->type != INSN_CALL)
1695 continue;
1696
1697 reloc = insn_reloc(file, insn);
1698 if (!reloc) {
1699 dest_off = arch_jump_destination(insn);
1700 dest = find_call_destination(insn->sec, dest_off);
1701
1702 add_call_dest(file, insn, dest, false);
1703
1704 if (insn->ignore)
1705 continue;
1706
1707 if (!insn_call_dest(insn)) {
1708 WARN_INSN(insn, "unannotated intra-function call");
1709 return -1;
1710 }
1711
1712 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1713 WARN_INSN(insn, "unsupported call to non-function");
1714 return -1;
1715 }
1716
1717 } else if (reloc->sym->type == STT_SECTION) {
1718 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1719 dest = find_call_destination(reloc->sym->sec, dest_off);
1720 if (!dest) {
1721 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1722 reloc->sym->sec->name, dest_off);
1723 return -1;
1724 }
1725
1726 add_call_dest(file, insn, dest, false);
1727
1728 } else if (reloc->sym->retpoline_thunk) {
1729 add_retpoline_call(file, insn);
1730
1731 } else
1732 add_call_dest(file, insn, reloc->sym, false);
1733 }
1734
1735 return 0;
1736}
1737
1738/*
1739 * The .alternatives section requires some extra special care over and above
1740 * other special sections because alternatives are patched in place.
1741 */
1742static int handle_group_alt(struct objtool_file *file,
1743 struct special_alt *special_alt,
1744 struct instruction *orig_insn,
1745 struct instruction **new_insn)
1746{
1747 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1748 struct alt_group *orig_alt_group, *new_alt_group;
1749 unsigned long dest_off;
1750
1751 orig_alt_group = orig_insn->alt_group;
1752 if (!orig_alt_group) {
1753 struct instruction *last_orig_insn = NULL;
1754
1755 orig_alt_group = malloc(sizeof(*orig_alt_group));
1756 if (!orig_alt_group) {
1757 WARN("malloc failed");
1758 return -1;
1759 }
1760 orig_alt_group->cfi = calloc(special_alt->orig_len,
1761 sizeof(struct cfi_state *));
1762 if (!orig_alt_group->cfi) {
1763 WARN("calloc failed");
1764 return -1;
1765 }
1766
1767 insn = orig_insn;
1768 sec_for_each_insn_from(file, insn) {
1769 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1770 break;
1771
1772 insn->alt_group = orig_alt_group;
1773 last_orig_insn = insn;
1774 }
1775 orig_alt_group->orig_group = NULL;
1776 orig_alt_group->first_insn = orig_insn;
1777 orig_alt_group->last_insn = last_orig_insn;
1778 orig_alt_group->nop = NULL;
1779 } else {
1780 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1781 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1782 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1783 orig_alt_group->last_insn->offset +
1784 orig_alt_group->last_insn->len -
1785 orig_alt_group->first_insn->offset,
1786 special_alt->orig_len);
1787 return -1;
1788 }
1789 }
1790
1791 new_alt_group = malloc(sizeof(*new_alt_group));
1792 if (!new_alt_group) {
1793 WARN("malloc failed");
1794 return -1;
1795 }
1796
1797 if (special_alt->new_len < special_alt->orig_len) {
1798 /*
1799 * Insert a fake nop at the end to make the replacement
1800 * alt_group the same size as the original. This is needed to
1801 * allow propagate_alt_cfi() to do its magic. When the last
1802 * instruction affects the stack, the instruction after it (the
1803 * nop) will propagate the new state to the shared CFI array.
1804 */
1805 nop = malloc(sizeof(*nop));
1806 if (!nop) {
1807 WARN("malloc failed");
1808 return -1;
1809 }
1810 memset(nop, 0, sizeof(*nop));
1811
1812 nop->sec = special_alt->new_sec;
1813 nop->offset = special_alt->new_off + special_alt->new_len;
1814 nop->len = special_alt->orig_len - special_alt->new_len;
1815 nop->type = INSN_NOP;
1816 nop->sym = orig_insn->sym;
1817 nop->alt_group = new_alt_group;
1818 nop->ignore = orig_insn->ignore_alts;
1819 }
1820
1821 if (!special_alt->new_len) {
1822 *new_insn = nop;
1823 goto end;
1824 }
1825
1826 insn = *new_insn;
1827 sec_for_each_insn_from(file, insn) {
1828 struct reloc *alt_reloc;
1829
1830 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1831 break;
1832
1833 last_new_insn = insn;
1834
1835 insn->ignore = orig_insn->ignore_alts;
1836 insn->sym = orig_insn->sym;
1837 insn->alt_group = new_alt_group;
1838
1839 /*
1840 * Since alternative replacement code is copy/pasted by the
1841 * kernel after applying relocations, generally such code can't
1842 * have relative-address relocation references to outside the
1843 * .altinstr_replacement section, unless the arch's
1844 * alternatives code can adjust the relative offsets
1845 * accordingly.
1846 */
1847 alt_reloc = insn_reloc(file, insn);
1848 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1849 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1850
1851 WARN_INSN(insn, "unsupported relocation in alternatives section");
1852 return -1;
1853 }
1854
1855 if (!is_static_jump(insn))
1856 continue;
1857
1858 if (!insn->immediate)
1859 continue;
1860
1861 dest_off = arch_jump_destination(insn);
1862 if (dest_off == special_alt->new_off + special_alt->new_len) {
1863 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1864 if (!insn->jump_dest) {
1865 WARN_INSN(insn, "can't find alternative jump destination");
1866 return -1;
1867 }
1868 }
1869 }
1870
1871 if (!last_new_insn) {
1872 WARN_FUNC("can't find last new alternative instruction",
1873 special_alt->new_sec, special_alt->new_off);
1874 return -1;
1875 }
1876
1877end:
1878 new_alt_group->orig_group = orig_alt_group;
1879 new_alt_group->first_insn = *new_insn;
1880 new_alt_group->last_insn = last_new_insn;
1881 new_alt_group->nop = nop;
1882 new_alt_group->cfi = orig_alt_group->cfi;
1883 return 0;
1884}
1885
1886/*
1887 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1888 * If the original instruction is a jump, make the alt entry an effective nop
1889 * by just skipping the original instruction.
1890 */
1891static int handle_jump_alt(struct objtool_file *file,
1892 struct special_alt *special_alt,
1893 struct instruction *orig_insn,
1894 struct instruction **new_insn)
1895{
1896 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1897 orig_insn->type != INSN_NOP) {
1898
1899 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1900 return -1;
1901 }
1902
1903 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1904 struct reloc *reloc = insn_reloc(file, orig_insn);
1905
1906 if (reloc)
1907 set_reloc_type(file->elf, reloc, R_NONE);
1908 elf_write_insn(file->elf, orig_insn->sec,
1909 orig_insn->offset, orig_insn->len,
1910 arch_nop_insn(orig_insn->len));
1911 orig_insn->type = INSN_NOP;
1912 }
1913
1914 if (orig_insn->type == INSN_NOP) {
1915 if (orig_insn->len == 2)
1916 file->jl_nop_short++;
1917 else
1918 file->jl_nop_long++;
1919
1920 return 0;
1921 }
1922
1923 if (orig_insn->len == 2)
1924 file->jl_short++;
1925 else
1926 file->jl_long++;
1927
1928 *new_insn = next_insn_same_sec(file, orig_insn);
1929 return 0;
1930}
1931
1932/*
1933 * Read all the special sections which have alternate instructions which can be
1934 * patched in or redirected to at runtime. Each instruction having alternate
1935 * instruction(s) has them added to its insn->alts list, which will be
1936 * traversed in validate_branch().
1937 */
1938static int add_special_section_alts(struct objtool_file *file)
1939{
1940 struct list_head special_alts;
1941 struct instruction *orig_insn, *new_insn;
1942 struct special_alt *special_alt, *tmp;
1943 struct alternative *alt;
1944 int ret;
1945
1946 ret = special_get_alts(file->elf, &special_alts);
1947 if (ret)
1948 return ret;
1949
1950 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1951
1952 orig_insn = find_insn(file, special_alt->orig_sec,
1953 special_alt->orig_off);
1954 if (!orig_insn) {
1955 WARN_FUNC("special: can't find orig instruction",
1956 special_alt->orig_sec, special_alt->orig_off);
1957 ret = -1;
1958 goto out;
1959 }
1960
1961 new_insn = NULL;
1962 if (!special_alt->group || special_alt->new_len) {
1963 new_insn = find_insn(file, special_alt->new_sec,
1964 special_alt->new_off);
1965 if (!new_insn) {
1966 WARN_FUNC("special: can't find new instruction",
1967 special_alt->new_sec,
1968 special_alt->new_off);
1969 ret = -1;
1970 goto out;
1971 }
1972 }
1973
1974 if (special_alt->group) {
1975 if (!special_alt->orig_len) {
1976 WARN_INSN(orig_insn, "empty alternative entry");
1977 continue;
1978 }
1979
1980 ret = handle_group_alt(file, special_alt, orig_insn,
1981 &new_insn);
1982 if (ret)
1983 goto out;
1984 } else if (special_alt->jump_or_nop) {
1985 ret = handle_jump_alt(file, special_alt, orig_insn,
1986 &new_insn);
1987 if (ret)
1988 goto out;
1989 }
1990
1991 alt = malloc(sizeof(*alt));
1992 if (!alt) {
1993 WARN("malloc failed");
1994 ret = -1;
1995 goto out;
1996 }
1997
1998 alt->insn = new_insn;
1999 alt->skip_orig = special_alt->skip_orig;
2000 orig_insn->ignore_alts |= special_alt->skip_alt;
2001 alt->next = orig_insn->alts;
2002 orig_insn->alts = alt;
2003
2004 list_del(&special_alt->list);
2005 free(special_alt);
2006 }
2007
2008 if (opts.stats) {
2009 printf("jl\\\tNOP\tJMP\n");
2010 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2011 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2012 }
2013
2014out:
2015 return ret;
2016}
2017
2018static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2019 struct reloc *next_table)
2020{
2021 struct symbol *pfunc = insn_func(insn)->pfunc;
2022 struct reloc *table = insn_jump_table(insn);
2023 struct instruction *dest_insn;
2024 unsigned int prev_offset = 0;
2025 struct reloc *reloc = table;
2026 struct alternative *alt;
2027
2028 /*
2029 * Each @reloc is a switch table relocation which points to the target
2030 * instruction.
2031 */
2032 for_each_reloc_from(table->sec, reloc) {
2033
2034 /* Check for the end of the table: */
2035 if (reloc != table && reloc == next_table)
2036 break;
2037
2038 /* Make sure the table entries are consecutive: */
2039 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2040 break;
2041
2042 /* Detect function pointers from contiguous objects: */
2043 if (reloc->sym->sec == pfunc->sec &&
2044 reloc_addend(reloc) == pfunc->offset)
2045 break;
2046
2047 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2048 if (!dest_insn)
2049 break;
2050
2051 /* Make sure the destination is in the same function: */
2052 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2053 break;
2054
2055 alt = malloc(sizeof(*alt));
2056 if (!alt) {
2057 WARN("malloc failed");
2058 return -1;
2059 }
2060
2061 alt->insn = dest_insn;
2062 alt->next = insn->alts;
2063 insn->alts = alt;
2064 prev_offset = reloc_offset(reloc);
2065 }
2066
2067 if (!prev_offset) {
2068 WARN_INSN(insn, "can't find switch jump table");
2069 return -1;
2070 }
2071
2072 return 0;
2073}
2074
2075/*
2076 * find_jump_table() - Given a dynamic jump, find the switch jump table
2077 * associated with it.
2078 */
2079static struct reloc *find_jump_table(struct objtool_file *file,
2080 struct symbol *func,
2081 struct instruction *insn)
2082{
2083 struct reloc *table_reloc;
2084 struct instruction *dest_insn, *orig_insn = insn;
2085
2086 /*
2087 * Backward search using the @first_jump_src links, these help avoid
2088 * much of the 'in between' code. Which avoids us getting confused by
2089 * it.
2090 */
2091 for (;
2092 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2093 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2094
2095 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2096 break;
2097
2098 /* allow small jumps within the range */
2099 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2100 insn->jump_dest &&
2101 (insn->jump_dest->offset <= insn->offset ||
2102 insn->jump_dest->offset > orig_insn->offset))
2103 break;
2104
2105 table_reloc = arch_find_switch_table(file, insn);
2106 if (!table_reloc)
2107 continue;
2108 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2109 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2110 continue;
2111
2112 return table_reloc;
2113 }
2114
2115 return NULL;
2116}
2117
2118/*
2119 * First pass: Mark the head of each jump table so that in the next pass,
2120 * we know when a given jump table ends and the next one starts.
2121 */
2122static void mark_func_jump_tables(struct objtool_file *file,
2123 struct symbol *func)
2124{
2125 struct instruction *insn, *last = NULL;
2126 struct reloc *reloc;
2127
2128 func_for_each_insn(file, func, insn) {
2129 if (!last)
2130 last = insn;
2131
2132 /*
2133 * Store back-pointers for unconditional forward jumps such
2134 * that find_jump_table() can back-track using those and
2135 * avoid some potentially confusing code.
2136 */
2137 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2138 insn->offset > last->offset &&
2139 insn->jump_dest->offset > insn->offset &&
2140 !insn->jump_dest->first_jump_src) {
2141
2142 insn->jump_dest->first_jump_src = insn;
2143 last = insn->jump_dest;
2144 }
2145
2146 if (insn->type != INSN_JUMP_DYNAMIC)
2147 continue;
2148
2149 reloc = find_jump_table(file, func, insn);
2150 if (reloc)
2151 insn->_jump_table = reloc;
2152 }
2153}
2154
2155static int add_func_jump_tables(struct objtool_file *file,
2156 struct symbol *func)
2157{
2158 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2159 int ret = 0;
2160
2161 func_for_each_insn(file, func, insn) {
2162 if (!insn_jump_table(insn))
2163 continue;
2164
2165 if (!insn_t1) {
2166 insn_t1 = insn;
2167 continue;
2168 }
2169
2170 insn_t2 = insn;
2171
2172 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2173 if (ret)
2174 return ret;
2175
2176 insn_t1 = insn_t2;
2177 }
2178
2179 if (insn_t1)
2180 ret = add_jump_table(file, insn_t1, NULL);
2181
2182 return ret;
2183}
2184
2185/*
2186 * For some switch statements, gcc generates a jump table in the .rodata
2187 * section which contains a list of addresses within the function to jump to.
2188 * This finds these jump tables and adds them to the insn->alts lists.
2189 */
2190static int add_jump_table_alts(struct objtool_file *file)
2191{
2192 struct symbol *func;
2193 int ret;
2194
2195 if (!file->rodata)
2196 return 0;
2197
2198 for_each_sym(file, func) {
2199 if (func->type != STT_FUNC)
2200 continue;
2201
2202 mark_func_jump_tables(file, func);
2203 ret = add_func_jump_tables(file, func);
2204 if (ret)
2205 return ret;
2206 }
2207
2208 return 0;
2209}
2210
2211static void set_func_state(struct cfi_state *state)
2212{
2213 state->cfa = initial_func_cfi.cfa;
2214 memcpy(&state->regs, &initial_func_cfi.regs,
2215 CFI_NUM_REGS * sizeof(struct cfi_reg));
2216 state->stack_size = initial_func_cfi.cfa.offset;
2217 state->type = UNWIND_HINT_TYPE_CALL;
2218}
2219
2220static int read_unwind_hints(struct objtool_file *file)
2221{
2222 struct cfi_state cfi = init_cfi;
2223 struct section *sec;
2224 struct unwind_hint *hint;
2225 struct instruction *insn;
2226 struct reloc *reloc;
2227 int i;
2228
2229 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2230 if (!sec)
2231 return 0;
2232
2233 if (!sec->rsec) {
2234 WARN("missing .rela.discard.unwind_hints section");
2235 return -1;
2236 }
2237
2238 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2239 WARN("struct unwind_hint size mismatch");
2240 return -1;
2241 }
2242
2243 file->hints = true;
2244
2245 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2246 hint = (struct unwind_hint *)sec->data->d_buf + i;
2247
2248 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2249 if (!reloc) {
2250 WARN("can't find reloc for unwind_hints[%d]", i);
2251 return -1;
2252 }
2253
2254 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2255 if (!insn) {
2256 WARN("can't find insn for unwind_hints[%d]", i);
2257 return -1;
2258 }
2259
2260 insn->hint = true;
2261
2262 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2263 insn->cfi = &force_undefined_cfi;
2264 continue;
2265 }
2266
2267 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2268 insn->hint = false;
2269 insn->save = true;
2270 continue;
2271 }
2272
2273 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2274 insn->restore = true;
2275 continue;
2276 }
2277
2278 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2279 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2280
2281 if (sym && sym->bind == STB_GLOBAL) {
2282 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2283 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2284 }
2285 }
2286 }
2287
2288 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2289 insn->cfi = &func_cfi;
2290 continue;
2291 }
2292
2293 if (insn->cfi)
2294 cfi = *(insn->cfi);
2295
2296 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2297 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2298 return -1;
2299 }
2300
2301 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2302 cfi.type = hint->type;
2303 cfi.signal = hint->signal;
2304
2305 insn->cfi = cfi_hash_find_or_add(&cfi);
2306 }
2307
2308 return 0;
2309}
2310
2311static int read_noendbr_hints(struct objtool_file *file)
2312{
2313 struct instruction *insn;
2314 struct section *rsec;
2315 struct reloc *reloc;
2316
2317 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2318 if (!rsec)
2319 return 0;
2320
2321 for_each_reloc(rsec, reloc) {
2322 insn = find_insn(file, reloc->sym->sec,
2323 reloc->sym->offset + reloc_addend(reloc));
2324 if (!insn) {
2325 WARN("bad .discard.noendbr entry");
2326 return -1;
2327 }
2328
2329 insn->noendbr = 1;
2330 }
2331
2332 return 0;
2333}
2334
2335static int read_retpoline_hints(struct objtool_file *file)
2336{
2337 struct section *rsec;
2338 struct instruction *insn;
2339 struct reloc *reloc;
2340
2341 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2342 if (!rsec)
2343 return 0;
2344
2345 for_each_reloc(rsec, reloc) {
2346 if (reloc->sym->type != STT_SECTION) {
2347 WARN("unexpected relocation symbol type in %s", rsec->name);
2348 return -1;
2349 }
2350
2351 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2352 if (!insn) {
2353 WARN("bad .discard.retpoline_safe entry");
2354 return -1;
2355 }
2356
2357 if (insn->type != INSN_JUMP_DYNAMIC &&
2358 insn->type != INSN_CALL_DYNAMIC &&
2359 insn->type != INSN_RETURN &&
2360 insn->type != INSN_NOP) {
2361 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2362 return -1;
2363 }
2364
2365 insn->retpoline_safe = true;
2366 }
2367
2368 return 0;
2369}
2370
2371static int read_instr_hints(struct objtool_file *file)
2372{
2373 struct section *rsec;
2374 struct instruction *insn;
2375 struct reloc *reloc;
2376
2377 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2378 if (!rsec)
2379 return 0;
2380
2381 for_each_reloc(rsec, reloc) {
2382 if (reloc->sym->type != STT_SECTION) {
2383 WARN("unexpected relocation symbol type in %s", rsec->name);
2384 return -1;
2385 }
2386
2387 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2388 if (!insn) {
2389 WARN("bad .discard.instr_end entry");
2390 return -1;
2391 }
2392
2393 insn->instr--;
2394 }
2395
2396 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2397 if (!rsec)
2398 return 0;
2399
2400 for_each_reloc(rsec, reloc) {
2401 if (reloc->sym->type != STT_SECTION) {
2402 WARN("unexpected relocation symbol type in %s", rsec->name);
2403 return -1;
2404 }
2405
2406 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2407 if (!insn) {
2408 WARN("bad .discard.instr_begin entry");
2409 return -1;
2410 }
2411
2412 insn->instr++;
2413 }
2414
2415 return 0;
2416}
2417
2418static int read_validate_unret_hints(struct objtool_file *file)
2419{
2420 struct section *rsec;
2421 struct instruction *insn;
2422 struct reloc *reloc;
2423
2424 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2425 if (!rsec)
2426 return 0;
2427
2428 for_each_reloc(rsec, reloc) {
2429 if (reloc->sym->type != STT_SECTION) {
2430 WARN("unexpected relocation symbol type in %s", rsec->name);
2431 return -1;
2432 }
2433
2434 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2435 if (!insn) {
2436 WARN("bad .discard.instr_end entry");
2437 return -1;
2438 }
2439 insn->unret = 1;
2440 }
2441
2442 return 0;
2443}
2444
2445
2446static int read_intra_function_calls(struct objtool_file *file)
2447{
2448 struct instruction *insn;
2449 struct section *rsec;
2450 struct reloc *reloc;
2451
2452 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2453 if (!rsec)
2454 return 0;
2455
2456 for_each_reloc(rsec, reloc) {
2457 unsigned long dest_off;
2458
2459 if (reloc->sym->type != STT_SECTION) {
2460 WARN("unexpected relocation symbol type in %s",
2461 rsec->name);
2462 return -1;
2463 }
2464
2465 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2466 if (!insn) {
2467 WARN("bad .discard.intra_function_call entry");
2468 return -1;
2469 }
2470
2471 if (insn->type != INSN_CALL) {
2472 WARN_INSN(insn, "intra_function_call not a direct call");
2473 return -1;
2474 }
2475
2476 /*
2477 * Treat intra-function CALLs as JMPs, but with a stack_op.
2478 * See add_call_destinations(), which strips stack_ops from
2479 * normal CALLs.
2480 */
2481 insn->type = INSN_JUMP_UNCONDITIONAL;
2482
2483 dest_off = arch_jump_destination(insn);
2484 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2485 if (!insn->jump_dest) {
2486 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2487 insn->sec->name, dest_off);
2488 return -1;
2489 }
2490 }
2491
2492 return 0;
2493}
2494
2495/*
2496 * Return true if name matches an instrumentation function, where calls to that
2497 * function from noinstr code can safely be removed, but compilers won't do so.
2498 */
2499static bool is_profiling_func(const char *name)
2500{
2501 /*
2502 * Many compilers cannot disable KCOV with a function attribute.
2503 */
2504 if (!strncmp(name, "__sanitizer_cov_", 16))
2505 return true;
2506
2507 /*
2508 * Some compilers currently do not remove __tsan_func_entry/exit nor
2509 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2510 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2511 * minimum Clang version is 14.0, this can be removed.
2512 */
2513 if (!strncmp(name, "__tsan_func_", 12) ||
2514 !strcmp(name, "__tsan_atomic_signal_fence"))
2515 return true;
2516
2517 return false;
2518}
2519
2520static int classify_symbols(struct objtool_file *file)
2521{
2522 struct symbol *func;
2523
2524 for_each_sym(file, func) {
2525 if (func->bind != STB_GLOBAL)
2526 continue;
2527
2528 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2529 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2530 func->static_call_tramp = true;
2531
2532 if (arch_is_retpoline(func))
2533 func->retpoline_thunk = true;
2534
2535 if (arch_is_rethunk(func))
2536 func->return_thunk = true;
2537
2538 if (arch_is_embedded_insn(func))
2539 func->embedded_insn = true;
2540
2541 if (arch_ftrace_match(func->name))
2542 func->fentry = true;
2543
2544 if (is_profiling_func(func->name))
2545 func->profiling_func = true;
2546 }
2547
2548 return 0;
2549}
2550
2551static void mark_rodata(struct objtool_file *file)
2552{
2553 struct section *sec;
2554 bool found = false;
2555
2556 /*
2557 * Search for the following rodata sections, each of which can
2558 * potentially contain jump tables:
2559 *
2560 * - .rodata: can contain GCC switch tables
2561 * - .rodata.<func>: same, if -fdata-sections is being used
2562 * - .rodata..c_jump_table: contains C annotated jump tables
2563 *
2564 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2565 */
2566 for_each_sec(file, sec) {
2567 if (!strncmp(sec->name, ".rodata", 7) &&
2568 !strstr(sec->name, ".str1.")) {
2569 sec->rodata = true;
2570 found = true;
2571 }
2572 }
2573
2574 file->rodata = found;
2575}
2576
2577static int decode_sections(struct objtool_file *file)
2578{
2579 int ret;
2580
2581 mark_rodata(file);
2582
2583 ret = init_pv_ops(file);
2584 if (ret)
2585 return ret;
2586
2587 /*
2588 * Must be before add_{jump_call}_destination.
2589 */
2590 ret = classify_symbols(file);
2591 if (ret)
2592 return ret;
2593
2594 ret = decode_instructions(file);
2595 if (ret)
2596 return ret;
2597
2598 add_ignores(file);
2599 add_uaccess_safe(file);
2600
2601 ret = add_ignore_alternatives(file);
2602 if (ret)
2603 return ret;
2604
2605 /*
2606 * Must be before read_unwind_hints() since that needs insn->noendbr.
2607 */
2608 ret = read_noendbr_hints(file);
2609 if (ret)
2610 return ret;
2611
2612 /*
2613 * Must be before add_jump_destinations(), which depends on 'func'
2614 * being set for alternatives, to enable proper sibling call detection.
2615 */
2616 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2617 ret = add_special_section_alts(file);
2618 if (ret)
2619 return ret;
2620 }
2621
2622 ret = add_jump_destinations(file);
2623 if (ret)
2624 return ret;
2625
2626 /*
2627 * Must be before add_call_destination(); it changes INSN_CALL to
2628 * INSN_JUMP.
2629 */
2630 ret = read_intra_function_calls(file);
2631 if (ret)
2632 return ret;
2633
2634 ret = add_call_destinations(file);
2635 if (ret)
2636 return ret;
2637
2638 /*
2639 * Must be after add_call_destinations() such that it can override
2640 * dead_end_function() marks.
2641 */
2642 ret = add_dead_ends(file);
2643 if (ret)
2644 return ret;
2645
2646 ret = add_jump_table_alts(file);
2647 if (ret)
2648 return ret;
2649
2650 ret = read_unwind_hints(file);
2651 if (ret)
2652 return ret;
2653
2654 ret = read_retpoline_hints(file);
2655 if (ret)
2656 return ret;
2657
2658 ret = read_instr_hints(file);
2659 if (ret)
2660 return ret;
2661
2662 ret = read_validate_unret_hints(file);
2663 if (ret)
2664 return ret;
2665
2666 return 0;
2667}
2668
2669static bool is_special_call(struct instruction *insn)
2670{
2671 if (insn->type == INSN_CALL) {
2672 struct symbol *dest = insn_call_dest(insn);
2673
2674 if (!dest)
2675 return false;
2676
2677 if (dest->fentry || dest->embedded_insn)
2678 return true;
2679 }
2680
2681 return false;
2682}
2683
2684static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2685{
2686 struct cfi_state *cfi = &state->cfi;
2687 int i;
2688
2689 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2690 return true;
2691
2692 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2693 return true;
2694
2695 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2696 return true;
2697
2698 for (i = 0; i < CFI_NUM_REGS; i++) {
2699 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2700 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2701 return true;
2702 }
2703
2704 return false;
2705}
2706
2707static bool check_reg_frame_pos(const struct cfi_reg *reg,
2708 int expected_offset)
2709{
2710 return reg->base == CFI_CFA &&
2711 reg->offset == expected_offset;
2712}
2713
2714static bool has_valid_stack_frame(struct insn_state *state)
2715{
2716 struct cfi_state *cfi = &state->cfi;
2717
2718 if (cfi->cfa.base == CFI_BP &&
2719 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2720 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2721 return true;
2722
2723 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2724 return true;
2725
2726 return false;
2727}
2728
2729static int update_cfi_state_regs(struct instruction *insn,
2730 struct cfi_state *cfi,
2731 struct stack_op *op)
2732{
2733 struct cfi_reg *cfa = &cfi->cfa;
2734
2735 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2736 return 0;
2737
2738 /* push */
2739 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2740 cfa->offset += 8;
2741
2742 /* pop */
2743 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2744 cfa->offset -= 8;
2745
2746 /* add immediate to sp */
2747 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2748 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2749 cfa->offset -= op->src.offset;
2750
2751 return 0;
2752}
2753
2754static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2755{
2756 if (arch_callee_saved_reg(reg) &&
2757 cfi->regs[reg].base == CFI_UNDEFINED) {
2758 cfi->regs[reg].base = base;
2759 cfi->regs[reg].offset = offset;
2760 }
2761}
2762
2763static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2764{
2765 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2766 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2767}
2768
2769/*
2770 * A note about DRAP stack alignment:
2771 *
2772 * GCC has the concept of a DRAP register, which is used to help keep track of
2773 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2774 * register. The typical DRAP pattern is:
2775 *
2776 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2777 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2778 * 41 ff 72 f8 pushq -0x8(%r10)
2779 * 55 push %rbp
2780 * 48 89 e5 mov %rsp,%rbp
2781 * (more pushes)
2782 * 41 52 push %r10
2783 * ...
2784 * 41 5a pop %r10
2785 * (more pops)
2786 * 5d pop %rbp
2787 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2788 * c3 retq
2789 *
2790 * There are some variations in the epilogues, like:
2791 *
2792 * 5b pop %rbx
2793 * 41 5a pop %r10
2794 * 41 5c pop %r12
2795 * 41 5d pop %r13
2796 * 41 5e pop %r14
2797 * c9 leaveq
2798 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2799 * c3 retq
2800 *
2801 * and:
2802 *
2803 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2804 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2805 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2806 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2807 * c9 leaveq
2808 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2809 * c3 retq
2810 *
2811 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2812 * restored beforehand:
2813 *
2814 * 41 55 push %r13
2815 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2816 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2817 * ...
2818 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2819 * 41 5d pop %r13
2820 * c3 retq
2821 */
2822static int update_cfi_state(struct instruction *insn,
2823 struct instruction *next_insn,
2824 struct cfi_state *cfi, struct stack_op *op)
2825{
2826 struct cfi_reg *cfa = &cfi->cfa;
2827 struct cfi_reg *regs = cfi->regs;
2828
2829 /* ignore UNWIND_HINT_UNDEFINED regions */
2830 if (cfi->force_undefined)
2831 return 0;
2832
2833 /* stack operations don't make sense with an undefined CFA */
2834 if (cfa->base == CFI_UNDEFINED) {
2835 if (insn_func(insn)) {
2836 WARN_INSN(insn, "undefined stack state");
2837 return -1;
2838 }
2839 return 0;
2840 }
2841
2842 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2843 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2844 return update_cfi_state_regs(insn, cfi, op);
2845
2846 switch (op->dest.type) {
2847
2848 case OP_DEST_REG:
2849 switch (op->src.type) {
2850
2851 case OP_SRC_REG:
2852 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2853 cfa->base == CFI_SP &&
2854 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2855
2856 /* mov %rsp, %rbp */
2857 cfa->base = op->dest.reg;
2858 cfi->bp_scratch = false;
2859 }
2860
2861 else if (op->src.reg == CFI_SP &&
2862 op->dest.reg == CFI_BP && cfi->drap) {
2863
2864 /* drap: mov %rsp, %rbp */
2865 regs[CFI_BP].base = CFI_BP;
2866 regs[CFI_BP].offset = -cfi->stack_size;
2867 cfi->bp_scratch = false;
2868 }
2869
2870 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2871
2872 /*
2873 * mov %rsp, %reg
2874 *
2875 * This is needed for the rare case where GCC
2876 * does:
2877 *
2878 * mov %rsp, %rax
2879 * ...
2880 * mov %rax, %rsp
2881 */
2882 cfi->vals[op->dest.reg].base = CFI_CFA;
2883 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2884 }
2885
2886 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2887 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2888
2889 /*
2890 * mov %rbp, %rsp
2891 *
2892 * Restore the original stack pointer (Clang).
2893 */
2894 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2895 }
2896
2897 else if (op->dest.reg == cfa->base) {
2898
2899 /* mov %reg, %rsp */
2900 if (cfa->base == CFI_SP &&
2901 cfi->vals[op->src.reg].base == CFI_CFA) {
2902
2903 /*
2904 * This is needed for the rare case
2905 * where GCC does something dumb like:
2906 *
2907 * lea 0x8(%rsp), %rcx
2908 * ...
2909 * mov %rcx, %rsp
2910 */
2911 cfa->offset = -cfi->vals[op->src.reg].offset;
2912 cfi->stack_size = cfa->offset;
2913
2914 } else if (cfa->base == CFI_SP &&
2915 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2916 cfi->vals[op->src.reg].offset == cfa->offset) {
2917
2918 /*
2919 * Stack swizzle:
2920 *
2921 * 1: mov %rsp, (%[tos])
2922 * 2: mov %[tos], %rsp
2923 * ...
2924 * 3: pop %rsp
2925 *
2926 * Where:
2927 *
2928 * 1 - places a pointer to the previous
2929 * stack at the Top-of-Stack of the
2930 * new stack.
2931 *
2932 * 2 - switches to the new stack.
2933 *
2934 * 3 - pops the Top-of-Stack to restore
2935 * the original stack.
2936 *
2937 * Note: we set base to SP_INDIRECT
2938 * here and preserve offset. Therefore
2939 * when the unwinder reaches ToS it
2940 * will dereference SP and then add the
2941 * offset to find the next frame, IOW:
2942 * (%rsp) + offset.
2943 */
2944 cfa->base = CFI_SP_INDIRECT;
2945
2946 } else {
2947 cfa->base = CFI_UNDEFINED;
2948 cfa->offset = 0;
2949 }
2950 }
2951
2952 else if (op->dest.reg == CFI_SP &&
2953 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2954 cfi->vals[op->src.reg].offset == cfa->offset) {
2955
2956 /*
2957 * The same stack swizzle case 2) as above. But
2958 * because we can't change cfa->base, case 3)
2959 * will become a regular POP. Pretend we're a
2960 * PUSH so things don't go unbalanced.
2961 */
2962 cfi->stack_size += 8;
2963 }
2964
2965
2966 break;
2967
2968 case OP_SRC_ADD:
2969 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2970
2971 /* add imm, %rsp */
2972 cfi->stack_size -= op->src.offset;
2973 if (cfa->base == CFI_SP)
2974 cfa->offset -= op->src.offset;
2975 break;
2976 }
2977
2978 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2979
2980 /* lea disp(%rbp), %rsp */
2981 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2982 break;
2983 }
2984
2985 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2986
2987 /* drap: lea disp(%rsp), %drap */
2988 cfi->drap_reg = op->dest.reg;
2989
2990 /*
2991 * lea disp(%rsp), %reg
2992 *
2993 * This is needed for the rare case where GCC
2994 * does something dumb like:
2995 *
2996 * lea 0x8(%rsp), %rcx
2997 * ...
2998 * mov %rcx, %rsp
2999 */
3000 cfi->vals[op->dest.reg].base = CFI_CFA;
3001 cfi->vals[op->dest.reg].offset = \
3002 -cfi->stack_size + op->src.offset;
3003
3004 break;
3005 }
3006
3007 if (cfi->drap && op->dest.reg == CFI_SP &&
3008 op->src.reg == cfi->drap_reg) {
3009
3010 /* drap: lea disp(%drap), %rsp */
3011 cfa->base = CFI_SP;
3012 cfa->offset = cfi->stack_size = -op->src.offset;
3013 cfi->drap_reg = CFI_UNDEFINED;
3014 cfi->drap = false;
3015 break;
3016 }
3017
3018 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3019 WARN_INSN(insn, "unsupported stack register modification");
3020 return -1;
3021 }
3022
3023 break;
3024
3025 case OP_SRC_AND:
3026 if (op->dest.reg != CFI_SP ||
3027 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3028 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3029 WARN_INSN(insn, "unsupported stack pointer realignment");
3030 return -1;
3031 }
3032
3033 if (cfi->drap_reg != CFI_UNDEFINED) {
3034 /* drap: and imm, %rsp */
3035 cfa->base = cfi->drap_reg;
3036 cfa->offset = cfi->stack_size = 0;
3037 cfi->drap = true;
3038 }
3039
3040 /*
3041 * Older versions of GCC (4.8ish) realign the stack
3042 * without DRAP, with a frame pointer.
3043 */
3044
3045 break;
3046
3047 case OP_SRC_POP:
3048 case OP_SRC_POPF:
3049 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3050
3051 /* pop %rsp; # restore from a stack swizzle */
3052 cfa->base = CFI_SP;
3053 break;
3054 }
3055
3056 if (!cfi->drap && op->dest.reg == cfa->base) {
3057
3058 /* pop %rbp */
3059 cfa->base = CFI_SP;
3060 }
3061
3062 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3063 op->dest.reg == cfi->drap_reg &&
3064 cfi->drap_offset == -cfi->stack_size) {
3065
3066 /* drap: pop %drap */
3067 cfa->base = cfi->drap_reg;
3068 cfa->offset = 0;
3069 cfi->drap_offset = -1;
3070
3071 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3072
3073 /* pop %reg */
3074 restore_reg(cfi, op->dest.reg);
3075 }
3076
3077 cfi->stack_size -= 8;
3078 if (cfa->base == CFI_SP)
3079 cfa->offset -= 8;
3080
3081 break;
3082
3083 case OP_SRC_REG_INDIRECT:
3084 if (!cfi->drap && op->dest.reg == cfa->base &&
3085 op->dest.reg == CFI_BP) {
3086
3087 /* mov disp(%rsp), %rbp */
3088 cfa->base = CFI_SP;
3089 cfa->offset = cfi->stack_size;
3090 }
3091
3092 if (cfi->drap && op->src.reg == CFI_BP &&
3093 op->src.offset == cfi->drap_offset) {
3094
3095 /* drap: mov disp(%rbp), %drap */
3096 cfa->base = cfi->drap_reg;
3097 cfa->offset = 0;
3098 cfi->drap_offset = -1;
3099 }
3100
3101 if (cfi->drap && op->src.reg == CFI_BP &&
3102 op->src.offset == regs[op->dest.reg].offset) {
3103
3104 /* drap: mov disp(%rbp), %reg */
3105 restore_reg(cfi, op->dest.reg);
3106
3107 } else if (op->src.reg == cfa->base &&
3108 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3109
3110 /* mov disp(%rbp), %reg */
3111 /* mov disp(%rsp), %reg */
3112 restore_reg(cfi, op->dest.reg);
3113
3114 } else if (op->src.reg == CFI_SP &&
3115 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3116
3117 /* mov disp(%rsp), %reg */
3118 restore_reg(cfi, op->dest.reg);
3119 }
3120
3121 break;
3122
3123 default:
3124 WARN_INSN(insn, "unknown stack-related instruction");
3125 return -1;
3126 }
3127
3128 break;
3129
3130 case OP_DEST_PUSH:
3131 case OP_DEST_PUSHF:
3132 cfi->stack_size += 8;
3133 if (cfa->base == CFI_SP)
3134 cfa->offset += 8;
3135
3136 if (op->src.type != OP_SRC_REG)
3137 break;
3138
3139 if (cfi->drap) {
3140 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3141
3142 /* drap: push %drap */
3143 cfa->base = CFI_BP_INDIRECT;
3144 cfa->offset = -cfi->stack_size;
3145
3146 /* save drap so we know when to restore it */
3147 cfi->drap_offset = -cfi->stack_size;
3148
3149 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3150
3151 /* drap: push %rbp */
3152 cfi->stack_size = 0;
3153
3154 } else {
3155
3156 /* drap: push %reg */
3157 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3158 }
3159
3160 } else {
3161
3162 /* push %reg */
3163 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3164 }
3165
3166 /* detect when asm code uses rbp as a scratch register */
3167 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3168 cfa->base != CFI_BP)
3169 cfi->bp_scratch = true;
3170 break;
3171
3172 case OP_DEST_REG_INDIRECT:
3173
3174 if (cfi->drap) {
3175 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3176
3177 /* drap: mov %drap, disp(%rbp) */
3178 cfa->base = CFI_BP_INDIRECT;
3179 cfa->offset = op->dest.offset;
3180
3181 /* save drap offset so we know when to restore it */
3182 cfi->drap_offset = op->dest.offset;
3183 } else {
3184
3185 /* drap: mov reg, disp(%rbp) */
3186 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3187 }
3188
3189 } else if (op->dest.reg == cfa->base) {
3190
3191 /* mov reg, disp(%rbp) */
3192 /* mov reg, disp(%rsp) */
3193 save_reg(cfi, op->src.reg, CFI_CFA,
3194 op->dest.offset - cfi->cfa.offset);
3195
3196 } else if (op->dest.reg == CFI_SP) {
3197
3198 /* mov reg, disp(%rsp) */
3199 save_reg(cfi, op->src.reg, CFI_CFA,
3200 op->dest.offset - cfi->stack_size);
3201
3202 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3203
3204 /* mov %rsp, (%reg); # setup a stack swizzle. */
3205 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3206 cfi->vals[op->dest.reg].offset = cfa->offset;
3207 }
3208
3209 break;
3210
3211 case OP_DEST_MEM:
3212 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3213 WARN_INSN(insn, "unknown stack-related memory operation");
3214 return -1;
3215 }
3216
3217 /* pop mem */
3218 cfi->stack_size -= 8;
3219 if (cfa->base == CFI_SP)
3220 cfa->offset -= 8;
3221
3222 break;
3223
3224 default:
3225 WARN_INSN(insn, "unknown stack-related instruction");
3226 return -1;
3227 }
3228
3229 return 0;
3230}
3231
3232/*
3233 * The stack layouts of alternatives instructions can sometimes diverge when
3234 * they have stack modifications. That's fine as long as the potential stack
3235 * layouts don't conflict at any given potential instruction boundary.
3236 *
3237 * Flatten the CFIs of the different alternative code streams (both original
3238 * and replacement) into a single shared CFI array which can be used to detect
3239 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3240 */
3241static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3242{
3243 struct cfi_state **alt_cfi;
3244 int group_off;
3245
3246 if (!insn->alt_group)
3247 return 0;
3248
3249 if (!insn->cfi) {
3250 WARN("CFI missing");
3251 return -1;
3252 }
3253
3254 alt_cfi = insn->alt_group->cfi;
3255 group_off = insn->offset - insn->alt_group->first_insn->offset;
3256
3257 if (!alt_cfi[group_off]) {
3258 alt_cfi[group_off] = insn->cfi;
3259 } else {
3260 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3261 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3262 struct instruction *orig = orig_group->first_insn;
3263 char *where = offstr(insn->sec, insn->offset);
3264 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3265 free(where);
3266 return -1;
3267 }
3268 }
3269
3270 return 0;
3271}
3272
3273static int handle_insn_ops(struct instruction *insn,
3274 struct instruction *next_insn,
3275 struct insn_state *state)
3276{
3277 struct stack_op *op;
3278
3279 for (op = insn->stack_ops; op; op = op->next) {
3280
3281 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3282 return 1;
3283
3284 if (!insn->alt_group)
3285 continue;
3286
3287 if (op->dest.type == OP_DEST_PUSHF) {
3288 if (!state->uaccess_stack) {
3289 state->uaccess_stack = 1;
3290 } else if (state->uaccess_stack >> 31) {
3291 WARN_INSN(insn, "PUSHF stack exhausted");
3292 return 1;
3293 }
3294 state->uaccess_stack <<= 1;
3295 state->uaccess_stack |= state->uaccess;
3296 }
3297
3298 if (op->src.type == OP_SRC_POPF) {
3299 if (state->uaccess_stack) {
3300 state->uaccess = state->uaccess_stack & 1;
3301 state->uaccess_stack >>= 1;
3302 if (state->uaccess_stack == 1)
3303 state->uaccess_stack = 0;
3304 }
3305 }
3306 }
3307
3308 return 0;
3309}
3310
3311static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3312{
3313 struct cfi_state *cfi1 = insn->cfi;
3314 int i;
3315
3316 if (!cfi1) {
3317 WARN("CFI missing");
3318 return false;
3319 }
3320
3321 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3322
3323 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3324 cfi1->cfa.base, cfi1->cfa.offset,
3325 cfi2->cfa.base, cfi2->cfa.offset);
3326
3327 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3328 for (i = 0; i < CFI_NUM_REGS; i++) {
3329 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3330 sizeof(struct cfi_reg)))
3331 continue;
3332
3333 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3334 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3335 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3336 break;
3337 }
3338
3339 } else if (cfi1->type != cfi2->type) {
3340
3341 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3342 cfi1->type, cfi2->type);
3343
3344 } else if (cfi1->drap != cfi2->drap ||
3345 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3346 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3347
3348 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3349 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3350 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3351
3352 } else
3353 return true;
3354
3355 return false;
3356}
3357
3358static inline bool func_uaccess_safe(struct symbol *func)
3359{
3360 if (func)
3361 return func->uaccess_safe;
3362
3363 return false;
3364}
3365
3366static inline const char *call_dest_name(struct instruction *insn)
3367{
3368 static char pvname[19];
3369 struct reloc *reloc;
3370 int idx;
3371
3372 if (insn_call_dest(insn))
3373 return insn_call_dest(insn)->name;
3374
3375 reloc = insn_reloc(NULL, insn);
3376 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3377 idx = (reloc_addend(reloc) / sizeof(void *));
3378 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3379 return pvname;
3380 }
3381
3382 return "{dynamic}";
3383}
3384
3385static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3386{
3387 struct symbol *target;
3388 struct reloc *reloc;
3389 int idx;
3390
3391 reloc = insn_reloc(file, insn);
3392 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3393 return false;
3394
3395 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3396
3397 if (file->pv_ops[idx].clean)
3398 return true;
3399
3400 file->pv_ops[idx].clean = true;
3401
3402 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3403 if (!target->sec->noinstr) {
3404 WARN("pv_ops[%d]: %s", idx, target->name);
3405 file->pv_ops[idx].clean = false;
3406 }
3407 }
3408
3409 return file->pv_ops[idx].clean;
3410}
3411
3412static inline bool noinstr_call_dest(struct objtool_file *file,
3413 struct instruction *insn,
3414 struct symbol *func)
3415{
3416 /*
3417 * We can't deal with indirect function calls at present;
3418 * assume they're instrumented.
3419 */
3420 if (!func) {
3421 if (file->pv_ops)
3422 return pv_call_dest(file, insn);
3423
3424 return false;
3425 }
3426
3427 /*
3428 * If the symbol is from a noinstr section; we good.
3429 */
3430 if (func->sec->noinstr)
3431 return true;
3432
3433 /*
3434 * If the symbol is a static_call trampoline, we can't tell.
3435 */
3436 if (func->static_call_tramp)
3437 return true;
3438
3439 /*
3440 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3441 * something 'BAD' happened. At the risk of taking the machine down,
3442 * let them proceed to get the message out.
3443 */
3444 if (!strncmp(func->name, "__ubsan_handle_", 15))
3445 return true;
3446
3447 return false;
3448}
3449
3450static int validate_call(struct objtool_file *file,
3451 struct instruction *insn,
3452 struct insn_state *state)
3453{
3454 if (state->noinstr && state->instr <= 0 &&
3455 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3456 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3457 return 1;
3458 }
3459
3460 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3461 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3462 return 1;
3463 }
3464
3465 if (state->df) {
3466 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3467 return 1;
3468 }
3469
3470 return 0;
3471}
3472
3473static int validate_sibling_call(struct objtool_file *file,
3474 struct instruction *insn,
3475 struct insn_state *state)
3476{
3477 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3478 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3479 return 1;
3480 }
3481
3482 return validate_call(file, insn, state);
3483}
3484
3485static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3486{
3487 if (state->noinstr && state->instr > 0) {
3488 WARN_INSN(insn, "return with instrumentation enabled");
3489 return 1;
3490 }
3491
3492 if (state->uaccess && !func_uaccess_safe(func)) {
3493 WARN_INSN(insn, "return with UACCESS enabled");
3494 return 1;
3495 }
3496
3497 if (!state->uaccess && func_uaccess_safe(func)) {
3498 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3499 return 1;
3500 }
3501
3502 if (state->df) {
3503 WARN_INSN(insn, "return with DF set");
3504 return 1;
3505 }
3506
3507 if (func && has_modified_stack_frame(insn, state)) {
3508 WARN_INSN(insn, "return with modified stack frame");
3509 return 1;
3510 }
3511
3512 if (state->cfi.bp_scratch) {
3513 WARN_INSN(insn, "BP used as a scratch register");
3514 return 1;
3515 }
3516
3517 return 0;
3518}
3519
3520static struct instruction *next_insn_to_validate(struct objtool_file *file,
3521 struct instruction *insn)
3522{
3523 struct alt_group *alt_group = insn->alt_group;
3524
3525 /*
3526 * Simulate the fact that alternatives are patched in-place. When the
3527 * end of a replacement alt_group is reached, redirect objtool flow to
3528 * the end of the original alt_group.
3529 *
3530 * insn->alts->insn -> alt_group->first_insn
3531 * ...
3532 * alt_group->last_insn
3533 * [alt_group->nop] -> next(orig_group->last_insn)
3534 */
3535 if (alt_group) {
3536 if (alt_group->nop) {
3537 /* ->nop implies ->orig_group */
3538 if (insn == alt_group->last_insn)
3539 return alt_group->nop;
3540 if (insn == alt_group->nop)
3541 goto next_orig;
3542 }
3543 if (insn == alt_group->last_insn && alt_group->orig_group)
3544 goto next_orig;
3545 }
3546
3547 return next_insn_same_sec(file, insn);
3548
3549next_orig:
3550 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3551}
3552
3553/*
3554 * Follow the branch starting at the given instruction, and recursively follow
3555 * any other branches (jumps). Meanwhile, track the frame pointer state at
3556 * each instruction and validate all the rules described in
3557 * tools/objtool/Documentation/objtool.txt.
3558 */
3559static int validate_branch(struct objtool_file *file, struct symbol *func,
3560 struct instruction *insn, struct insn_state state)
3561{
3562 struct alternative *alt;
3563 struct instruction *next_insn, *prev_insn = NULL;
3564 struct section *sec;
3565 u8 visited;
3566 int ret;
3567
3568 sec = insn->sec;
3569
3570 while (1) {
3571 next_insn = next_insn_to_validate(file, insn);
3572
3573 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3574 /* Ignore KCFI type preambles, which always fall through */
3575 if (!strncmp(func->name, "__cfi_", 6) ||
3576 !strncmp(func->name, "__pfx_", 6))
3577 return 0;
3578
3579 WARN("%s() falls through to next function %s()",
3580 func->name, insn_func(insn)->name);
3581 return 1;
3582 }
3583
3584 if (func && insn->ignore) {
3585 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3586 return 1;
3587 }
3588
3589 visited = VISITED_BRANCH << state.uaccess;
3590 if (insn->visited & VISITED_BRANCH_MASK) {
3591 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3592 return 1;
3593
3594 if (insn->visited & visited)
3595 return 0;
3596 } else {
3597 nr_insns_visited++;
3598 }
3599
3600 if (state.noinstr)
3601 state.instr += insn->instr;
3602
3603 if (insn->hint) {
3604 if (insn->restore) {
3605 struct instruction *save_insn, *i;
3606
3607 i = insn;
3608 save_insn = NULL;
3609
3610 sym_for_each_insn_continue_reverse(file, func, i) {
3611 if (i->save) {
3612 save_insn = i;
3613 break;
3614 }
3615 }
3616
3617 if (!save_insn) {
3618 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3619 return 1;
3620 }
3621
3622 if (!save_insn->visited) {
3623 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3624 return 1;
3625 }
3626
3627 insn->cfi = save_insn->cfi;
3628 nr_cfi_reused++;
3629 }
3630
3631 state.cfi = *insn->cfi;
3632 } else {
3633 /* XXX track if we actually changed state.cfi */
3634
3635 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3636 insn->cfi = prev_insn->cfi;
3637 nr_cfi_reused++;
3638 } else {
3639 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3640 }
3641 }
3642
3643 insn->visited |= visited;
3644
3645 if (propagate_alt_cfi(file, insn))
3646 return 1;
3647
3648 if (!insn->ignore_alts && insn->alts) {
3649 bool skip_orig = false;
3650
3651 for (alt = insn->alts; alt; alt = alt->next) {
3652 if (alt->skip_orig)
3653 skip_orig = true;
3654
3655 ret = validate_branch(file, func, alt->insn, state);
3656 if (ret) {
3657 BT_INSN(insn, "(alt)");
3658 return ret;
3659 }
3660 }
3661
3662 if (skip_orig)
3663 return 0;
3664 }
3665
3666 if (handle_insn_ops(insn, next_insn, &state))
3667 return 1;
3668
3669 switch (insn->type) {
3670
3671 case INSN_RETURN:
3672 return validate_return(func, insn, &state);
3673
3674 case INSN_CALL:
3675 case INSN_CALL_DYNAMIC:
3676 ret = validate_call(file, insn, &state);
3677 if (ret)
3678 return ret;
3679
3680 if (opts.stackval && func && !is_special_call(insn) &&
3681 !has_valid_stack_frame(&state)) {
3682 WARN_INSN(insn, "call without frame pointer save/setup");
3683 return 1;
3684 }
3685
3686 if (insn->dead_end)
3687 return 0;
3688
3689 break;
3690
3691 case INSN_JUMP_CONDITIONAL:
3692 case INSN_JUMP_UNCONDITIONAL:
3693 if (is_sibling_call(insn)) {
3694 ret = validate_sibling_call(file, insn, &state);
3695 if (ret)
3696 return ret;
3697
3698 } else if (insn->jump_dest) {
3699 ret = validate_branch(file, func,
3700 insn->jump_dest, state);
3701 if (ret) {
3702 BT_INSN(insn, "(branch)");
3703 return ret;
3704 }
3705 }
3706
3707 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3708 return 0;
3709
3710 break;
3711
3712 case INSN_JUMP_DYNAMIC:
3713 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3714 if (is_sibling_call(insn)) {
3715 ret = validate_sibling_call(file, insn, &state);
3716 if (ret)
3717 return ret;
3718 }
3719
3720 if (insn->type == INSN_JUMP_DYNAMIC)
3721 return 0;
3722
3723 break;
3724
3725 case INSN_CONTEXT_SWITCH:
3726 if (func && (!next_insn || !next_insn->hint)) {
3727 WARN_INSN(insn, "unsupported instruction in callable function");
3728 return 1;
3729 }
3730 return 0;
3731
3732 case INSN_STAC:
3733 if (state.uaccess) {
3734 WARN_INSN(insn, "recursive UACCESS enable");
3735 return 1;
3736 }
3737
3738 state.uaccess = true;
3739 break;
3740
3741 case INSN_CLAC:
3742 if (!state.uaccess && func) {
3743 WARN_INSN(insn, "redundant UACCESS disable");
3744 return 1;
3745 }
3746
3747 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3748 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3749 return 1;
3750 }
3751
3752 state.uaccess = false;
3753 break;
3754
3755 case INSN_STD:
3756 if (state.df) {
3757 WARN_INSN(insn, "recursive STD");
3758 return 1;
3759 }
3760
3761 state.df = true;
3762 break;
3763
3764 case INSN_CLD:
3765 if (!state.df && func) {
3766 WARN_INSN(insn, "redundant CLD");
3767 return 1;
3768 }
3769
3770 state.df = false;
3771 break;
3772
3773 default:
3774 break;
3775 }
3776
3777 if (insn->dead_end)
3778 return 0;
3779
3780 if (!next_insn) {
3781 if (state.cfi.cfa.base == CFI_UNDEFINED)
3782 return 0;
3783 WARN("%s: unexpected end of section", sec->name);
3784 return 1;
3785 }
3786
3787 prev_insn = insn;
3788 insn = next_insn;
3789 }
3790
3791 return 0;
3792}
3793
3794static int validate_unwind_hint(struct objtool_file *file,
3795 struct instruction *insn,
3796 struct insn_state *state)
3797{
3798 if (insn->hint && !insn->visited && !insn->ignore) {
3799 int ret = validate_branch(file, insn_func(insn), insn, *state);
3800 if (ret)
3801 BT_INSN(insn, "<=== (hint)");
3802 return ret;
3803 }
3804
3805 return 0;
3806}
3807
3808static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3809{
3810 struct instruction *insn;
3811 struct insn_state state;
3812 int warnings = 0;
3813
3814 if (!file->hints)
3815 return 0;
3816
3817 init_insn_state(file, &state, sec);
3818
3819 if (sec) {
3820 sec_for_each_insn(file, sec, insn)
3821 warnings += validate_unwind_hint(file, insn, &state);
3822 } else {
3823 for_each_insn(file, insn)
3824 warnings += validate_unwind_hint(file, insn, &state);
3825 }
3826
3827 return warnings;
3828}
3829
3830/*
3831 * Validate rethunk entry constraint: must untrain RET before the first RET.
3832 *
3833 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3834 * before an actual RET instruction.
3835 */
3836static int validate_unret(struct objtool_file *file, struct instruction *insn)
3837{
3838 struct instruction *next, *dest;
3839 int ret;
3840
3841 for (;;) {
3842 next = next_insn_to_validate(file, insn);
3843
3844 if (insn->visited & VISITED_UNRET)
3845 return 0;
3846
3847 insn->visited |= VISITED_UNRET;
3848
3849 if (!insn->ignore_alts && insn->alts) {
3850 struct alternative *alt;
3851 bool skip_orig = false;
3852
3853 for (alt = insn->alts; alt; alt = alt->next) {
3854 if (alt->skip_orig)
3855 skip_orig = true;
3856
3857 ret = validate_unret(file, alt->insn);
3858 if (ret) {
3859 BT_INSN(insn, "(alt)");
3860 return ret;
3861 }
3862 }
3863
3864 if (skip_orig)
3865 return 0;
3866 }
3867
3868 switch (insn->type) {
3869
3870 case INSN_CALL_DYNAMIC:
3871 case INSN_JUMP_DYNAMIC:
3872 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3873 WARN_INSN(insn, "early indirect call");
3874 return 1;
3875
3876 case INSN_JUMP_UNCONDITIONAL:
3877 case INSN_JUMP_CONDITIONAL:
3878 if (!is_sibling_call(insn)) {
3879 if (!insn->jump_dest) {
3880 WARN_INSN(insn, "unresolved jump target after linking?!?");
3881 return -1;
3882 }
3883 ret = validate_unret(file, insn->jump_dest);
3884 if (ret) {
3885 BT_INSN(insn, "(branch%s)",
3886 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3887 return ret;
3888 }
3889
3890 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3891 return 0;
3892
3893 break;
3894 }
3895
3896 /* fallthrough */
3897 case INSN_CALL:
3898 dest = find_insn(file, insn_call_dest(insn)->sec,
3899 insn_call_dest(insn)->offset);
3900 if (!dest) {
3901 WARN("Unresolved function after linking!?: %s",
3902 insn_call_dest(insn)->name);
3903 return -1;
3904 }
3905
3906 ret = validate_unret(file, dest);
3907 if (ret) {
3908 BT_INSN(insn, "(call)");
3909 return ret;
3910 }
3911 /*
3912 * If a call returns without error, it must have seen UNTRAIN_RET.
3913 * Therefore any non-error return is a success.
3914 */
3915 return 0;
3916
3917 case INSN_RETURN:
3918 WARN_INSN(insn, "RET before UNTRAIN");
3919 return 1;
3920
3921 case INSN_NOP:
3922 if (insn->retpoline_safe)
3923 return 0;
3924 break;
3925
3926 default:
3927 break;
3928 }
3929
3930 if (!next) {
3931 WARN_INSN(insn, "teh end!");
3932 return -1;
3933 }
3934 insn = next;
3935 }
3936
3937 return 0;
3938}
3939
3940/*
3941 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3942 * VALIDATE_UNRET_END before RET.
3943 */
3944static int validate_unrets(struct objtool_file *file)
3945{
3946 struct instruction *insn;
3947 int ret, warnings = 0;
3948
3949 for_each_insn(file, insn) {
3950 if (!insn->unret)
3951 continue;
3952
3953 ret = validate_unret(file, insn);
3954 if (ret < 0) {
3955 WARN_INSN(insn, "Failed UNRET validation");
3956 return ret;
3957 }
3958 warnings += ret;
3959 }
3960
3961 return warnings;
3962}
3963
3964static int validate_retpoline(struct objtool_file *file)
3965{
3966 struct instruction *insn;
3967 int warnings = 0;
3968
3969 for_each_insn(file, insn) {
3970 if (insn->type != INSN_JUMP_DYNAMIC &&
3971 insn->type != INSN_CALL_DYNAMIC &&
3972 insn->type != INSN_RETURN)
3973 continue;
3974
3975 if (insn->retpoline_safe)
3976 continue;
3977
3978 if (insn->sec->init)
3979 continue;
3980
3981 if (insn->type == INSN_RETURN) {
3982 if (opts.rethunk) {
3983 WARN_INSN(insn, "'naked' return found in RETHUNK build");
3984 } else
3985 continue;
3986 } else {
3987 WARN_INSN(insn, "indirect %s found in RETPOLINE build",
3988 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3989 }
3990
3991 warnings++;
3992 }
3993
3994 return warnings;
3995}
3996
3997static bool is_kasan_insn(struct instruction *insn)
3998{
3999 return (insn->type == INSN_CALL &&
4000 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4001}
4002
4003static bool is_ubsan_insn(struct instruction *insn)
4004{
4005 return (insn->type == INSN_CALL &&
4006 !strcmp(insn_call_dest(insn)->name,
4007 "__ubsan_handle_builtin_unreachable"));
4008}
4009
4010static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4011{
4012 int i;
4013 struct instruction *prev_insn;
4014
4015 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4016 return true;
4017
4018 /*
4019 * Ignore alternative replacement instructions. This can happen
4020 * when a whitelisted function uses one of the ALTERNATIVE macros.
4021 */
4022 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4023 !strcmp(insn->sec->name, ".altinstr_aux"))
4024 return true;
4025
4026 /*
4027 * Whole archive runs might encounter dead code from weak symbols.
4028 * This is where the linker will have dropped the weak symbol in
4029 * favour of a regular symbol, but leaves the code in place.
4030 *
4031 * In this case we'll find a piece of code (whole function) that is not
4032 * covered by a !section symbol. Ignore them.
4033 */
4034 if (opts.link && !insn_func(insn)) {
4035 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4036 unsigned long end = insn->offset + size;
4037
4038 if (!size) /* not a hole */
4039 return false;
4040
4041 if (size < 0) /* hole until the end */
4042 return true;
4043
4044 sec_for_each_insn_continue(file, insn) {
4045 /*
4046 * If we reach a visited instruction at or before the
4047 * end of the hole, ignore the unreachable.
4048 */
4049 if (insn->visited)
4050 return true;
4051
4052 if (insn->offset >= end)
4053 break;
4054
4055 /*
4056 * If this hole jumps to a .cold function, mark it ignore too.
4057 */
4058 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4059 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4060 struct instruction *dest = insn->jump_dest;
4061 func_for_each_insn(file, insn_func(dest), dest)
4062 dest->ignore = true;
4063 }
4064 }
4065
4066 return false;
4067 }
4068
4069 if (!insn_func(insn))
4070 return false;
4071
4072 if (insn_func(insn)->static_call_tramp)
4073 return true;
4074
4075 /*
4076 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4077 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4078 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4079 * (or occasionally a JMP to UD2).
4080 *
4081 * It may also insert a UD2 after calling a __noreturn function.
4082 */
4083 prev_insn = prev_insn_same_sec(file, insn);
4084 if (prev_insn->dead_end &&
4085 (insn->type == INSN_BUG ||
4086 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4087 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4088 return true;
4089
4090 /*
4091 * Check if this (or a subsequent) instruction is related to
4092 * CONFIG_UBSAN or CONFIG_KASAN.
4093 *
4094 * End the search at 5 instructions to avoid going into the weeds.
4095 */
4096 for (i = 0; i < 5; i++) {
4097
4098 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4099 return true;
4100
4101 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4102 if (insn->jump_dest &&
4103 insn_func(insn->jump_dest) == insn_func(insn)) {
4104 insn = insn->jump_dest;
4105 continue;
4106 }
4107
4108 break;
4109 }
4110
4111 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4112 break;
4113
4114 insn = next_insn_same_sec(file, insn);
4115 }
4116
4117 return false;
4118}
4119
4120static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4121{
4122 struct instruction *insn, *prev;
4123 struct cfi_state *cfi;
4124
4125 insn = find_insn(file, func->sec, func->offset);
4126 if (!insn)
4127 return -1;
4128
4129 for (prev = prev_insn_same_sec(file, insn);
4130 prev;
4131 prev = prev_insn_same_sec(file, prev)) {
4132 u64 offset;
4133
4134 if (prev->type != INSN_NOP)
4135 return -1;
4136
4137 offset = func->offset - prev->offset;
4138
4139 if (offset > opts.prefix)
4140 return -1;
4141
4142 if (offset < opts.prefix)
4143 continue;
4144
4145 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4146 break;
4147 }
4148
4149 if (!prev)
4150 return -1;
4151
4152 if (!insn->cfi) {
4153 /*
4154 * This can happen if stack validation isn't enabled or the
4155 * function is annotated with STACK_FRAME_NON_STANDARD.
4156 */
4157 return 0;
4158 }
4159
4160 /* Propagate insn->cfi to the prefix code */
4161 cfi = cfi_hash_find_or_add(insn->cfi);
4162 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4163 prev->cfi = cfi;
4164
4165 return 0;
4166}
4167
4168static int add_prefix_symbols(struct objtool_file *file)
4169{
4170 struct section *sec;
4171 struct symbol *func;
4172
4173 for_each_sec(file, sec) {
4174 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4175 continue;
4176
4177 sec_for_each_sym(sec, func) {
4178 if (func->type != STT_FUNC)
4179 continue;
4180
4181 add_prefix_symbol(file, func);
4182 }
4183 }
4184
4185 return 0;
4186}
4187
4188static int validate_symbol(struct objtool_file *file, struct section *sec,
4189 struct symbol *sym, struct insn_state *state)
4190{
4191 struct instruction *insn;
4192 int ret;
4193
4194 if (!sym->len) {
4195 WARN("%s() is missing an ELF size annotation", sym->name);
4196 return 1;
4197 }
4198
4199 if (sym->pfunc != sym || sym->alias != sym)
4200 return 0;
4201
4202 insn = find_insn(file, sec, sym->offset);
4203 if (!insn || insn->ignore || insn->visited)
4204 return 0;
4205
4206 state->uaccess = sym->uaccess_safe;
4207
4208 ret = validate_branch(file, insn_func(insn), insn, *state);
4209 if (ret)
4210 BT_INSN(insn, "<=== (sym)");
4211 return ret;
4212}
4213
4214static int validate_section(struct objtool_file *file, struct section *sec)
4215{
4216 struct insn_state state;
4217 struct symbol *func;
4218 int warnings = 0;
4219
4220 sec_for_each_sym(sec, func) {
4221 if (func->type != STT_FUNC)
4222 continue;
4223
4224 init_insn_state(file, &state, sec);
4225 set_func_state(&state.cfi);
4226
4227 warnings += validate_symbol(file, sec, func, &state);
4228 }
4229
4230 return warnings;
4231}
4232
4233static int validate_noinstr_sections(struct objtool_file *file)
4234{
4235 struct section *sec;
4236 int warnings = 0;
4237
4238 sec = find_section_by_name(file->elf, ".noinstr.text");
4239 if (sec) {
4240 warnings += validate_section(file, sec);
4241 warnings += validate_unwind_hints(file, sec);
4242 }
4243
4244 sec = find_section_by_name(file->elf, ".entry.text");
4245 if (sec) {
4246 warnings += validate_section(file, sec);
4247 warnings += validate_unwind_hints(file, sec);
4248 }
4249
4250 sec = find_section_by_name(file->elf, ".cpuidle.text");
4251 if (sec) {
4252 warnings += validate_section(file, sec);
4253 warnings += validate_unwind_hints(file, sec);
4254 }
4255
4256 return warnings;
4257}
4258
4259static int validate_functions(struct objtool_file *file)
4260{
4261 struct section *sec;
4262 int warnings = 0;
4263
4264 for_each_sec(file, sec) {
4265 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4266 continue;
4267
4268 warnings += validate_section(file, sec);
4269 }
4270
4271 return warnings;
4272}
4273
4274static void mark_endbr_used(struct instruction *insn)
4275{
4276 if (!list_empty(&insn->call_node))
4277 list_del_init(&insn->call_node);
4278}
4279
4280static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4281{
4282 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4283 struct instruction *first;
4284
4285 if (!sym)
4286 return false;
4287
4288 first = find_insn(file, sym->sec, sym->offset);
4289 if (!first)
4290 return false;
4291
4292 if (first->type != INSN_ENDBR && !first->noendbr)
4293 return false;
4294
4295 return insn->offset == sym->offset + sym->len;
4296}
4297
4298static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4299{
4300 struct instruction *dest;
4301 struct reloc *reloc;
4302 unsigned long off;
4303 int warnings = 0;
4304
4305 /*
4306 * Looking for function pointer load relocations. Ignore
4307 * direct/indirect branches:
4308 */
4309 switch (insn->type) {
4310 case INSN_CALL:
4311 case INSN_CALL_DYNAMIC:
4312 case INSN_JUMP_CONDITIONAL:
4313 case INSN_JUMP_UNCONDITIONAL:
4314 case INSN_JUMP_DYNAMIC:
4315 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4316 case INSN_RETURN:
4317 case INSN_NOP:
4318 return 0;
4319 default:
4320 break;
4321 }
4322
4323 for (reloc = insn_reloc(file, insn);
4324 reloc;
4325 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4326 reloc_offset(reloc) + 1,
4327 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4328
4329 /*
4330 * static_call_update() references the trampoline, which
4331 * doesn't have (or need) ENDBR. Skip warning in that case.
4332 */
4333 if (reloc->sym->static_call_tramp)
4334 continue;
4335
4336 off = reloc->sym->offset;
4337 if (reloc_type(reloc) == R_X86_64_PC32 ||
4338 reloc_type(reloc) == R_X86_64_PLT32)
4339 off += arch_dest_reloc_offset(reloc_addend(reloc));
4340 else
4341 off += reloc_addend(reloc);
4342
4343 dest = find_insn(file, reloc->sym->sec, off);
4344 if (!dest)
4345 continue;
4346
4347 if (dest->type == INSN_ENDBR) {
4348 mark_endbr_used(dest);
4349 continue;
4350 }
4351
4352 if (insn_func(dest) && insn_func(insn) &&
4353 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4354 /*
4355 * Anything from->to self is either _THIS_IP_ or
4356 * IRET-to-self.
4357 *
4358 * There is no sane way to annotate _THIS_IP_ since the
4359 * compiler treats the relocation as a constant and is
4360 * happy to fold in offsets, skewing any annotation we
4361 * do, leading to vast amounts of false-positives.
4362 *
4363 * There's also compiler generated _THIS_IP_ through
4364 * KCOV and such which we have no hope of annotating.
4365 *
4366 * As such, blanket accept self-references without
4367 * issue.
4368 */
4369 continue;
4370 }
4371
4372 /*
4373 * Accept anything ANNOTATE_NOENDBR.
4374 */
4375 if (dest->noendbr)
4376 continue;
4377
4378 /*
4379 * Accept if this is the instruction after a symbol
4380 * that is (no)endbr -- typical code-range usage.
4381 */
4382 if (noendbr_range(file, dest))
4383 continue;
4384
4385 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4386
4387 warnings++;
4388 }
4389
4390 return warnings;
4391}
4392
4393static int validate_ibt_data_reloc(struct objtool_file *file,
4394 struct reloc *reloc)
4395{
4396 struct instruction *dest;
4397
4398 dest = find_insn(file, reloc->sym->sec,
4399 reloc->sym->offset + reloc_addend(reloc));
4400 if (!dest)
4401 return 0;
4402
4403 if (dest->type == INSN_ENDBR) {
4404 mark_endbr_used(dest);
4405 return 0;
4406 }
4407
4408 if (dest->noendbr)
4409 return 0;
4410
4411 WARN_FUNC("data relocation to !ENDBR: %s",
4412 reloc->sec->base, reloc_offset(reloc),
4413 offstr(dest->sec, dest->offset));
4414
4415 return 1;
4416}
4417
4418/*
4419 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4420 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4421 * NOPs) later, in create_ibt_endbr_seal_sections().
4422 */
4423static int validate_ibt(struct objtool_file *file)
4424{
4425 struct section *sec;
4426 struct reloc *reloc;
4427 struct instruction *insn;
4428 int warnings = 0;
4429
4430 for_each_insn(file, insn)
4431 warnings += validate_ibt_insn(file, insn);
4432
4433 for_each_sec(file, sec) {
4434
4435 /* Already done by validate_ibt_insn() */
4436 if (sec->sh.sh_flags & SHF_EXECINSTR)
4437 continue;
4438
4439 if (!sec->rsec)
4440 continue;
4441
4442 /*
4443 * These sections can reference text addresses, but not with
4444 * the intent to indirect branch to them.
4445 */
4446 if ((!strncmp(sec->name, ".discard", 8) &&
4447 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4448 !strncmp(sec->name, ".debug", 6) ||
4449 !strcmp(sec->name, ".altinstructions") ||
4450 !strcmp(sec->name, ".ibt_endbr_seal") ||
4451 !strcmp(sec->name, ".orc_unwind_ip") ||
4452 !strcmp(sec->name, ".parainstructions") ||
4453 !strcmp(sec->name, ".retpoline_sites") ||
4454 !strcmp(sec->name, ".smp_locks") ||
4455 !strcmp(sec->name, ".static_call_sites") ||
4456 !strcmp(sec->name, "_error_injection_whitelist") ||
4457 !strcmp(sec->name, "_kprobe_blacklist") ||
4458 !strcmp(sec->name, "__bug_table") ||
4459 !strcmp(sec->name, "__ex_table") ||
4460 !strcmp(sec->name, "__jump_table") ||
4461 !strcmp(sec->name, "__mcount_loc") ||
4462 !strcmp(sec->name, ".kcfi_traps") ||
4463 strstr(sec->name, "__patchable_function_entries"))
4464 continue;
4465
4466 for_each_reloc(sec->rsec, reloc)
4467 warnings += validate_ibt_data_reloc(file, reloc);
4468 }
4469
4470 return warnings;
4471}
4472
4473static int validate_sls(struct objtool_file *file)
4474{
4475 struct instruction *insn, *next_insn;
4476 int warnings = 0;
4477
4478 for_each_insn(file, insn) {
4479 next_insn = next_insn_same_sec(file, insn);
4480
4481 if (insn->retpoline_safe)
4482 continue;
4483
4484 switch (insn->type) {
4485 case INSN_RETURN:
4486 if (!next_insn || next_insn->type != INSN_TRAP) {
4487 WARN_INSN(insn, "missing int3 after ret");
4488 warnings++;
4489 }
4490
4491 break;
4492 case INSN_JUMP_DYNAMIC:
4493 if (!next_insn || next_insn->type != INSN_TRAP) {
4494 WARN_INSN(insn, "missing int3 after indirect jump");
4495 warnings++;
4496 }
4497 break;
4498 default:
4499 break;
4500 }
4501 }
4502
4503 return warnings;
4504}
4505
4506static bool ignore_noreturn_call(struct instruction *insn)
4507{
4508 struct symbol *call_dest = insn_call_dest(insn);
4509
4510 /*
4511 * FIXME: hack, we need a real noreturn solution
4512 *
4513 * Problem is, exc_double_fault() may or may not return, depending on
4514 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4515 * to the kernel config.
4516 *
4517 * Other potential ways to fix it:
4518 *
4519 * - have compiler communicate __noreturn functions somehow
4520 * - remove CONFIG_X86_ESPFIX64
4521 * - read the .config file
4522 * - add a cmdline option
4523 * - create a generic objtool annotation format (vs a bunch of custom
4524 * formats) and annotate it
4525 */
4526 if (!strcmp(call_dest->name, "exc_double_fault")) {
4527 /* prevent further unreachable warnings for the caller */
4528 insn->sym->warned = 1;
4529 return true;
4530 }
4531
4532 return false;
4533}
4534
4535static int validate_reachable_instructions(struct objtool_file *file)
4536{
4537 struct instruction *insn, *prev_insn;
4538 struct symbol *call_dest;
4539 int warnings = 0;
4540
4541 if (file->ignore_unreachables)
4542 return 0;
4543
4544 for_each_insn(file, insn) {
4545 if (insn->visited || ignore_unreachable_insn(file, insn))
4546 continue;
4547
4548 prev_insn = prev_insn_same_sec(file, insn);
4549 if (prev_insn && prev_insn->dead_end) {
4550 call_dest = insn_call_dest(prev_insn);
4551 if (call_dest && !ignore_noreturn_call(prev_insn)) {
4552 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4553 call_dest->name);
4554 warnings++;
4555 continue;
4556 }
4557 }
4558
4559 WARN_INSN(insn, "unreachable instruction");
4560 warnings++;
4561 }
4562
4563 return warnings;
4564}
4565
4566/* 'funcs' is a space-separated list of function names */
4567static int disas_funcs(const char *funcs)
4568{
4569 const char *objdump_str, *cross_compile;
4570 int size, ret;
4571 char *cmd;
4572
4573 cross_compile = getenv("CROSS_COMPILE");
4574
4575 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4576 "BEGIN { split(_funcs, funcs); }"
4577 "/^$/ { func_match = 0; }"
4578 "/<.*>:/ { "
4579 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4580 "for (i in funcs) {"
4581 "if (funcs[i] == f) {"
4582 "func_match = 1;"
4583 "base = strtonum(\"0x\" $1);"
4584 "break;"
4585 "}"
4586 "}"
4587 "}"
4588 "{"
4589 "if (func_match) {"
4590 "addr = strtonum(\"0x\" $1);"
4591 "printf(\"%%04x \", addr - base);"
4592 "print;"
4593 "}"
4594 "}' 1>&2";
4595
4596 /* fake snprintf() to calculate the size */
4597 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4598 if (size <= 0) {
4599 WARN("objdump string size calculation failed");
4600 return -1;
4601 }
4602
4603 cmd = malloc(size);
4604
4605 /* real snprintf() */
4606 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4607 ret = system(cmd);
4608 if (ret) {
4609 WARN("disassembly failed: %d", ret);
4610 return -1;
4611 }
4612
4613 return 0;
4614}
4615
4616static int disas_warned_funcs(struct objtool_file *file)
4617{
4618 struct symbol *sym;
4619 char *funcs = NULL, *tmp;
4620
4621 for_each_sym(file, sym) {
4622 if (sym->warned) {
4623 if (!funcs) {
4624 funcs = malloc(strlen(sym->name) + 1);
4625 strcpy(funcs, sym->name);
4626 } else {
4627 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4628 sprintf(tmp, "%s %s", funcs, sym->name);
4629 free(funcs);
4630 funcs = tmp;
4631 }
4632 }
4633 }
4634
4635 if (funcs)
4636 disas_funcs(funcs);
4637
4638 return 0;
4639}
4640
4641struct insn_chunk {
4642 void *addr;
4643 struct insn_chunk *next;
4644};
4645
4646/*
4647 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4648 * which can trigger more allocations for .debug_* sections whose data hasn't
4649 * been read yet.
4650 */
4651static void free_insns(struct objtool_file *file)
4652{
4653 struct instruction *insn;
4654 struct insn_chunk *chunks = NULL, *chunk;
4655
4656 for_each_insn(file, insn) {
4657 if (!insn->idx) {
4658 chunk = malloc(sizeof(*chunk));
4659 chunk->addr = insn;
4660 chunk->next = chunks;
4661 chunks = chunk;
4662 }
4663 }
4664
4665 for (chunk = chunks; chunk; chunk = chunk->next)
4666 free(chunk->addr);
4667}
4668
4669int check(struct objtool_file *file)
4670{
4671 int ret, warnings = 0;
4672
4673 arch_initial_func_cfi_state(&initial_func_cfi);
4674 init_cfi_state(&init_cfi);
4675 init_cfi_state(&func_cfi);
4676 set_func_state(&func_cfi);
4677 init_cfi_state(&force_undefined_cfi);
4678 force_undefined_cfi.force_undefined = true;
4679
4680 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4681 goto out;
4682
4683 cfi_hash_add(&init_cfi);
4684 cfi_hash_add(&func_cfi);
4685
4686 ret = decode_sections(file);
4687 if (ret < 0)
4688 goto out;
4689
4690 warnings += ret;
4691
4692 if (!nr_insns)
4693 goto out;
4694
4695 if (opts.retpoline) {
4696 ret = validate_retpoline(file);
4697 if (ret < 0)
4698 return ret;
4699 warnings += ret;
4700 }
4701
4702 if (opts.stackval || opts.orc || opts.uaccess) {
4703 ret = validate_functions(file);
4704 if (ret < 0)
4705 goto out;
4706 warnings += ret;
4707
4708 ret = validate_unwind_hints(file, NULL);
4709 if (ret < 0)
4710 goto out;
4711 warnings += ret;
4712
4713 if (!warnings) {
4714 ret = validate_reachable_instructions(file);
4715 if (ret < 0)
4716 goto out;
4717 warnings += ret;
4718 }
4719
4720 } else if (opts.noinstr) {
4721 ret = validate_noinstr_sections(file);
4722 if (ret < 0)
4723 goto out;
4724 warnings += ret;
4725 }
4726
4727 if (opts.unret) {
4728 /*
4729 * Must be after validate_branch() and friends, it plays
4730 * further games with insn->visited.
4731 */
4732 ret = validate_unrets(file);
4733 if (ret < 0)
4734 return ret;
4735 warnings += ret;
4736 }
4737
4738 if (opts.ibt) {
4739 ret = validate_ibt(file);
4740 if (ret < 0)
4741 goto out;
4742 warnings += ret;
4743 }
4744
4745 if (opts.sls) {
4746 ret = validate_sls(file);
4747 if (ret < 0)
4748 goto out;
4749 warnings += ret;
4750 }
4751
4752 if (opts.static_call) {
4753 ret = create_static_call_sections(file);
4754 if (ret < 0)
4755 goto out;
4756 warnings += ret;
4757 }
4758
4759 if (opts.retpoline) {
4760 ret = create_retpoline_sites_sections(file);
4761 if (ret < 0)
4762 goto out;
4763 warnings += ret;
4764 }
4765
4766 if (opts.cfi) {
4767 ret = create_cfi_sections(file);
4768 if (ret < 0)
4769 goto out;
4770 warnings += ret;
4771 }
4772
4773 if (opts.rethunk) {
4774 ret = create_return_sites_sections(file);
4775 if (ret < 0)
4776 goto out;
4777 warnings += ret;
4778
4779 if (opts.hack_skylake) {
4780 ret = create_direct_call_sections(file);
4781 if (ret < 0)
4782 goto out;
4783 warnings += ret;
4784 }
4785 }
4786
4787 if (opts.mcount) {
4788 ret = create_mcount_loc_sections(file);
4789 if (ret < 0)
4790 goto out;
4791 warnings += ret;
4792 }
4793
4794 if (opts.prefix) {
4795 ret = add_prefix_symbols(file);
4796 if (ret < 0)
4797 return ret;
4798 warnings += ret;
4799 }
4800
4801 if (opts.ibt) {
4802 ret = create_ibt_endbr_seal_sections(file);
4803 if (ret < 0)
4804 goto out;
4805 warnings += ret;
4806 }
4807
4808 if (opts.orc && nr_insns) {
4809 ret = orc_create(file);
4810 if (ret < 0)
4811 goto out;
4812 warnings += ret;
4813 }
4814
4815 free_insns(file);
4816
4817 if (opts.verbose)
4818 disas_warned_funcs(file);
4819
4820 if (opts.stats) {
4821 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4822 printf("nr_cfi: %ld\n", nr_cfi);
4823 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4824 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4825 }
4826
4827out:
4828 /*
4829 * For now, don't fail the kernel build on fatal warnings. These
4830 * errors are still fairly common due to the growing matrix of
4831 * supported toolchains and their recent pace of change.
4832 */
4833 return 0;
4834}