Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2#define pr_fmt(fmt) "SMP alternatives: " fmt
3
4#include <linux/module.h>
5#include <linux/sched.h>
6#include <linux/mutex.h>
7#include <linux/list.h>
8#include <linux/stringify.h>
9#include <linux/mm.h>
10#include <linux/vmalloc.h>
11#include <linux/memory.h>
12#include <linux/stop_machine.h>
13#include <linux/slab.h>
14#include <linux/kdebug.h>
15#include <linux/kprobes.h>
16#include <linux/mmu_context.h>
17#include <linux/bsearch.h>
18#include <asm/text-patching.h>
19#include <asm/alternative.h>
20#include <asm/sections.h>
21#include <asm/pgtable.h>
22#include <asm/mce.h>
23#include <asm/nmi.h>
24#include <asm/cacheflush.h>
25#include <asm/tlbflush.h>
26#include <asm/io.h>
27#include <asm/fixmap.h>
28
29int __read_mostly alternatives_patched;
30
31EXPORT_SYMBOL_GPL(alternatives_patched);
32
33#define MAX_PATCH_LEN (255-1)
34
35static int __initdata_or_module debug_alternative;
36
37static int __init debug_alt(char *str)
38{
39 debug_alternative = 1;
40 return 1;
41}
42__setup("debug-alternative", debug_alt);
43
44static int noreplace_smp;
45
46static int __init setup_noreplace_smp(char *str)
47{
48 noreplace_smp = 1;
49 return 1;
50}
51__setup("noreplace-smp", setup_noreplace_smp);
52
53#define DPRINTK(fmt, args...) \
54do { \
55 if (debug_alternative) \
56 printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args); \
57} while (0)
58
59#define DUMP_BYTES(buf, len, fmt, args...) \
60do { \
61 if (unlikely(debug_alternative)) { \
62 int j; \
63 \
64 if (!(len)) \
65 break; \
66 \
67 printk(KERN_DEBUG fmt, ##args); \
68 for (j = 0; j < (len) - 1; j++) \
69 printk(KERN_CONT "%02hhx ", buf[j]); \
70 printk(KERN_CONT "%02hhx\n", buf[j]); \
71 } \
72} while (0)
73
74/*
75 * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
76 * that correspond to that nop. Getting from one nop to the next, we
77 * add to the array the offset that is equal to the sum of all sizes of
78 * nops preceding the one we are after.
79 *
80 * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
81 * nice symmetry of sizes of the previous nops.
82 */
83#if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
84static const unsigned char intelnops[] =
85{
86 GENERIC_NOP1,
87 GENERIC_NOP2,
88 GENERIC_NOP3,
89 GENERIC_NOP4,
90 GENERIC_NOP5,
91 GENERIC_NOP6,
92 GENERIC_NOP7,
93 GENERIC_NOP8,
94 GENERIC_NOP5_ATOMIC
95};
96static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
97{
98 NULL,
99 intelnops,
100 intelnops + 1,
101 intelnops + 1 + 2,
102 intelnops + 1 + 2 + 3,
103 intelnops + 1 + 2 + 3 + 4,
104 intelnops + 1 + 2 + 3 + 4 + 5,
105 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
106 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
107 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
108};
109#endif
110
111#ifdef K8_NOP1
112static const unsigned char k8nops[] =
113{
114 K8_NOP1,
115 K8_NOP2,
116 K8_NOP3,
117 K8_NOP4,
118 K8_NOP5,
119 K8_NOP6,
120 K8_NOP7,
121 K8_NOP8,
122 K8_NOP5_ATOMIC
123};
124static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
125{
126 NULL,
127 k8nops,
128 k8nops + 1,
129 k8nops + 1 + 2,
130 k8nops + 1 + 2 + 3,
131 k8nops + 1 + 2 + 3 + 4,
132 k8nops + 1 + 2 + 3 + 4 + 5,
133 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
134 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
135 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
136};
137#endif
138
139#if defined(K7_NOP1) && !defined(CONFIG_X86_64)
140static const unsigned char k7nops[] =
141{
142 K7_NOP1,
143 K7_NOP2,
144 K7_NOP3,
145 K7_NOP4,
146 K7_NOP5,
147 K7_NOP6,
148 K7_NOP7,
149 K7_NOP8,
150 K7_NOP5_ATOMIC
151};
152static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
153{
154 NULL,
155 k7nops,
156 k7nops + 1,
157 k7nops + 1 + 2,
158 k7nops + 1 + 2 + 3,
159 k7nops + 1 + 2 + 3 + 4,
160 k7nops + 1 + 2 + 3 + 4 + 5,
161 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
162 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
163 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
164};
165#endif
166
167#ifdef P6_NOP1
168static const unsigned char p6nops[] =
169{
170 P6_NOP1,
171 P6_NOP2,
172 P6_NOP3,
173 P6_NOP4,
174 P6_NOP5,
175 P6_NOP6,
176 P6_NOP7,
177 P6_NOP8,
178 P6_NOP5_ATOMIC
179};
180static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
181{
182 NULL,
183 p6nops,
184 p6nops + 1,
185 p6nops + 1 + 2,
186 p6nops + 1 + 2 + 3,
187 p6nops + 1 + 2 + 3 + 4,
188 p6nops + 1 + 2 + 3 + 4 + 5,
189 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
190 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
191 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
192};
193#endif
194
195/* Initialize these to a safe default */
196#ifdef CONFIG_X86_64
197const unsigned char * const *ideal_nops = p6_nops;
198#else
199const unsigned char * const *ideal_nops = intel_nops;
200#endif
201
202void __init arch_init_ideal_nops(void)
203{
204 switch (boot_cpu_data.x86_vendor) {
205 case X86_VENDOR_INTEL:
206 /*
207 * Due to a decoder implementation quirk, some
208 * specific Intel CPUs actually perform better with
209 * the "k8_nops" than with the SDM-recommended NOPs.
210 */
211 if (boot_cpu_data.x86 == 6 &&
212 boot_cpu_data.x86_model >= 0x0f &&
213 boot_cpu_data.x86_model != 0x1c &&
214 boot_cpu_data.x86_model != 0x26 &&
215 boot_cpu_data.x86_model != 0x27 &&
216 boot_cpu_data.x86_model < 0x30) {
217 ideal_nops = k8_nops;
218 } else if (boot_cpu_has(X86_FEATURE_NOPL)) {
219 ideal_nops = p6_nops;
220 } else {
221#ifdef CONFIG_X86_64
222 ideal_nops = k8_nops;
223#else
224 ideal_nops = intel_nops;
225#endif
226 }
227 break;
228
229 case X86_VENDOR_HYGON:
230 ideal_nops = p6_nops;
231 return;
232
233 case X86_VENDOR_AMD:
234 if (boot_cpu_data.x86 > 0xf) {
235 ideal_nops = p6_nops;
236 return;
237 }
238
239 /* fall through */
240
241 default:
242#ifdef CONFIG_X86_64
243 ideal_nops = k8_nops;
244#else
245 if (boot_cpu_has(X86_FEATURE_K8))
246 ideal_nops = k8_nops;
247 else if (boot_cpu_has(X86_FEATURE_K7))
248 ideal_nops = k7_nops;
249 else
250 ideal_nops = intel_nops;
251#endif
252 }
253}
254
255/* Use this to add nops to a buffer, then text_poke the whole buffer. */
256static void __init_or_module add_nops(void *insns, unsigned int len)
257{
258 while (len > 0) {
259 unsigned int noplen = len;
260 if (noplen > ASM_NOP_MAX)
261 noplen = ASM_NOP_MAX;
262 memcpy(insns, ideal_nops[noplen], noplen);
263 insns += noplen;
264 len -= noplen;
265 }
266}
267
268extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
269extern s32 __smp_locks[], __smp_locks_end[];
270void text_poke_early(void *addr, const void *opcode, size_t len);
271
272/*
273 * Are we looking at a near JMP with a 1 or 4-byte displacement.
274 */
275static inline bool is_jmp(const u8 opcode)
276{
277 return opcode == 0xeb || opcode == 0xe9;
278}
279
280static void __init_or_module
281recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
282{
283 u8 *next_rip, *tgt_rip;
284 s32 n_dspl, o_dspl;
285 int repl_len;
286
287 if (a->replacementlen != 5)
288 return;
289
290 o_dspl = *(s32 *)(insn_buff + 1);
291
292 /* next_rip of the replacement JMP */
293 next_rip = repl_insn + a->replacementlen;
294 /* target rip of the replacement JMP */
295 tgt_rip = next_rip + o_dspl;
296 n_dspl = tgt_rip - orig_insn;
297
298 DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);
299
300 if (tgt_rip - orig_insn >= 0) {
301 if (n_dspl - 2 <= 127)
302 goto two_byte_jmp;
303 else
304 goto five_byte_jmp;
305 /* negative offset */
306 } else {
307 if (((n_dspl - 2) & 0xff) == (n_dspl - 2))
308 goto two_byte_jmp;
309 else
310 goto five_byte_jmp;
311 }
312
313two_byte_jmp:
314 n_dspl -= 2;
315
316 insn_buff[0] = 0xeb;
317 insn_buff[1] = (s8)n_dspl;
318 add_nops(insn_buff + 2, 3);
319
320 repl_len = 2;
321 goto done;
322
323five_byte_jmp:
324 n_dspl -= 5;
325
326 insn_buff[0] = 0xe9;
327 *(s32 *)&insn_buff[1] = n_dspl;
328
329 repl_len = 5;
330
331done:
332
333 DPRINTK("final displ: 0x%08x, JMP 0x%lx",
334 n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
335}
336
337/*
338 * "noinline" to cause control flow change and thus invalidate I$ and
339 * cause refetch after modification.
340 */
341static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr)
342{
343 unsigned long flags;
344 int i;
345
346 for (i = 0; i < a->padlen; i++) {
347 if (instr[i] != 0x90)
348 return;
349 }
350
351 local_irq_save(flags);
352 add_nops(instr + (a->instrlen - a->padlen), a->padlen);
353 local_irq_restore(flags);
354
355 DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
356 instr, a->instrlen - a->padlen, a->padlen);
357}
358
359/*
360 * Replace instructions with better alternatives for this CPU type. This runs
361 * before SMP is initialized to avoid SMP problems with self modifying code.
362 * This implies that asymmetric systems where APs have less capabilities than
363 * the boot processor are not handled. Tough. Make sure you disable such
364 * features by hand.
365 *
366 * Marked "noinline" to cause control flow change and thus insn cache
367 * to refetch changed I$ lines.
368 */
369void __init_or_module noinline apply_alternatives(struct alt_instr *start,
370 struct alt_instr *end)
371{
372 struct alt_instr *a;
373 u8 *instr, *replacement;
374 u8 insn_buff[MAX_PATCH_LEN];
375
376 DPRINTK("alt table %px, -> %px", start, end);
377 /*
378 * The scan order should be from start to end. A later scanned
379 * alternative code can overwrite previously scanned alternative code.
380 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
381 * patch code.
382 *
383 * So be careful if you want to change the scan order to any other
384 * order.
385 */
386 for (a = start; a < end; a++) {
387 int insn_buff_sz = 0;
388
389 instr = (u8 *)&a->instr_offset + a->instr_offset;
390 replacement = (u8 *)&a->repl_offset + a->repl_offset;
391 BUG_ON(a->instrlen > sizeof(insn_buff));
392 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
393 if (!boot_cpu_has(a->cpuid)) {
394 if (a->padlen > 1)
395 optimize_nops(a, instr);
396
397 continue;
398 }
399
400 DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
401 a->cpuid >> 5,
402 a->cpuid & 0x1f,
403 instr, instr, a->instrlen,
404 replacement, a->replacementlen, a->padlen);
405
406 DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
407 DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
408
409 memcpy(insn_buff, replacement, a->replacementlen);
410 insn_buff_sz = a->replacementlen;
411
412 /*
413 * 0xe8 is a relative jump; fix the offset.
414 *
415 * Instruction length is checked before the opcode to avoid
416 * accessing uninitialized bytes for zero-length replacements.
417 */
418 if (a->replacementlen == 5 && *insn_buff == 0xe8) {
419 *(s32 *)(insn_buff + 1) += replacement - instr;
420 DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
421 *(s32 *)(insn_buff + 1),
422 (unsigned long)instr + *(s32 *)(insn_buff + 1) + 5);
423 }
424
425 if (a->replacementlen && is_jmp(replacement[0]))
426 recompute_jump(a, instr, replacement, insn_buff);
427
428 if (a->instrlen > a->replacementlen) {
429 add_nops(insn_buff + a->replacementlen,
430 a->instrlen - a->replacementlen);
431 insn_buff_sz += a->instrlen - a->replacementlen;
432 }
433 DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
434
435 text_poke_early(instr, insn_buff, insn_buff_sz);
436 }
437}
438
439#ifdef CONFIG_SMP
440static void alternatives_smp_lock(const s32 *start, const s32 *end,
441 u8 *text, u8 *text_end)
442{
443 const s32 *poff;
444
445 for (poff = start; poff < end; poff++) {
446 u8 *ptr = (u8 *)poff + *poff;
447
448 if (!*poff || ptr < text || ptr >= text_end)
449 continue;
450 /* turn DS segment override prefix into lock prefix */
451 if (*ptr == 0x3e)
452 text_poke(ptr, ((unsigned char []){0xf0}), 1);
453 }
454}
455
456static void alternatives_smp_unlock(const s32 *start, const s32 *end,
457 u8 *text, u8 *text_end)
458{
459 const s32 *poff;
460
461 for (poff = start; poff < end; poff++) {
462 u8 *ptr = (u8 *)poff + *poff;
463
464 if (!*poff || ptr < text || ptr >= text_end)
465 continue;
466 /* turn lock prefix into DS segment override prefix */
467 if (*ptr == 0xf0)
468 text_poke(ptr, ((unsigned char []){0x3E}), 1);
469 }
470}
471
472struct smp_alt_module {
473 /* what is this ??? */
474 struct module *mod;
475 char *name;
476
477 /* ptrs to lock prefixes */
478 const s32 *locks;
479 const s32 *locks_end;
480
481 /* .text segment, needed to avoid patching init code ;) */
482 u8 *text;
483 u8 *text_end;
484
485 struct list_head next;
486};
487static LIST_HEAD(smp_alt_modules);
488static bool uniproc_patched = false; /* protected by text_mutex */
489
490void __init_or_module alternatives_smp_module_add(struct module *mod,
491 char *name,
492 void *locks, void *locks_end,
493 void *text, void *text_end)
494{
495 struct smp_alt_module *smp;
496
497 mutex_lock(&text_mutex);
498 if (!uniproc_patched)
499 goto unlock;
500
501 if (num_possible_cpus() == 1)
502 /* Don't bother remembering, we'll never have to undo it. */
503 goto smp_unlock;
504
505 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
506 if (NULL == smp)
507 /* we'll run the (safe but slow) SMP code then ... */
508 goto unlock;
509
510 smp->mod = mod;
511 smp->name = name;
512 smp->locks = locks;
513 smp->locks_end = locks_end;
514 smp->text = text;
515 smp->text_end = text_end;
516 DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",
517 smp->locks, smp->locks_end,
518 smp->text, smp->text_end, smp->name);
519
520 list_add_tail(&smp->next, &smp_alt_modules);
521smp_unlock:
522 alternatives_smp_unlock(locks, locks_end, text, text_end);
523unlock:
524 mutex_unlock(&text_mutex);
525}
526
527void __init_or_module alternatives_smp_module_del(struct module *mod)
528{
529 struct smp_alt_module *item;
530
531 mutex_lock(&text_mutex);
532 list_for_each_entry(item, &smp_alt_modules, next) {
533 if (mod != item->mod)
534 continue;
535 list_del(&item->next);
536 kfree(item);
537 break;
538 }
539 mutex_unlock(&text_mutex);
540}
541
542void alternatives_enable_smp(void)
543{
544 struct smp_alt_module *mod;
545
546 /* Why bother if there are no other CPUs? */
547 BUG_ON(num_possible_cpus() == 1);
548
549 mutex_lock(&text_mutex);
550
551 if (uniproc_patched) {
552 pr_info("switching to SMP code\n");
553 BUG_ON(num_online_cpus() != 1);
554 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
555 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
556 list_for_each_entry(mod, &smp_alt_modules, next)
557 alternatives_smp_lock(mod->locks, mod->locks_end,
558 mod->text, mod->text_end);
559 uniproc_patched = false;
560 }
561 mutex_unlock(&text_mutex);
562}
563
564/*
565 * Return 1 if the address range is reserved for SMP-alternatives.
566 * Must hold text_mutex.
567 */
568int alternatives_text_reserved(void *start, void *end)
569{
570 struct smp_alt_module *mod;
571 const s32 *poff;
572 u8 *text_start = start;
573 u8 *text_end = end;
574
575 lockdep_assert_held(&text_mutex);
576
577 list_for_each_entry(mod, &smp_alt_modules, next) {
578 if (mod->text > text_end || mod->text_end < text_start)
579 continue;
580 for (poff = mod->locks; poff < mod->locks_end; poff++) {
581 const u8 *ptr = (const u8 *)poff + *poff;
582
583 if (text_start <= ptr && text_end > ptr)
584 return 1;
585 }
586 }
587
588 return 0;
589}
590#endif /* CONFIG_SMP */
591
592#ifdef CONFIG_PARAVIRT
593void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
594 struct paravirt_patch_site *end)
595{
596 struct paravirt_patch_site *p;
597 char insn_buff[MAX_PATCH_LEN];
598
599 for (p = start; p < end; p++) {
600 unsigned int used;
601
602 BUG_ON(p->len > MAX_PATCH_LEN);
603 /* prep the buffer with the original instructions */
604 memcpy(insn_buff, p->instr, p->len);
605 used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
606
607 BUG_ON(used > p->len);
608
609 /* Pad the rest with nops */
610 add_nops(insn_buff + used, p->len - used);
611 text_poke_early(p->instr, insn_buff, p->len);
612 }
613}
614extern struct paravirt_patch_site __start_parainstructions[],
615 __stop_parainstructions[];
616#endif /* CONFIG_PARAVIRT */
617
618/*
619 * Self-test for the INT3 based CALL emulation code.
620 *
621 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
622 * properly and that there is a stack gap between the INT3 frame and the
623 * previous context. Without this gap doing a virtual PUSH on the interrupted
624 * stack would corrupt the INT3 IRET frame.
625 *
626 * See entry_{32,64}.S for more details.
627 */
628
629/*
630 * We define the int3_magic() function in assembly to control the calling
631 * convention such that we can 'call' it from assembly.
632 */
633
634extern void int3_magic(unsigned int *ptr); /* defined in asm */
635
636asm (
637" .pushsection .init.text, \"ax\", @progbits\n"
638" .type int3_magic, @function\n"
639"int3_magic:\n"
640" movl $1, (%" _ASM_ARG1 ")\n"
641" ret\n"
642" .size int3_magic, .-int3_magic\n"
643" .popsection\n"
644);
645
646extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
647
648static int __init
649int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
650{
651 struct die_args *args = data;
652 struct pt_regs *regs = args->regs;
653
654 if (!regs || user_mode(regs))
655 return NOTIFY_DONE;
656
657 if (val != DIE_INT3)
658 return NOTIFY_DONE;
659
660 if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
661 return NOTIFY_DONE;
662
663 int3_emulate_call(regs, (unsigned long)&int3_magic);
664 return NOTIFY_STOP;
665}
666
667static void __init int3_selftest(void)
668{
669 static __initdata struct notifier_block int3_exception_nb = {
670 .notifier_call = int3_exception_notify,
671 .priority = INT_MAX-1, /* last */
672 };
673 unsigned int val = 0;
674
675 BUG_ON(register_die_notifier(&int3_exception_nb));
676
677 /*
678 * Basically: int3_magic(&val); but really complicated :-)
679 *
680 * Stick the address of the INT3 instruction into int3_selftest_ip,
681 * then trigger the INT3, padded with NOPs to match a CALL instruction
682 * length.
683 */
684 asm volatile ("1: int3; nop; nop; nop; nop\n\t"
685 ".pushsection .init.data,\"aw\"\n\t"
686 ".align " __ASM_SEL(4, 8) "\n\t"
687 ".type int3_selftest_ip, @object\n\t"
688 ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
689 "int3_selftest_ip:\n\t"
690 __ASM_SEL(.long, .quad) " 1b\n\t"
691 ".popsection\n\t"
692 : ASM_CALL_CONSTRAINT
693 : __ASM_SEL_RAW(a, D) (&val)
694 : "memory");
695
696 BUG_ON(val != 1);
697
698 unregister_die_notifier(&int3_exception_nb);
699}
700
701void __init alternative_instructions(void)
702{
703 int3_selftest();
704
705 /*
706 * The patching is not fully atomic, so try to avoid local
707 * interruptions that might execute the to be patched code.
708 * Other CPUs are not running.
709 */
710 stop_nmi();
711
712 /*
713 * Don't stop machine check exceptions while patching.
714 * MCEs only happen when something got corrupted and in this
715 * case we must do something about the corruption.
716 * Ignoring it is worse than an unlikely patching race.
717 * Also machine checks tend to be broadcast and if one CPU
718 * goes into machine check the others follow quickly, so we don't
719 * expect a machine check to cause undue problems during to code
720 * patching.
721 */
722
723 apply_alternatives(__alt_instructions, __alt_instructions_end);
724
725#ifdef CONFIG_SMP
726 /* Patch to UP if other cpus not imminent. */
727 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
728 uniproc_patched = true;
729 alternatives_smp_module_add(NULL, "core kernel",
730 __smp_locks, __smp_locks_end,
731 _text, _etext);
732 }
733
734 if (!uniproc_patched || num_possible_cpus() == 1) {
735 free_init_pages("SMP alternatives",
736 (unsigned long)__smp_locks,
737 (unsigned long)__smp_locks_end);
738 }
739#endif
740
741 apply_paravirt(__parainstructions, __parainstructions_end);
742
743 restart_nmi();
744 alternatives_patched = 1;
745}
746
747/**
748 * text_poke_early - Update instructions on a live kernel at boot time
749 * @addr: address to modify
750 * @opcode: source of the copy
751 * @len: length to copy
752 *
753 * When you use this code to patch more than one byte of an instruction
754 * you need to make sure that other CPUs cannot execute this code in parallel.
755 * Also no thread must be currently preempted in the middle of these
756 * instructions. And on the local CPU you need to be protected against NMI or
757 * MCE handlers seeing an inconsistent instruction while you patch.
758 */
759void __init_or_module text_poke_early(void *addr, const void *opcode,
760 size_t len)
761{
762 unsigned long flags;
763
764 if (boot_cpu_has(X86_FEATURE_NX) &&
765 is_module_text_address((unsigned long)addr)) {
766 /*
767 * Modules text is marked initially as non-executable, so the
768 * code cannot be running and speculative code-fetches are
769 * prevented. Just change the code.
770 */
771 memcpy(addr, opcode, len);
772 } else {
773 local_irq_save(flags);
774 memcpy(addr, opcode, len);
775 local_irq_restore(flags);
776 sync_core();
777
778 /*
779 * Could also do a CLFLUSH here to speed up CPU recovery; but
780 * that causes hangs on some VIA CPUs.
781 */
782 }
783}
784
785__ro_after_init struct mm_struct *poking_mm;
786__ro_after_init unsigned long poking_addr;
787
788static void *__text_poke(void *addr, const void *opcode, size_t len)
789{
790 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
791 struct page *pages[2] = {NULL};
792 temp_mm_state_t prev;
793 unsigned long flags;
794 pte_t pte, *ptep;
795 spinlock_t *ptl;
796 pgprot_t pgprot;
797
798 /*
799 * While boot memory allocator is running we cannot use struct pages as
800 * they are not yet initialized. There is no way to recover.
801 */
802 BUG_ON(!after_bootmem);
803
804 if (!core_kernel_text((unsigned long)addr)) {
805 pages[0] = vmalloc_to_page(addr);
806 if (cross_page_boundary)
807 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
808 } else {
809 pages[0] = virt_to_page(addr);
810 WARN_ON(!PageReserved(pages[0]));
811 if (cross_page_boundary)
812 pages[1] = virt_to_page(addr + PAGE_SIZE);
813 }
814 /*
815 * If something went wrong, crash and burn since recovery paths are not
816 * implemented.
817 */
818 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
819
820 local_irq_save(flags);
821
822 /*
823 * Map the page without the global bit, as TLB flushing is done with
824 * flush_tlb_mm_range(), which is intended for non-global PTEs.
825 */
826 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
827
828 /*
829 * The lock is not really needed, but this allows to avoid open-coding.
830 */
831 ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
832
833 /*
834 * This must not fail; preallocated in poking_init().
835 */
836 VM_BUG_ON(!ptep);
837
838 pte = mk_pte(pages[0], pgprot);
839 set_pte_at(poking_mm, poking_addr, ptep, pte);
840
841 if (cross_page_boundary) {
842 pte = mk_pte(pages[1], pgprot);
843 set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte);
844 }
845
846 /*
847 * Loading the temporary mm behaves as a compiler barrier, which
848 * guarantees that the PTE will be set at the time memcpy() is done.
849 */
850 prev = use_temporary_mm(poking_mm);
851
852 kasan_disable_current();
853 memcpy((u8 *)poking_addr + offset_in_page(addr), opcode, len);
854 kasan_enable_current();
855
856 /*
857 * Ensure that the PTE is only cleared after the instructions of memcpy
858 * were issued by using a compiler barrier.
859 */
860 barrier();
861
862 pte_clear(poking_mm, poking_addr, ptep);
863 if (cross_page_boundary)
864 pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1);
865
866 /*
867 * Loading the previous page-table hierarchy requires a serializing
868 * instruction that already allows the core to see the updated version.
869 * Xen-PV is assumed to serialize execution in a similar manner.
870 */
871 unuse_temporary_mm(prev);
872
873 /*
874 * Flushing the TLB might involve IPIs, which would require enabled
875 * IRQs, but not if the mm is not used, as it is in this point.
876 */
877 flush_tlb_mm_range(poking_mm, poking_addr, poking_addr +
878 (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
879 PAGE_SHIFT, false);
880
881 /*
882 * If the text does not match what we just wrote then something is
883 * fundamentally screwy; there's nothing we can really do about that.
884 */
885 BUG_ON(memcmp(addr, opcode, len));
886
887 pte_unmap_unlock(ptep, ptl);
888 local_irq_restore(flags);
889 return addr;
890}
891
892/**
893 * text_poke - Update instructions on a live kernel
894 * @addr: address to modify
895 * @opcode: source of the copy
896 * @len: length to copy
897 *
898 * Only atomic text poke/set should be allowed when not doing early patching.
899 * It means the size must be writable atomically and the address must be aligned
900 * in a way that permits an atomic write. It also makes sure we fit on a single
901 * page.
902 *
903 * Note that the caller must ensure that if the modified code is part of a
904 * module, the module would not be removed during poking. This can be achieved
905 * by registering a module notifier, and ordering module removal and patching
906 * trough a mutex.
907 */
908void *text_poke(void *addr, const void *opcode, size_t len)
909{
910 lockdep_assert_held(&text_mutex);
911
912 return __text_poke(addr, opcode, len);
913}
914
915/**
916 * text_poke_kgdb - Update instructions on a live kernel by kgdb
917 * @addr: address to modify
918 * @opcode: source of the copy
919 * @len: length to copy
920 *
921 * Only atomic text poke/set should be allowed when not doing early patching.
922 * It means the size must be writable atomically and the address must be aligned
923 * in a way that permits an atomic write. It also makes sure we fit on a single
924 * page.
925 *
926 * Context: should only be used by kgdb, which ensures no other core is running,
927 * despite the fact it does not hold the text_mutex.
928 */
929void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
930{
931 return __text_poke(addr, opcode, len);
932}
933
934static void do_sync_core(void *info)
935{
936 sync_core();
937}
938
939static struct bp_patching_desc {
940 struct text_poke_loc *vec;
941 int nr_entries;
942} bp_patching;
943
944static int patch_cmp(const void *key, const void *elt)
945{
946 struct text_poke_loc *tp = (struct text_poke_loc *) elt;
947
948 if (key < tp->addr)
949 return -1;
950 if (key > tp->addr)
951 return 1;
952 return 0;
953}
954NOKPROBE_SYMBOL(patch_cmp);
955
956int poke_int3_handler(struct pt_regs *regs)
957{
958 struct text_poke_loc *tp;
959 unsigned char int3 = 0xcc;
960 void *ip;
961
962 /*
963 * Having observed our INT3 instruction, we now must observe
964 * bp_patching.nr_entries.
965 *
966 * nr_entries != 0 INT3
967 * WMB RMB
968 * write INT3 if (nr_entries)
969 *
970 * Idem for other elements in bp_patching.
971 */
972 smp_rmb();
973
974 if (likely(!bp_patching.nr_entries))
975 return 0;
976
977 if (user_mode(regs))
978 return 0;
979
980 /*
981 * Discount the sizeof(int3). See text_poke_bp_batch().
982 */
983 ip = (void *) regs->ip - sizeof(int3);
984
985 /*
986 * Skip the binary search if there is a single member in the vector.
987 */
988 if (unlikely(bp_patching.nr_entries > 1)) {
989 tp = bsearch(ip, bp_patching.vec, bp_patching.nr_entries,
990 sizeof(struct text_poke_loc),
991 patch_cmp);
992 if (!tp)
993 return 0;
994 } else {
995 tp = bp_patching.vec;
996 if (tp->addr != ip)
997 return 0;
998 }
999
1000 /* set up the specified breakpoint detour */
1001 regs->ip = (unsigned long) tp->detour;
1002
1003 return 1;
1004}
1005NOKPROBE_SYMBOL(poke_int3_handler);
1006
1007/**
1008 * text_poke_bp_batch() -- update instructions on live kernel on SMP
1009 * @tp: vector of instructions to patch
1010 * @nr_entries: number of entries in the vector
1011 *
1012 * Modify multi-byte instruction by using int3 breakpoint on SMP.
1013 * We completely avoid stop_machine() here, and achieve the
1014 * synchronization using int3 breakpoint.
1015 *
1016 * The way it is done:
1017 * - For each entry in the vector:
1018 * - add a int3 trap to the address that will be patched
1019 * - sync cores
1020 * - For each entry in the vector:
1021 * - update all but the first byte of the patched range
1022 * - sync cores
1023 * - For each entry in the vector:
1024 * - replace the first byte (int3) by the first byte of
1025 * replacing opcode
1026 * - sync cores
1027 */
1028void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
1029{
1030 int patched_all_but_first = 0;
1031 unsigned char int3 = 0xcc;
1032 unsigned int i;
1033
1034 lockdep_assert_held(&text_mutex);
1035
1036 bp_patching.vec = tp;
1037 bp_patching.nr_entries = nr_entries;
1038
1039 /*
1040 * Corresponding read barrier in int3 notifier for making sure the
1041 * nr_entries and handler are correctly ordered wrt. patching.
1042 */
1043 smp_wmb();
1044
1045 /*
1046 * First step: add a int3 trap to the address that will be patched.
1047 */
1048 for (i = 0; i < nr_entries; i++)
1049 text_poke(tp[i].addr, &int3, sizeof(int3));
1050
1051 on_each_cpu(do_sync_core, NULL, 1);
1052
1053 /*
1054 * Second step: update all but the first byte of the patched range.
1055 */
1056 for (i = 0; i < nr_entries; i++) {
1057 if (tp[i].len - sizeof(int3) > 0) {
1058 text_poke((char *)tp[i].addr + sizeof(int3),
1059 (const char *)tp[i].opcode + sizeof(int3),
1060 tp[i].len - sizeof(int3));
1061 patched_all_but_first++;
1062 }
1063 }
1064
1065 if (patched_all_but_first) {
1066 /*
1067 * According to Intel, this core syncing is very likely
1068 * not necessary and we'd be safe even without it. But
1069 * better safe than sorry (plus there's not only Intel).
1070 */
1071 on_each_cpu(do_sync_core, NULL, 1);
1072 }
1073
1074 /*
1075 * Third step: replace the first byte (int3) by the first byte of
1076 * replacing opcode.
1077 */
1078 for (i = 0; i < nr_entries; i++)
1079 text_poke(tp[i].addr, tp[i].opcode, sizeof(int3));
1080
1081 on_each_cpu(do_sync_core, NULL, 1);
1082 /*
1083 * sync_core() implies an smp_mb() and orders this store against
1084 * the writing of the new instruction.
1085 */
1086 bp_patching.vec = NULL;
1087 bp_patching.nr_entries = 0;
1088}
1089
1090/**
1091 * text_poke_bp() -- update instructions on live kernel on SMP
1092 * @addr: address to patch
1093 * @opcode: opcode of new instruction
1094 * @len: length to copy
1095 * @handler: address to jump to when the temporary breakpoint is hit
1096 *
1097 * Update a single instruction with the vector in the stack, avoiding
1098 * dynamically allocated memory. This function should be used when it is
1099 * not possible to allocate memory.
1100 */
1101void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
1102{
1103 struct text_poke_loc tp = {
1104 .detour = handler,
1105 .addr = addr,
1106 .len = len,
1107 };
1108
1109 if (len > POKE_MAX_OPCODE_SIZE) {
1110 WARN_ONCE(1, "len is larger than %d\n", POKE_MAX_OPCODE_SIZE);
1111 return;
1112 }
1113
1114 memcpy((void *)tp.opcode, opcode, len);
1115
1116 text_poke_bp_batch(&tp, 1);
1117}
1// SPDX-License-Identifier: GPL-2.0-only
2#define pr_fmt(fmt) "SMP alternatives: " fmt
3
4#include <linux/module.h>
5#include <linux/sched.h>
6#include <linux/perf_event.h>
7#include <linux/mutex.h>
8#include <linux/list.h>
9#include <linux/stringify.h>
10#include <linux/highmem.h>
11#include <linux/mm.h>
12#include <linux/vmalloc.h>
13#include <linux/memory.h>
14#include <linux/stop_machine.h>
15#include <linux/slab.h>
16#include <linux/kdebug.h>
17#include <linux/kprobes.h>
18#include <linux/mmu_context.h>
19#include <linux/bsearch.h>
20#include <linux/sync_core.h>
21#include <asm/text-patching.h>
22#include <asm/alternative.h>
23#include <asm/sections.h>
24#include <asm/mce.h>
25#include <asm/nmi.h>
26#include <asm/cacheflush.h>
27#include <asm/tlbflush.h>
28#include <asm/insn.h>
29#include <asm/io.h>
30#include <asm/fixmap.h>
31#include <asm/paravirt.h>
32#include <asm/asm-prototypes.h>
33#include <asm/cfi.h>
34
35int __read_mostly alternatives_patched;
36
37EXPORT_SYMBOL_GPL(alternatives_patched);
38
39#define MAX_PATCH_LEN (255-1)
40
41#define DA_ALL (~0)
42#define DA_ALT 0x01
43#define DA_RET 0x02
44#define DA_RETPOLINE 0x04
45#define DA_ENDBR 0x08
46#define DA_SMP 0x10
47
48static unsigned int __initdata_or_module debug_alternative;
49
50static int __init debug_alt(char *str)
51{
52 if (str && *str == '=')
53 str++;
54
55 if (!str || kstrtouint(str, 0, &debug_alternative))
56 debug_alternative = DA_ALL;
57
58 return 1;
59}
60__setup("debug-alternative", debug_alt);
61
62static int noreplace_smp;
63
64static int __init setup_noreplace_smp(char *str)
65{
66 noreplace_smp = 1;
67 return 1;
68}
69__setup("noreplace-smp", setup_noreplace_smp);
70
71#define DPRINTK(type, fmt, args...) \
72do { \
73 if (debug_alternative & DA_##type) \
74 printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \
75} while (0)
76
77#define DUMP_BYTES(type, buf, len, fmt, args...) \
78do { \
79 if (unlikely(debug_alternative & DA_##type)) { \
80 int j; \
81 \
82 if (!(len)) \
83 break; \
84 \
85 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
86 for (j = 0; j < (len) - 1; j++) \
87 printk(KERN_CONT "%02hhx ", buf[j]); \
88 printk(KERN_CONT "%02hhx\n", buf[j]); \
89 } \
90} while (0)
91
92static const unsigned char x86nops[] =
93{
94 BYTES_NOP1,
95 BYTES_NOP2,
96 BYTES_NOP3,
97 BYTES_NOP4,
98 BYTES_NOP5,
99 BYTES_NOP6,
100 BYTES_NOP7,
101 BYTES_NOP8,
102#ifdef CONFIG_64BIT
103 BYTES_NOP9,
104 BYTES_NOP10,
105 BYTES_NOP11,
106#endif
107};
108
109const unsigned char * const x86_nops[ASM_NOP_MAX+1] =
110{
111 NULL,
112 x86nops,
113 x86nops + 1,
114 x86nops + 1 + 2,
115 x86nops + 1 + 2 + 3,
116 x86nops + 1 + 2 + 3 + 4,
117 x86nops + 1 + 2 + 3 + 4 + 5,
118 x86nops + 1 + 2 + 3 + 4 + 5 + 6,
119 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
120#ifdef CONFIG_64BIT
121 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
122 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9,
123 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10,
124#endif
125};
126
127/*
128 * Fill the buffer with a single effective instruction of size @len.
129 *
130 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info)
131 * for every single-byte NOP, try to generate the maximally available NOP of
132 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for
133 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
134 * *jump* over instead of executing long and daft NOPs.
135 */
136static void __init_or_module add_nop(u8 *instr, unsigned int len)
137{
138 u8 *target = instr + len;
139
140 if (!len)
141 return;
142
143 if (len <= ASM_NOP_MAX) {
144 memcpy(instr, x86_nops[len], len);
145 return;
146 }
147
148 if (len < 128) {
149 __text_gen_insn(instr, JMP8_INSN_OPCODE, instr, target, JMP8_INSN_SIZE);
150 instr += JMP8_INSN_SIZE;
151 } else {
152 __text_gen_insn(instr, JMP32_INSN_OPCODE, instr, target, JMP32_INSN_SIZE);
153 instr += JMP32_INSN_SIZE;
154 }
155
156 for (;instr < target; instr++)
157 *instr = INT3_INSN_OPCODE;
158}
159
160extern s32 __retpoline_sites[], __retpoline_sites_end[];
161extern s32 __return_sites[], __return_sites_end[];
162extern s32 __cfi_sites[], __cfi_sites_end[];
163extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
164extern s32 __smp_locks[], __smp_locks_end[];
165void text_poke_early(void *addr, const void *opcode, size_t len);
166
167/*
168 * Matches NOP and NOPL, not any of the other possible NOPs.
169 */
170static bool insn_is_nop(struct insn *insn)
171{
172 /* Anything NOP, but no REP NOP */
173 if (insn->opcode.bytes[0] == 0x90 &&
174 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3))
175 return true;
176
177 /* NOPL */
178 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F)
179 return true;
180
181 /* TODO: more nops */
182
183 return false;
184}
185
186/*
187 * Find the offset of the first non-NOP instruction starting at @offset
188 * but no further than @len.
189 */
190static int skip_nops(u8 *instr, int offset, int len)
191{
192 struct insn insn;
193
194 for (; offset < len; offset += insn.length) {
195 if (insn_decode_kernel(&insn, &instr[offset]))
196 break;
197
198 if (!insn_is_nop(&insn))
199 break;
200 }
201
202 return offset;
203}
204
205/*
206 * Optimize a sequence of NOPs, possibly preceded by an unconditional jump
207 * to the end of the NOP sequence into a single NOP.
208 */
209static bool __init_or_module
210__optimize_nops(u8 *instr, size_t len, struct insn *insn, int *next, int *prev, int *target)
211{
212 int i = *next - insn->length;
213
214 switch (insn->opcode.bytes[0]) {
215 case JMP8_INSN_OPCODE:
216 case JMP32_INSN_OPCODE:
217 *prev = i;
218 *target = *next + insn->immediate.value;
219 return false;
220 }
221
222 if (insn_is_nop(insn)) {
223 int nop = i;
224
225 *next = skip_nops(instr, *next, len);
226 if (*target && *next == *target)
227 nop = *prev;
228
229 add_nop(instr + nop, *next - nop);
230 DUMP_BYTES(ALT, instr, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, *next);
231 return true;
232 }
233
234 *target = 0;
235 return false;
236}
237
238/*
239 * "noinline" to cause control flow change and thus invalidate I$ and
240 * cause refetch after modification.
241 */
242static void __init_or_module noinline optimize_nops(u8 *instr, size_t len)
243{
244 int prev, target = 0;
245
246 for (int next, i = 0; i < len; i = next) {
247 struct insn insn;
248
249 if (insn_decode_kernel(&insn, &instr[i]))
250 return;
251
252 next = i + insn.length;
253
254 __optimize_nops(instr, len, &insn, &next, &prev, &target);
255 }
256}
257
258static void __init_or_module noinline optimize_nops_inplace(u8 *instr, size_t len)
259{
260 unsigned long flags;
261
262 local_irq_save(flags);
263 optimize_nops(instr, len);
264 sync_core();
265 local_irq_restore(flags);
266}
267
268/*
269 * In this context, "source" is where the instructions are placed in the
270 * section .altinstr_replacement, for example during kernel build by the
271 * toolchain.
272 * "Destination" is where the instructions are being patched in by this
273 * machinery.
274 *
275 * The source offset is:
276 *
277 * src_imm = target - src_next_ip (1)
278 *
279 * and the target offset is:
280 *
281 * dst_imm = target - dst_next_ip (2)
282 *
283 * so rework (1) as an expression for target like:
284 *
285 * target = src_imm + src_next_ip (1a)
286 *
287 * and substitute in (2) to get:
288 *
289 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
290 *
291 * Now, since the instruction stream is 'identical' at src and dst (it
292 * is being copied after all) it can be stated that:
293 *
294 * src_next_ip = src + ip_offset
295 * dst_next_ip = dst + ip_offset (4)
296 *
297 * Substitute (4) in (3) and observe ip_offset being cancelled out to
298 * obtain:
299 *
300 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
301 * = src_imm + src - dst + ip_offset - ip_offset
302 * = src_imm + src - dst (5)
303 *
304 * IOW, only the relative displacement of the code block matters.
305 */
306
307#define apply_reloc_n(n_, p_, d_) \
308 do { \
309 s32 v = *(s##n_ *)(p_); \
310 v += (d_); \
311 BUG_ON((v >> 31) != (v >> (n_-1))); \
312 *(s##n_ *)(p_) = (s##n_)v; \
313 } while (0)
314
315
316static __always_inline
317void apply_reloc(int n, void *ptr, uintptr_t diff)
318{
319 switch (n) {
320 case 1: apply_reloc_n(8, ptr, diff); break;
321 case 2: apply_reloc_n(16, ptr, diff); break;
322 case 4: apply_reloc_n(32, ptr, diff); break;
323 default: BUG();
324 }
325}
326
327static __always_inline
328bool need_reloc(unsigned long offset, u8 *src, size_t src_len)
329{
330 u8 *target = src + offset;
331 /*
332 * If the target is inside the patched block, it's relative to the
333 * block itself and does not need relocation.
334 */
335 return (target < src || target > src + src_len);
336}
337
338static void __init_or_module noinline
339apply_relocation(u8 *buf, size_t len, u8 *dest, u8 *src, size_t src_len)
340{
341 int prev, target = 0;
342
343 for (int next, i = 0; i < len; i = next) {
344 struct insn insn;
345
346 if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i])))
347 return;
348
349 next = i + insn.length;
350
351 if (__optimize_nops(buf, len, &insn, &next, &prev, &target))
352 continue;
353
354 switch (insn.opcode.bytes[0]) {
355 case 0x0f:
356 if (insn.opcode.bytes[1] < 0x80 ||
357 insn.opcode.bytes[1] > 0x8f)
358 break;
359
360 fallthrough; /* Jcc.d32 */
361 case 0x70 ... 0x7f: /* Jcc.d8 */
362 case JMP8_INSN_OPCODE:
363 case JMP32_INSN_OPCODE:
364 case CALL_INSN_OPCODE:
365 if (need_reloc(next + insn.immediate.value, src, src_len)) {
366 apply_reloc(insn.immediate.nbytes,
367 buf + i + insn_offset_immediate(&insn),
368 src - dest);
369 }
370
371 /*
372 * Where possible, convert JMP.d32 into JMP.d8.
373 */
374 if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {
375 s32 imm = insn.immediate.value;
376 imm += src - dest;
377 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;
378 if ((imm >> 31) == (imm >> 7)) {
379 buf[i+0] = JMP8_INSN_OPCODE;
380 buf[i+1] = (s8)imm;
381
382 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2);
383 }
384 }
385 break;
386 }
387
388 if (insn_rip_relative(&insn)) {
389 if (need_reloc(next + insn.displacement.value, src, src_len)) {
390 apply_reloc(insn.displacement.nbytes,
391 buf + i + insn_offset_displacement(&insn),
392 src - dest);
393 }
394 }
395 }
396}
397
398/* Low-level backend functions usable from alternative code replacements. */
399DEFINE_ASM_FUNC(nop_func, "", .entry.text);
400EXPORT_SYMBOL_GPL(nop_func);
401
402noinstr void BUG_func(void)
403{
404 BUG();
405}
406EXPORT_SYMBOL(BUG_func);
407
408#define CALL_RIP_REL_OPCODE 0xff
409#define CALL_RIP_REL_MODRM 0x15
410
411/*
412 * Rewrite the "call BUG_func" replacement to point to the target of the
413 * indirect pv_ops call "call *disp(%ip)".
414 */
415static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)
416{
417 void *target, *bug = &BUG_func;
418 s32 disp;
419
420 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {
421 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n");
422 BUG();
423 }
424
425 if (a->instrlen != 6 ||
426 instr[0] != CALL_RIP_REL_OPCODE ||
427 instr[1] != CALL_RIP_REL_MODRM) {
428 pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n");
429 BUG();
430 }
431
432 /* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */
433 disp = *(s32 *)(instr + 2);
434#ifdef CONFIG_X86_64
435 /* ff 15 00 00 00 00 call *0x0(%rip) */
436 /* target address is stored at "next instruction + disp". */
437 target = *(void **)(instr + a->instrlen + disp);
438#else
439 /* ff 15 00 00 00 00 call *0x0 */
440 /* target address is stored at disp. */
441 target = *(void **)disp;
442#endif
443 if (!target)
444 target = bug;
445
446 /* (BUG_func - .) + (target - BUG_func) := target - . */
447 *(s32 *)(insn_buff + 1) += target - bug;
448
449 if (target == &nop_func)
450 return 0;
451
452 return 5;
453}
454
455/*
456 * Replace instructions with better alternatives for this CPU type. This runs
457 * before SMP is initialized to avoid SMP problems with self modifying code.
458 * This implies that asymmetric systems where APs have less capabilities than
459 * the boot processor are not handled. Tough. Make sure you disable such
460 * features by hand.
461 *
462 * Marked "noinline" to cause control flow change and thus insn cache
463 * to refetch changed I$ lines.
464 */
465void __init_or_module noinline apply_alternatives(struct alt_instr *start,
466 struct alt_instr *end)
467{
468 struct alt_instr *a;
469 u8 *instr, *replacement;
470 u8 insn_buff[MAX_PATCH_LEN];
471
472 DPRINTK(ALT, "alt table %px, -> %px", start, end);
473
474 /*
475 * In the case CONFIG_X86_5LEVEL=y, KASAN_SHADOW_START is defined using
476 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here.
477 * During the process, KASAN becomes confused seeing partial LA57
478 * conversion and triggers a false-positive out-of-bound report.
479 *
480 * Disable KASAN until the patching is complete.
481 */
482 kasan_disable_current();
483
484 /*
485 * The scan order should be from start to end. A later scanned
486 * alternative code can overwrite previously scanned alternative code.
487 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
488 * patch code.
489 *
490 * So be careful if you want to change the scan order to any other
491 * order.
492 */
493 for (a = start; a < end; a++) {
494 int insn_buff_sz = 0;
495
496 instr = (u8 *)&a->instr_offset + a->instr_offset;
497 replacement = (u8 *)&a->repl_offset + a->repl_offset;
498 BUG_ON(a->instrlen > sizeof(insn_buff));
499 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
500
501 /*
502 * Patch if either:
503 * - feature is present
504 * - feature not present but ALT_FLAG_NOT is set to mean,
505 * patch if feature is *NOT* present.
506 */
507 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) {
508 optimize_nops_inplace(instr, a->instrlen);
509 continue;
510 }
511
512 DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x",
513 a->cpuid >> 5,
514 a->cpuid & 0x1f,
515 instr, instr, a->instrlen,
516 replacement, a->replacementlen, a->flags);
517
518 memcpy(insn_buff, replacement, a->replacementlen);
519 insn_buff_sz = a->replacementlen;
520
521 if (a->flags & ALT_FLAG_DIRECT_CALL) {
522 insn_buff_sz = alt_replace_call(instr, insn_buff, a);
523 if (insn_buff_sz < 0)
524 continue;
525 }
526
527 for (; insn_buff_sz < a->instrlen; insn_buff_sz++)
528 insn_buff[insn_buff_sz] = 0x90;
529
530 apply_relocation(insn_buff, a->instrlen, instr, replacement, a->replacementlen);
531
532 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr);
533 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
534 DUMP_BYTES(ALT, insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
535
536 text_poke_early(instr, insn_buff, insn_buff_sz);
537 }
538
539 kasan_enable_current();
540}
541
542static inline bool is_jcc32(struct insn *insn)
543{
544 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */
545 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;
546}
547
548#if defined(CONFIG_RETPOLINE) && defined(CONFIG_OBJTOOL)
549
550/*
551 * CALL/JMP *%\reg
552 */
553static int emit_indirect(int op, int reg, u8 *bytes)
554{
555 int i = 0;
556 u8 modrm;
557
558 switch (op) {
559 case CALL_INSN_OPCODE:
560 modrm = 0x10; /* Reg = 2; CALL r/m */
561 break;
562
563 case JMP32_INSN_OPCODE:
564 modrm = 0x20; /* Reg = 4; JMP r/m */
565 break;
566
567 default:
568 WARN_ON_ONCE(1);
569 return -1;
570 }
571
572 if (reg >= 8) {
573 bytes[i++] = 0x41; /* REX.B prefix */
574 reg -= 8;
575 }
576
577 modrm |= 0xc0; /* Mod = 3 */
578 modrm += reg;
579
580 bytes[i++] = 0xff; /* opcode */
581 bytes[i++] = modrm;
582
583 return i;
584}
585
586static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
587{
588 u8 op = insn->opcode.bytes[0];
589 int i = 0;
590
591 /*
592 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
593 * tail-calls. Deal with them.
594 */
595 if (is_jcc32(insn)) {
596 bytes[i++] = op;
597 op = insn->opcode.bytes[1];
598 goto clang_jcc;
599 }
600
601 if (insn->length == 6)
602 bytes[i++] = 0x2e; /* CS-prefix */
603
604 switch (op) {
605 case CALL_INSN_OPCODE:
606 __text_gen_insn(bytes+i, op, addr+i,
607 __x86_indirect_call_thunk_array[reg],
608 CALL_INSN_SIZE);
609 i += CALL_INSN_SIZE;
610 break;
611
612 case JMP32_INSN_OPCODE:
613clang_jcc:
614 __text_gen_insn(bytes+i, op, addr+i,
615 __x86_indirect_jump_thunk_array[reg],
616 JMP32_INSN_SIZE);
617 i += JMP32_INSN_SIZE;
618 break;
619
620 default:
621 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
622 return -1;
623 }
624
625 WARN_ON_ONCE(i != insn->length);
626
627 return i;
628}
629
630/*
631 * Rewrite the compiler generated retpoline thunk calls.
632 *
633 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate
634 * indirect instructions, avoiding the extra indirection.
635 *
636 * For example, convert:
637 *
638 * CALL __x86_indirect_thunk_\reg
639 *
640 * into:
641 *
642 * CALL *%\reg
643 *
644 * It also tries to inline spectre_v2=retpoline,lfence when size permits.
645 */
646static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
647{
648 retpoline_thunk_t *target;
649 int reg, ret, i = 0;
650 u8 op, cc;
651
652 target = addr + insn->length + insn->immediate.value;
653 reg = target - __x86_indirect_thunk_array;
654
655 if (WARN_ON_ONCE(reg & ~0xf))
656 return -1;
657
658 /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */
659 BUG_ON(reg == 4);
660
661 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
662 !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
663 if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
664 return emit_call_track_retpoline(addr, insn, reg, bytes);
665
666 return -1;
667 }
668
669 op = insn->opcode.bytes[0];
670
671 /*
672 * Convert:
673 *
674 * Jcc.d32 __x86_indirect_thunk_\reg
675 *
676 * into:
677 *
678 * Jncc.d8 1f
679 * [ LFENCE ]
680 * JMP *%\reg
681 * [ NOP ]
682 * 1:
683 */
684 if (is_jcc32(insn)) {
685 cc = insn->opcode.bytes[1] & 0xf;
686 cc ^= 1; /* invert condition */
687
688 bytes[i++] = 0x70 + cc; /* Jcc.d8 */
689 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */
690
691 /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */
692 op = JMP32_INSN_OPCODE;
693 }
694
695 /*
696 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
697 */
698 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
699 bytes[i++] = 0x0f;
700 bytes[i++] = 0xae;
701 bytes[i++] = 0xe8; /* LFENCE */
702 }
703
704 ret = emit_indirect(op, reg, bytes + i);
705 if (ret < 0)
706 return ret;
707 i += ret;
708
709 /*
710 * The compiler is supposed to EMIT an INT3 after every unconditional
711 * JMP instruction due to AMD BTC. However, if the compiler is too old
712 * or SLS isn't enabled, we still need an INT3 after indirect JMPs
713 * even on Intel.
714 */
715 if (op == JMP32_INSN_OPCODE && i < insn->length)
716 bytes[i++] = INT3_INSN_OPCODE;
717
718 for (; i < insn->length;)
719 bytes[i++] = BYTES_NOP1;
720
721 return i;
722}
723
724/*
725 * Generated by 'objtool --retpoline'.
726 */
727void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
728{
729 s32 *s;
730
731 for (s = start; s < end; s++) {
732 void *addr = (void *)s + *s;
733 struct insn insn;
734 int len, ret;
735 u8 bytes[16];
736 u8 op1, op2;
737
738 ret = insn_decode_kernel(&insn, addr);
739 if (WARN_ON_ONCE(ret < 0))
740 continue;
741
742 op1 = insn.opcode.bytes[0];
743 op2 = insn.opcode.bytes[1];
744
745 switch (op1) {
746 case CALL_INSN_OPCODE:
747 case JMP32_INSN_OPCODE:
748 break;
749
750 case 0x0f: /* escape */
751 if (op2 >= 0x80 && op2 <= 0x8f)
752 break;
753 fallthrough;
754 default:
755 WARN_ON_ONCE(1);
756 continue;
757 }
758
759 DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS",
760 addr, addr, insn.length,
761 addr + insn.length + insn.immediate.value);
762
763 len = patch_retpoline(addr, &insn, bytes);
764 if (len == insn.length) {
765 optimize_nops(bytes, len);
766 DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr);
767 DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr);
768 text_poke_early(addr, bytes, len);
769 }
770 }
771}
772
773#ifdef CONFIG_RETHUNK
774
775/*
776 * Rewrite the compiler generated return thunk tail-calls.
777 *
778 * For example, convert:
779 *
780 * JMP __x86_return_thunk
781 *
782 * into:
783 *
784 * RET
785 */
786static int patch_return(void *addr, struct insn *insn, u8 *bytes)
787{
788 int i = 0;
789
790 /* Patch the custom return thunks... */
791 if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
792 i = JMP32_INSN_SIZE;
793 __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i);
794 } else {
795 /* ... or patch them out if not needed. */
796 bytes[i++] = RET_INSN_OPCODE;
797 }
798
799 for (; i < insn->length;)
800 bytes[i++] = INT3_INSN_OPCODE;
801 return i;
802}
803
804void __init_or_module noinline apply_returns(s32 *start, s32 *end)
805{
806 s32 *s;
807
808 if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
809 static_call_force_reinit();
810
811 for (s = start; s < end; s++) {
812 void *dest = NULL, *addr = (void *)s + *s;
813 struct insn insn;
814 int len, ret;
815 u8 bytes[16];
816 u8 op;
817
818 ret = insn_decode_kernel(&insn, addr);
819 if (WARN_ON_ONCE(ret < 0))
820 continue;
821
822 op = insn.opcode.bytes[0];
823 if (op == JMP32_INSN_OPCODE)
824 dest = addr + insn.length + insn.immediate.value;
825
826 if (__static_call_fixup(addr, op, dest) ||
827 WARN_ONCE(dest != &__x86_return_thunk,
828 "missing return thunk: %pS-%pS: %*ph",
829 addr, dest, 5, addr))
830 continue;
831
832 DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS",
833 addr, addr, insn.length,
834 addr + insn.length + insn.immediate.value);
835
836 len = patch_return(addr, &insn, bytes);
837 if (len == insn.length) {
838 DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr);
839 DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr);
840 text_poke_early(addr, bytes, len);
841 }
842 }
843}
844#else
845void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
846#endif /* CONFIG_RETHUNK */
847
848#else /* !CONFIG_RETPOLINE || !CONFIG_OBJTOOL */
849
850void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
851void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
852
853#endif /* CONFIG_RETPOLINE && CONFIG_OBJTOOL */
854
855#ifdef CONFIG_X86_KERNEL_IBT
856
857static void poison_cfi(void *addr);
858
859static void __init_or_module poison_endbr(void *addr, bool warn)
860{
861 u32 endbr, poison = gen_endbr_poison();
862
863 if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
864 return;
865
866 if (!is_endbr(endbr)) {
867 WARN_ON_ONCE(warn);
868 return;
869 }
870
871 DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr);
872
873 /*
874 * When we have IBT, the lack of ENDBR will trigger #CP
875 */
876 DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr);
877 DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr);
878 text_poke_early(addr, &poison, 4);
879}
880
881/*
882 * Generated by: objtool --ibt
883 *
884 * Seal the functions for indirect calls by clobbering the ENDBR instructions
885 * and the kCFI hash value.
886 */
887void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)
888{
889 s32 *s;
890
891 for (s = start; s < end; s++) {
892 void *addr = (void *)s + *s;
893
894 poison_endbr(addr, true);
895 if (IS_ENABLED(CONFIG_FINEIBT))
896 poison_cfi(addr - 16);
897 }
898}
899
900#else
901
902void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
903
904#endif /* CONFIG_X86_KERNEL_IBT */
905
906#ifdef CONFIG_FINEIBT
907#define __CFI_DEFAULT CFI_DEFAULT
908#elif defined(CONFIG_CFI_CLANG)
909#define __CFI_DEFAULT CFI_KCFI
910#else
911#define __CFI_DEFAULT CFI_OFF
912#endif
913
914enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
915
916#ifdef CONFIG_CFI_CLANG
917struct bpf_insn;
918
919/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
920extern unsigned int __bpf_prog_runX(const void *ctx,
921 const struct bpf_insn *insn);
922
923/*
924 * Force a reference to the external symbol so the compiler generates
925 * __kcfi_typid.
926 */
927__ADDRESSABLE(__bpf_prog_runX);
928
929/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
930asm (
931" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
932" .type cfi_bpf_hash,@object \n"
933" .globl cfi_bpf_hash \n"
934" .p2align 2, 0x0 \n"
935"cfi_bpf_hash: \n"
936" .long __kcfi_typeid___bpf_prog_runX \n"
937" .size cfi_bpf_hash, 4 \n"
938" .popsection \n"
939);
940
941/* Must match bpf_callback_t */
942extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
943
944__ADDRESSABLE(__bpf_callback_fn);
945
946/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
947asm (
948" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
949" .type cfi_bpf_subprog_hash,@object \n"
950" .globl cfi_bpf_subprog_hash \n"
951" .p2align 2, 0x0 \n"
952"cfi_bpf_subprog_hash: \n"
953" .long __kcfi_typeid___bpf_callback_fn \n"
954" .size cfi_bpf_subprog_hash, 4 \n"
955" .popsection \n"
956);
957
958u32 cfi_get_func_hash(void *func)
959{
960 u32 hash;
961
962 func -= cfi_get_offset();
963 switch (cfi_mode) {
964 case CFI_FINEIBT:
965 func += 7;
966 break;
967 case CFI_KCFI:
968 func += 1;
969 break;
970 default:
971 return 0;
972 }
973
974 if (get_kernel_nofault(hash, func))
975 return 0;
976
977 return hash;
978}
979#endif
980
981#ifdef CONFIG_FINEIBT
982
983static bool cfi_rand __ro_after_init = true;
984static u32 cfi_seed __ro_after_init;
985
986/*
987 * Re-hash the CFI hash with a boot-time seed while making sure the result is
988 * not a valid ENDBR instruction.
989 */
990static u32 cfi_rehash(u32 hash)
991{
992 hash ^= cfi_seed;
993 while (unlikely(is_endbr(hash) || is_endbr(-hash))) {
994 bool lsb = hash & 1;
995 hash >>= 1;
996 if (lsb)
997 hash ^= 0x80200003;
998 }
999 return hash;
1000}
1001
1002static __init int cfi_parse_cmdline(char *str)
1003{
1004 if (!str)
1005 return -EINVAL;
1006
1007 while (str) {
1008 char *next = strchr(str, ',');
1009 if (next) {
1010 *next = 0;
1011 next++;
1012 }
1013
1014 if (!strcmp(str, "auto")) {
1015 cfi_mode = CFI_DEFAULT;
1016 } else if (!strcmp(str, "off")) {
1017 cfi_mode = CFI_OFF;
1018 cfi_rand = false;
1019 } else if (!strcmp(str, "kcfi")) {
1020 cfi_mode = CFI_KCFI;
1021 } else if (!strcmp(str, "fineibt")) {
1022 cfi_mode = CFI_FINEIBT;
1023 } else if (!strcmp(str, "norand")) {
1024 cfi_rand = false;
1025 } else {
1026 pr_err("Ignoring unknown cfi option (%s).", str);
1027 }
1028
1029 str = next;
1030 }
1031
1032 return 0;
1033}
1034early_param("cfi", cfi_parse_cmdline);
1035
1036/*
1037 * kCFI FineIBT
1038 *
1039 * __cfi_\func: __cfi_\func:
1040 * movl $0x12345678,%eax // 5 endbr64 // 4
1041 * nop subl $0x12345678,%r10d // 7
1042 * nop jz 1f // 2
1043 * nop ud2 // 2
1044 * nop 1: nop // 1
1045 * nop
1046 * nop
1047 * nop
1048 * nop
1049 * nop
1050 * nop
1051 * nop
1052 *
1053 *
1054 * caller: caller:
1055 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
1056 * addl $-15(%r11),%r10d // 4 sub $16,%r11 // 4
1057 * je 1f // 2 nop4 // 4
1058 * ud2 // 2
1059 * 1: call __x86_indirect_thunk_r11 // 5 call *%r11; nop2; // 5
1060 *
1061 */
1062
1063asm( ".pushsection .rodata \n"
1064 "fineibt_preamble_start: \n"
1065 " endbr64 \n"
1066 " subl $0x12345678, %r10d \n"
1067 " je fineibt_preamble_end \n"
1068 " ud2 \n"
1069 " nop \n"
1070 "fineibt_preamble_end: \n"
1071 ".popsection\n"
1072);
1073
1074extern u8 fineibt_preamble_start[];
1075extern u8 fineibt_preamble_end[];
1076
1077#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1078#define fineibt_preamble_hash 7
1079
1080asm( ".pushsection .rodata \n"
1081 "fineibt_caller_start: \n"
1082 " movl $0x12345678, %r10d \n"
1083 " sub $16, %r11 \n"
1084 ASM_NOP4
1085 "fineibt_caller_end: \n"
1086 ".popsection \n"
1087);
1088
1089extern u8 fineibt_caller_start[];
1090extern u8 fineibt_caller_end[];
1091
1092#define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1093#define fineibt_caller_hash 2
1094
1095#define fineibt_caller_jmp (fineibt_caller_size - 2)
1096
1097static u32 decode_preamble_hash(void *addr)
1098{
1099 u8 *p = addr;
1100
1101 /* b8 78 56 34 12 mov $0x12345678,%eax */
1102 if (p[0] == 0xb8)
1103 return *(u32 *)(addr + 1);
1104
1105 return 0; /* invalid hash value */
1106}
1107
1108static u32 decode_caller_hash(void *addr)
1109{
1110 u8 *p = addr;
1111
1112 /* 41 ba 78 56 34 12 mov $0x12345678,%r10d */
1113 if (p[0] == 0x41 && p[1] == 0xba)
1114 return -*(u32 *)(addr + 2);
1115
1116 /* e8 0c 78 56 34 12 jmp.d8 +12 */
1117 if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
1118 return -*(u32 *)(addr + 2);
1119
1120 return 0; /* invalid hash value */
1121}
1122
1123/* .retpoline_sites */
1124static int cfi_disable_callers(s32 *start, s32 *end)
1125{
1126 /*
1127 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
1128 * in tact for later usage. Also see decode_caller_hash() and
1129 * cfi_rewrite_callers().
1130 */
1131 const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp };
1132 s32 *s;
1133
1134 for (s = start; s < end; s++) {
1135 void *addr = (void *)s + *s;
1136 u32 hash;
1137
1138 addr -= fineibt_caller_size;
1139 hash = decode_caller_hash(addr);
1140 if (!hash) /* nocfi callers */
1141 continue;
1142
1143 text_poke_early(addr, jmp, 2);
1144 }
1145
1146 return 0;
1147}
1148
1149static int cfi_enable_callers(s32 *start, s32 *end)
1150{
1151 /*
1152 * Re-enable kCFI, undo what cfi_disable_callers() did.
1153 */
1154 const u8 mov[] = { 0x41, 0xba };
1155 s32 *s;
1156
1157 for (s = start; s < end; s++) {
1158 void *addr = (void *)s + *s;
1159 u32 hash;
1160
1161 addr -= fineibt_caller_size;
1162 hash = decode_caller_hash(addr);
1163 if (!hash) /* nocfi callers */
1164 continue;
1165
1166 text_poke_early(addr, mov, 2);
1167 }
1168
1169 return 0;
1170}
1171
1172/* .cfi_sites */
1173static int cfi_rand_preamble(s32 *start, s32 *end)
1174{
1175 s32 *s;
1176
1177 for (s = start; s < end; s++) {
1178 void *addr = (void *)s + *s;
1179 u32 hash;
1180
1181 hash = decode_preamble_hash(addr);
1182 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
1183 addr, addr, 5, addr))
1184 return -EINVAL;
1185
1186 hash = cfi_rehash(hash);
1187 text_poke_early(addr + 1, &hash, 4);
1188 }
1189
1190 return 0;
1191}
1192
1193static int cfi_rewrite_preamble(s32 *start, s32 *end)
1194{
1195 s32 *s;
1196
1197 for (s = start; s < end; s++) {
1198 void *addr = (void *)s + *s;
1199 u32 hash;
1200
1201 hash = decode_preamble_hash(addr);
1202 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
1203 addr, addr, 5, addr))
1204 return -EINVAL;
1205
1206 text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
1207 WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);
1208 text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
1209 }
1210
1211 return 0;
1212}
1213
1214static void cfi_rewrite_endbr(s32 *start, s32 *end)
1215{
1216 s32 *s;
1217
1218 for (s = start; s < end; s++) {
1219 void *addr = (void *)s + *s;
1220
1221 poison_endbr(addr+16, false);
1222 }
1223}
1224
1225/* .retpoline_sites */
1226static int cfi_rand_callers(s32 *start, s32 *end)
1227{
1228 s32 *s;
1229
1230 for (s = start; s < end; s++) {
1231 void *addr = (void *)s + *s;
1232 u32 hash;
1233
1234 addr -= fineibt_caller_size;
1235 hash = decode_caller_hash(addr);
1236 if (hash) {
1237 hash = -cfi_rehash(hash);
1238 text_poke_early(addr + 2, &hash, 4);
1239 }
1240 }
1241
1242 return 0;
1243}
1244
1245static int cfi_rewrite_callers(s32 *start, s32 *end)
1246{
1247 s32 *s;
1248
1249 for (s = start; s < end; s++) {
1250 void *addr = (void *)s + *s;
1251 u32 hash;
1252
1253 addr -= fineibt_caller_size;
1254 hash = decode_caller_hash(addr);
1255 if (hash) {
1256 text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
1257 WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678);
1258 text_poke_early(addr + fineibt_caller_hash, &hash, 4);
1259 }
1260 /* rely on apply_retpolines() */
1261 }
1262
1263 return 0;
1264}
1265
1266static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
1267 s32 *start_cfi, s32 *end_cfi, bool builtin)
1268{
1269 int ret;
1270
1271 if (WARN_ONCE(fineibt_preamble_size != 16,
1272 "FineIBT preamble wrong size: %ld", fineibt_preamble_size))
1273 return;
1274
1275 if (cfi_mode == CFI_DEFAULT) {
1276 cfi_mode = CFI_KCFI;
1277 if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
1278 cfi_mode = CFI_FINEIBT;
1279 }
1280
1281 /*
1282 * Rewrite the callers to not use the __cfi_ stubs, such that we might
1283 * rewrite them. This disables all CFI. If this succeeds but any of the
1284 * later stages fails, we're without CFI.
1285 */
1286 ret = cfi_disable_callers(start_retpoline, end_retpoline);
1287 if (ret)
1288 goto err;
1289
1290 if (cfi_rand) {
1291 if (builtin) {
1292 cfi_seed = get_random_u32();
1293 cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
1294 cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
1295 }
1296
1297 ret = cfi_rand_preamble(start_cfi, end_cfi);
1298 if (ret)
1299 goto err;
1300
1301 ret = cfi_rand_callers(start_retpoline, end_retpoline);
1302 if (ret)
1303 goto err;
1304 }
1305
1306 switch (cfi_mode) {
1307 case CFI_OFF:
1308 if (builtin)
1309 pr_info("Disabling CFI\n");
1310 return;
1311
1312 case CFI_KCFI:
1313 ret = cfi_enable_callers(start_retpoline, end_retpoline);
1314 if (ret)
1315 goto err;
1316
1317 if (builtin)
1318 pr_info("Using kCFI\n");
1319 return;
1320
1321 case CFI_FINEIBT:
1322 /* place the FineIBT preamble at func()-16 */
1323 ret = cfi_rewrite_preamble(start_cfi, end_cfi);
1324 if (ret)
1325 goto err;
1326
1327 /* rewrite the callers to target func()-16 */
1328 ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
1329 if (ret)
1330 goto err;
1331
1332 /* now that nobody targets func()+0, remove ENDBR there */
1333 cfi_rewrite_endbr(start_cfi, end_cfi);
1334
1335 if (builtin)
1336 pr_info("Using FineIBT CFI\n");
1337 return;
1338
1339 default:
1340 break;
1341 }
1342
1343err:
1344 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n");
1345}
1346
1347static inline void poison_hash(void *addr)
1348{
1349 *(u32 *)addr = 0;
1350}
1351
1352static void poison_cfi(void *addr)
1353{
1354 switch (cfi_mode) {
1355 case CFI_FINEIBT:
1356 /*
1357 * __cfi_\func:
1358 * osp nopl (%rax)
1359 * subl $0, %r10d
1360 * jz 1f
1361 * ud2
1362 * 1: nop
1363 */
1364 poison_endbr(addr, false);
1365 poison_hash(addr + fineibt_preamble_hash);
1366 break;
1367
1368 case CFI_KCFI:
1369 /*
1370 * __cfi_\func:
1371 * movl $0, %eax
1372 * .skip 11, 0x90
1373 */
1374 poison_hash(addr + 1);
1375 break;
1376
1377 default:
1378 break;
1379 }
1380}
1381
1382#else
1383
1384static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
1385 s32 *start_cfi, s32 *end_cfi, bool builtin)
1386{
1387}
1388
1389#ifdef CONFIG_X86_KERNEL_IBT
1390static void poison_cfi(void *addr) { }
1391#endif
1392
1393#endif
1394
1395void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
1396 s32 *start_cfi, s32 *end_cfi)
1397{
1398 return __apply_fineibt(start_retpoline, end_retpoline,
1399 start_cfi, end_cfi,
1400 /* .builtin = */ false);
1401}
1402
1403#ifdef CONFIG_SMP
1404static void alternatives_smp_lock(const s32 *start, const s32 *end,
1405 u8 *text, u8 *text_end)
1406{
1407 const s32 *poff;
1408
1409 for (poff = start; poff < end; poff++) {
1410 u8 *ptr = (u8 *)poff + *poff;
1411
1412 if (!*poff || ptr < text || ptr >= text_end)
1413 continue;
1414 /* turn DS segment override prefix into lock prefix */
1415 if (*ptr == 0x3e)
1416 text_poke(ptr, ((unsigned char []){0xf0}), 1);
1417 }
1418}
1419
1420static void alternatives_smp_unlock(const s32 *start, const s32 *end,
1421 u8 *text, u8 *text_end)
1422{
1423 const s32 *poff;
1424
1425 for (poff = start; poff < end; poff++) {
1426 u8 *ptr = (u8 *)poff + *poff;
1427
1428 if (!*poff || ptr < text || ptr >= text_end)
1429 continue;
1430 /* turn lock prefix into DS segment override prefix */
1431 if (*ptr == 0xf0)
1432 text_poke(ptr, ((unsigned char []){0x3E}), 1);
1433 }
1434}
1435
1436struct smp_alt_module {
1437 /* what is this ??? */
1438 struct module *mod;
1439 char *name;
1440
1441 /* ptrs to lock prefixes */
1442 const s32 *locks;
1443 const s32 *locks_end;
1444
1445 /* .text segment, needed to avoid patching init code ;) */
1446 u8 *text;
1447 u8 *text_end;
1448
1449 struct list_head next;
1450};
1451static LIST_HEAD(smp_alt_modules);
1452static bool uniproc_patched = false; /* protected by text_mutex */
1453
1454void __init_or_module alternatives_smp_module_add(struct module *mod,
1455 char *name,
1456 void *locks, void *locks_end,
1457 void *text, void *text_end)
1458{
1459 struct smp_alt_module *smp;
1460
1461 mutex_lock(&text_mutex);
1462 if (!uniproc_patched)
1463 goto unlock;
1464
1465 if (num_possible_cpus() == 1)
1466 /* Don't bother remembering, we'll never have to undo it. */
1467 goto smp_unlock;
1468
1469 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
1470 if (NULL == smp)
1471 /* we'll run the (safe but slow) SMP code then ... */
1472 goto unlock;
1473
1474 smp->mod = mod;
1475 smp->name = name;
1476 smp->locks = locks;
1477 smp->locks_end = locks_end;
1478 smp->text = text;
1479 smp->text_end = text_end;
1480 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n",
1481 smp->locks, smp->locks_end,
1482 smp->text, smp->text_end, smp->name);
1483
1484 list_add_tail(&smp->next, &smp_alt_modules);
1485smp_unlock:
1486 alternatives_smp_unlock(locks, locks_end, text, text_end);
1487unlock:
1488 mutex_unlock(&text_mutex);
1489}
1490
1491void __init_or_module alternatives_smp_module_del(struct module *mod)
1492{
1493 struct smp_alt_module *item;
1494
1495 mutex_lock(&text_mutex);
1496 list_for_each_entry(item, &smp_alt_modules, next) {
1497 if (mod != item->mod)
1498 continue;
1499 list_del(&item->next);
1500 kfree(item);
1501 break;
1502 }
1503 mutex_unlock(&text_mutex);
1504}
1505
1506void alternatives_enable_smp(void)
1507{
1508 struct smp_alt_module *mod;
1509
1510 /* Why bother if there are no other CPUs? */
1511 BUG_ON(num_possible_cpus() == 1);
1512
1513 mutex_lock(&text_mutex);
1514
1515 if (uniproc_patched) {
1516 pr_info("switching to SMP code\n");
1517 BUG_ON(num_online_cpus() != 1);
1518 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
1519 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
1520 list_for_each_entry(mod, &smp_alt_modules, next)
1521 alternatives_smp_lock(mod->locks, mod->locks_end,
1522 mod->text, mod->text_end);
1523 uniproc_patched = false;
1524 }
1525 mutex_unlock(&text_mutex);
1526}
1527
1528/*
1529 * Return 1 if the address range is reserved for SMP-alternatives.
1530 * Must hold text_mutex.
1531 */
1532int alternatives_text_reserved(void *start, void *end)
1533{
1534 struct smp_alt_module *mod;
1535 const s32 *poff;
1536 u8 *text_start = start;
1537 u8 *text_end = end;
1538
1539 lockdep_assert_held(&text_mutex);
1540
1541 list_for_each_entry(mod, &smp_alt_modules, next) {
1542 if (mod->text > text_end || mod->text_end < text_start)
1543 continue;
1544 for (poff = mod->locks; poff < mod->locks_end; poff++) {
1545 const u8 *ptr = (const u8 *)poff + *poff;
1546
1547 if (text_start <= ptr && text_end > ptr)
1548 return 1;
1549 }
1550 }
1551
1552 return 0;
1553}
1554#endif /* CONFIG_SMP */
1555
1556/*
1557 * Self-test for the INT3 based CALL emulation code.
1558 *
1559 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
1560 * properly and that there is a stack gap between the INT3 frame and the
1561 * previous context. Without this gap doing a virtual PUSH on the interrupted
1562 * stack would corrupt the INT3 IRET frame.
1563 *
1564 * See entry_{32,64}.S for more details.
1565 */
1566
1567/*
1568 * We define the int3_magic() function in assembly to control the calling
1569 * convention such that we can 'call' it from assembly.
1570 */
1571
1572extern void int3_magic(unsigned int *ptr); /* defined in asm */
1573
1574asm (
1575" .pushsection .init.text, \"ax\", @progbits\n"
1576" .type int3_magic, @function\n"
1577"int3_magic:\n"
1578 ANNOTATE_NOENDBR
1579" movl $1, (%" _ASM_ARG1 ")\n"
1580 ASM_RET
1581" .size int3_magic, .-int3_magic\n"
1582" .popsection\n"
1583);
1584
1585extern void int3_selftest_ip(void); /* defined in asm below */
1586
1587static int __init
1588int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
1589{
1590 unsigned long selftest = (unsigned long)&int3_selftest_ip;
1591 struct die_args *args = data;
1592 struct pt_regs *regs = args->regs;
1593
1594 OPTIMIZER_HIDE_VAR(selftest);
1595
1596 if (!regs || user_mode(regs))
1597 return NOTIFY_DONE;
1598
1599 if (val != DIE_INT3)
1600 return NOTIFY_DONE;
1601
1602 if (regs->ip - INT3_INSN_SIZE != selftest)
1603 return NOTIFY_DONE;
1604
1605 int3_emulate_call(regs, (unsigned long)&int3_magic);
1606 return NOTIFY_STOP;
1607}
1608
1609/* Must be noinline to ensure uniqueness of int3_selftest_ip. */
1610static noinline void __init int3_selftest(void)
1611{
1612 static __initdata struct notifier_block int3_exception_nb = {
1613 .notifier_call = int3_exception_notify,
1614 .priority = INT_MAX-1, /* last */
1615 };
1616 unsigned int val = 0;
1617
1618 BUG_ON(register_die_notifier(&int3_exception_nb));
1619
1620 /*
1621 * Basically: int3_magic(&val); but really complicated :-)
1622 *
1623 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb
1624 * notifier above will emulate CALL for us.
1625 */
1626 asm volatile ("int3_selftest_ip:\n\t"
1627 ANNOTATE_NOENDBR
1628 " int3; nop; nop; nop; nop\n\t"
1629 : ASM_CALL_CONSTRAINT
1630 : __ASM_SEL_RAW(a, D) (&val)
1631 : "memory");
1632
1633 BUG_ON(val != 1);
1634
1635 unregister_die_notifier(&int3_exception_nb);
1636}
1637
1638static __initdata int __alt_reloc_selftest_addr;
1639
1640extern void __init __alt_reloc_selftest(void *arg);
1641__visible noinline void __init __alt_reloc_selftest(void *arg)
1642{
1643 WARN_ON(arg != &__alt_reloc_selftest_addr);
1644}
1645
1646static noinline void __init alt_reloc_selftest(void)
1647{
1648 /*
1649 * Tests apply_relocation().
1650 *
1651 * This has a relative immediate (CALL) in a place other than the first
1652 * instruction and additionally on x86_64 we get a RIP-relative LEA:
1653 *
1654 * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c
1655 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4
1656 *
1657 * Getting this wrong will either crash and burn or tickle the WARN
1658 * above.
1659 */
1660 asm_inline volatile (
1661 ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS)
1662 : /* output */
1663 : [mem] "m" (__alt_reloc_selftest_addr)
1664 : _ASM_ARG1
1665 );
1666}
1667
1668void __init alternative_instructions(void)
1669{
1670 int3_selftest();
1671
1672 /*
1673 * The patching is not fully atomic, so try to avoid local
1674 * interruptions that might execute the to be patched code.
1675 * Other CPUs are not running.
1676 */
1677 stop_nmi();
1678
1679 /*
1680 * Don't stop machine check exceptions while patching.
1681 * MCEs only happen when something got corrupted and in this
1682 * case we must do something about the corruption.
1683 * Ignoring it is worse than an unlikely patching race.
1684 * Also machine checks tend to be broadcast and if one CPU
1685 * goes into machine check the others follow quickly, so we don't
1686 * expect a machine check to cause undue problems during to code
1687 * patching.
1688 */
1689
1690 /*
1691 * Make sure to set (artificial) features depending on used paravirt
1692 * functions which can later influence alternative patching.
1693 */
1694 paravirt_set_cap();
1695
1696 __apply_fineibt(__retpoline_sites, __retpoline_sites_end,
1697 __cfi_sites, __cfi_sites_end, true);
1698
1699 /*
1700 * Rewrite the retpolines, must be done before alternatives since
1701 * those can rewrite the retpoline thunks.
1702 */
1703 apply_retpolines(__retpoline_sites, __retpoline_sites_end);
1704 apply_returns(__return_sites, __return_sites_end);
1705
1706 apply_alternatives(__alt_instructions, __alt_instructions_end);
1707
1708 /*
1709 * Now all calls are established. Apply the call thunks if
1710 * required.
1711 */
1712 callthunks_patch_builtin_calls();
1713
1714 /*
1715 * Seal all functions that do not have their address taken.
1716 */
1717 apply_seal_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
1718
1719#ifdef CONFIG_SMP
1720 /* Patch to UP if other cpus not imminent. */
1721 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
1722 uniproc_patched = true;
1723 alternatives_smp_module_add(NULL, "core kernel",
1724 __smp_locks, __smp_locks_end,
1725 _text, _etext);
1726 }
1727
1728 if (!uniproc_patched || num_possible_cpus() == 1) {
1729 free_init_pages("SMP alternatives",
1730 (unsigned long)__smp_locks,
1731 (unsigned long)__smp_locks_end);
1732 }
1733#endif
1734
1735 restart_nmi();
1736 alternatives_patched = 1;
1737
1738 alt_reloc_selftest();
1739}
1740
1741/**
1742 * text_poke_early - Update instructions on a live kernel at boot time
1743 * @addr: address to modify
1744 * @opcode: source of the copy
1745 * @len: length to copy
1746 *
1747 * When you use this code to patch more than one byte of an instruction
1748 * you need to make sure that other CPUs cannot execute this code in parallel.
1749 * Also no thread must be currently preempted in the middle of these
1750 * instructions. And on the local CPU you need to be protected against NMI or
1751 * MCE handlers seeing an inconsistent instruction while you patch.
1752 */
1753void __init_or_module text_poke_early(void *addr, const void *opcode,
1754 size_t len)
1755{
1756 unsigned long flags;
1757
1758 if (boot_cpu_has(X86_FEATURE_NX) &&
1759 is_module_text_address((unsigned long)addr)) {
1760 /*
1761 * Modules text is marked initially as non-executable, so the
1762 * code cannot be running and speculative code-fetches are
1763 * prevented. Just change the code.
1764 */
1765 memcpy(addr, opcode, len);
1766 } else {
1767 local_irq_save(flags);
1768 memcpy(addr, opcode, len);
1769 sync_core();
1770 local_irq_restore(flags);
1771
1772 /*
1773 * Could also do a CLFLUSH here to speed up CPU recovery; but
1774 * that causes hangs on some VIA CPUs.
1775 */
1776 }
1777}
1778
1779typedef struct {
1780 struct mm_struct *mm;
1781} temp_mm_state_t;
1782
1783/*
1784 * Using a temporary mm allows to set temporary mappings that are not accessible
1785 * by other CPUs. Such mappings are needed to perform sensitive memory writes
1786 * that override the kernel memory protections (e.g., W^X), without exposing the
1787 * temporary page-table mappings that are required for these write operations to
1788 * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
1789 * mapping is torn down.
1790 *
1791 * Context: The temporary mm needs to be used exclusively by a single core. To
1792 * harden security IRQs must be disabled while the temporary mm is
1793 * loaded, thereby preventing interrupt handler bugs from overriding
1794 * the kernel memory protection.
1795 */
1796static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
1797{
1798 temp_mm_state_t temp_state;
1799
1800 lockdep_assert_irqs_disabled();
1801
1802 /*
1803 * Make sure not to be in TLB lazy mode, as otherwise we'll end up
1804 * with a stale address space WITHOUT being in lazy mode after
1805 * restoring the previous mm.
1806 */
1807 if (this_cpu_read(cpu_tlbstate_shared.is_lazy))
1808 leave_mm(smp_processor_id());
1809
1810 temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
1811 switch_mm_irqs_off(NULL, mm, current);
1812
1813 /*
1814 * If breakpoints are enabled, disable them while the temporary mm is
1815 * used. Userspace might set up watchpoints on addresses that are used
1816 * in the temporary mm, which would lead to wrong signals being sent or
1817 * crashes.
1818 *
1819 * Note that breakpoints are not disabled selectively, which also causes
1820 * kernel breakpoints (e.g., perf's) to be disabled. This might be
1821 * undesirable, but still seems reasonable as the code that runs in the
1822 * temporary mm should be short.
1823 */
1824 if (hw_breakpoint_active())
1825 hw_breakpoint_disable();
1826
1827 return temp_state;
1828}
1829
1830static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
1831{
1832 lockdep_assert_irqs_disabled();
1833 switch_mm_irqs_off(NULL, prev_state.mm, current);
1834
1835 /*
1836 * Restore the breakpoints if they were disabled before the temporary mm
1837 * was loaded.
1838 */
1839 if (hw_breakpoint_active())
1840 hw_breakpoint_restore();
1841}
1842
1843__ro_after_init struct mm_struct *poking_mm;
1844__ro_after_init unsigned long poking_addr;
1845
1846static void text_poke_memcpy(void *dst, const void *src, size_t len)
1847{
1848 memcpy(dst, src, len);
1849}
1850
1851static void text_poke_memset(void *dst, const void *src, size_t len)
1852{
1853 int c = *(const int *)src;
1854
1855 memset(dst, c, len);
1856}
1857
1858typedef void text_poke_f(void *dst, const void *src, size_t len);
1859
1860static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
1861{
1862 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
1863 struct page *pages[2] = {NULL};
1864 temp_mm_state_t prev;
1865 unsigned long flags;
1866 pte_t pte, *ptep;
1867 spinlock_t *ptl;
1868 pgprot_t pgprot;
1869
1870 /*
1871 * While boot memory allocator is running we cannot use struct pages as
1872 * they are not yet initialized. There is no way to recover.
1873 */
1874 BUG_ON(!after_bootmem);
1875
1876 if (!core_kernel_text((unsigned long)addr)) {
1877 pages[0] = vmalloc_to_page(addr);
1878 if (cross_page_boundary)
1879 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
1880 } else {
1881 pages[0] = virt_to_page(addr);
1882 WARN_ON(!PageReserved(pages[0]));
1883 if (cross_page_boundary)
1884 pages[1] = virt_to_page(addr + PAGE_SIZE);
1885 }
1886 /*
1887 * If something went wrong, crash and burn since recovery paths are not
1888 * implemented.
1889 */
1890 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
1891
1892 /*
1893 * Map the page without the global bit, as TLB flushing is done with
1894 * flush_tlb_mm_range(), which is intended for non-global PTEs.
1895 */
1896 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
1897
1898 /*
1899 * The lock is not really needed, but this allows to avoid open-coding.
1900 */
1901 ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
1902
1903 /*
1904 * This must not fail; preallocated in poking_init().
1905 */
1906 VM_BUG_ON(!ptep);
1907
1908 local_irq_save(flags);
1909
1910 pte = mk_pte(pages[0], pgprot);
1911 set_pte_at(poking_mm, poking_addr, ptep, pte);
1912
1913 if (cross_page_boundary) {
1914 pte = mk_pte(pages[1], pgprot);
1915 set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte);
1916 }
1917
1918 /*
1919 * Loading the temporary mm behaves as a compiler barrier, which
1920 * guarantees that the PTE will be set at the time memcpy() is done.
1921 */
1922 prev = use_temporary_mm(poking_mm);
1923
1924 kasan_disable_current();
1925 func((u8 *)poking_addr + offset_in_page(addr), src, len);
1926 kasan_enable_current();
1927
1928 /*
1929 * Ensure that the PTE is only cleared after the instructions of memcpy
1930 * were issued by using a compiler barrier.
1931 */
1932 barrier();
1933
1934 pte_clear(poking_mm, poking_addr, ptep);
1935 if (cross_page_boundary)
1936 pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1);
1937
1938 /*
1939 * Loading the previous page-table hierarchy requires a serializing
1940 * instruction that already allows the core to see the updated version.
1941 * Xen-PV is assumed to serialize execution in a similar manner.
1942 */
1943 unuse_temporary_mm(prev);
1944
1945 /*
1946 * Flushing the TLB might involve IPIs, which would require enabled
1947 * IRQs, but not if the mm is not used, as it is in this point.
1948 */
1949 flush_tlb_mm_range(poking_mm, poking_addr, poking_addr +
1950 (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
1951 PAGE_SHIFT, false);
1952
1953 if (func == text_poke_memcpy) {
1954 /*
1955 * If the text does not match what we just wrote then something is
1956 * fundamentally screwy; there's nothing we can really do about that.
1957 */
1958 BUG_ON(memcmp(addr, src, len));
1959 }
1960
1961 local_irq_restore(flags);
1962 pte_unmap_unlock(ptep, ptl);
1963 return addr;
1964}
1965
1966/**
1967 * text_poke - Update instructions on a live kernel
1968 * @addr: address to modify
1969 * @opcode: source of the copy
1970 * @len: length to copy
1971 *
1972 * Only atomic text poke/set should be allowed when not doing early patching.
1973 * It means the size must be writable atomically and the address must be aligned
1974 * in a way that permits an atomic write. It also makes sure we fit on a single
1975 * page.
1976 *
1977 * Note that the caller must ensure that if the modified code is part of a
1978 * module, the module would not be removed during poking. This can be achieved
1979 * by registering a module notifier, and ordering module removal and patching
1980 * through a mutex.
1981 */
1982void *text_poke(void *addr, const void *opcode, size_t len)
1983{
1984 lockdep_assert_held(&text_mutex);
1985
1986 return __text_poke(text_poke_memcpy, addr, opcode, len);
1987}
1988
1989/**
1990 * text_poke_kgdb - Update instructions on a live kernel by kgdb
1991 * @addr: address to modify
1992 * @opcode: source of the copy
1993 * @len: length to copy
1994 *
1995 * Only atomic text poke/set should be allowed when not doing early patching.
1996 * It means the size must be writable atomically and the address must be aligned
1997 * in a way that permits an atomic write. It also makes sure we fit on a single
1998 * page.
1999 *
2000 * Context: should only be used by kgdb, which ensures no other core is running,
2001 * despite the fact it does not hold the text_mutex.
2002 */
2003void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
2004{
2005 return __text_poke(text_poke_memcpy, addr, opcode, len);
2006}
2007
2008void *text_poke_copy_locked(void *addr, const void *opcode, size_t len,
2009 bool core_ok)
2010{
2011 unsigned long start = (unsigned long)addr;
2012 size_t patched = 0;
2013
2014 if (WARN_ON_ONCE(!core_ok && core_kernel_text(start)))
2015 return NULL;
2016
2017 while (patched < len) {
2018 unsigned long ptr = start + patched;
2019 size_t s;
2020
2021 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
2022
2023 __text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s);
2024 patched += s;
2025 }
2026 return addr;
2027}
2028
2029/**
2030 * text_poke_copy - Copy instructions into (an unused part of) RX memory
2031 * @addr: address to modify
2032 * @opcode: source of the copy
2033 * @len: length to copy, could be more than 2x PAGE_SIZE
2034 *
2035 * Not safe against concurrent execution; useful for JITs to dump
2036 * new code blocks into unused regions of RX memory. Can be used in
2037 * conjunction with synchronize_rcu_tasks() to wait for existing
2038 * execution to quiesce after having made sure no existing functions
2039 * pointers are live.
2040 */
2041void *text_poke_copy(void *addr, const void *opcode, size_t len)
2042{
2043 mutex_lock(&text_mutex);
2044 addr = text_poke_copy_locked(addr, opcode, len, false);
2045 mutex_unlock(&text_mutex);
2046 return addr;
2047}
2048
2049/**
2050 * text_poke_set - memset into (an unused part of) RX memory
2051 * @addr: address to modify
2052 * @c: the byte to fill the area with
2053 * @len: length to copy, could be more than 2x PAGE_SIZE
2054 *
2055 * This is useful to overwrite unused regions of RX memory with illegal
2056 * instructions.
2057 */
2058void *text_poke_set(void *addr, int c, size_t len)
2059{
2060 unsigned long start = (unsigned long)addr;
2061 size_t patched = 0;
2062
2063 if (WARN_ON_ONCE(core_kernel_text(start)))
2064 return NULL;
2065
2066 mutex_lock(&text_mutex);
2067 while (patched < len) {
2068 unsigned long ptr = start + patched;
2069 size_t s;
2070
2071 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
2072
2073 __text_poke(text_poke_memset, (void *)ptr, (void *)&c, s);
2074 patched += s;
2075 }
2076 mutex_unlock(&text_mutex);
2077 return addr;
2078}
2079
2080static void do_sync_core(void *info)
2081{
2082 sync_core();
2083}
2084
2085void text_poke_sync(void)
2086{
2087 on_each_cpu(do_sync_core, NULL, 1);
2088}
2089
2090/*
2091 * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of
2092 * this thing. When len == 6 everything is prefixed with 0x0f and we map
2093 * opcode to Jcc.d8, using len to distinguish.
2094 */
2095struct text_poke_loc {
2096 /* addr := _stext + rel_addr */
2097 s32 rel_addr;
2098 s32 disp;
2099 u8 len;
2100 u8 opcode;
2101 const u8 text[POKE_MAX_OPCODE_SIZE];
2102 /* see text_poke_bp_batch() */
2103 u8 old;
2104};
2105
2106struct bp_patching_desc {
2107 struct text_poke_loc *vec;
2108 int nr_entries;
2109 atomic_t refs;
2110};
2111
2112static struct bp_patching_desc bp_desc;
2113
2114static __always_inline
2115struct bp_patching_desc *try_get_desc(void)
2116{
2117 struct bp_patching_desc *desc = &bp_desc;
2118
2119 if (!raw_atomic_inc_not_zero(&desc->refs))
2120 return NULL;
2121
2122 return desc;
2123}
2124
2125static __always_inline void put_desc(void)
2126{
2127 struct bp_patching_desc *desc = &bp_desc;
2128
2129 smp_mb__before_atomic();
2130 raw_atomic_dec(&desc->refs);
2131}
2132
2133static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
2134{
2135 return _stext + tp->rel_addr;
2136}
2137
2138static __always_inline int patch_cmp(const void *key, const void *elt)
2139{
2140 struct text_poke_loc *tp = (struct text_poke_loc *) elt;
2141
2142 if (key < text_poke_addr(tp))
2143 return -1;
2144 if (key > text_poke_addr(tp))
2145 return 1;
2146 return 0;
2147}
2148
2149noinstr int poke_int3_handler(struct pt_regs *regs)
2150{
2151 struct bp_patching_desc *desc;
2152 struct text_poke_loc *tp;
2153 int ret = 0;
2154 void *ip;
2155
2156 if (user_mode(regs))
2157 return 0;
2158
2159 /*
2160 * Having observed our INT3 instruction, we now must observe
2161 * bp_desc with non-zero refcount:
2162 *
2163 * bp_desc.refs = 1 INT3
2164 * WMB RMB
2165 * write INT3 if (bp_desc.refs != 0)
2166 */
2167 smp_rmb();
2168
2169 desc = try_get_desc();
2170 if (!desc)
2171 return 0;
2172
2173 /*
2174 * Discount the INT3. See text_poke_bp_batch().
2175 */
2176 ip = (void *) regs->ip - INT3_INSN_SIZE;
2177
2178 /*
2179 * Skip the binary search if there is a single member in the vector.
2180 */
2181 if (unlikely(desc->nr_entries > 1)) {
2182 tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,
2183 sizeof(struct text_poke_loc),
2184 patch_cmp);
2185 if (!tp)
2186 goto out_put;
2187 } else {
2188 tp = desc->vec;
2189 if (text_poke_addr(tp) != ip)
2190 goto out_put;
2191 }
2192
2193 ip += tp->len;
2194
2195 switch (tp->opcode) {
2196 case INT3_INSN_OPCODE:
2197 /*
2198 * Someone poked an explicit INT3, they'll want to handle it,
2199 * do not consume.
2200 */
2201 goto out_put;
2202
2203 case RET_INSN_OPCODE:
2204 int3_emulate_ret(regs);
2205 break;
2206
2207 case CALL_INSN_OPCODE:
2208 int3_emulate_call(regs, (long)ip + tp->disp);
2209 break;
2210
2211 case JMP32_INSN_OPCODE:
2212 case JMP8_INSN_OPCODE:
2213 int3_emulate_jmp(regs, (long)ip + tp->disp);
2214 break;
2215
2216 case 0x70 ... 0x7f: /* Jcc */
2217 int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp);
2218 break;
2219
2220 default:
2221 BUG();
2222 }
2223
2224 ret = 1;
2225
2226out_put:
2227 put_desc();
2228 return ret;
2229}
2230
2231#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
2232static struct text_poke_loc tp_vec[TP_VEC_MAX];
2233static int tp_vec_nr;
2234
2235/**
2236 * text_poke_bp_batch() -- update instructions on live kernel on SMP
2237 * @tp: vector of instructions to patch
2238 * @nr_entries: number of entries in the vector
2239 *
2240 * Modify multi-byte instruction by using int3 breakpoint on SMP.
2241 * We completely avoid stop_machine() here, and achieve the
2242 * synchronization using int3 breakpoint.
2243 *
2244 * The way it is done:
2245 * - For each entry in the vector:
2246 * - add a int3 trap to the address that will be patched
2247 * - sync cores
2248 * - For each entry in the vector:
2249 * - update all but the first byte of the patched range
2250 * - sync cores
2251 * - For each entry in the vector:
2252 * - replace the first byte (int3) by the first byte of
2253 * replacing opcode
2254 * - sync cores
2255 */
2256static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
2257{
2258 unsigned char int3 = INT3_INSN_OPCODE;
2259 unsigned int i;
2260 int do_sync;
2261
2262 lockdep_assert_held(&text_mutex);
2263
2264 bp_desc.vec = tp;
2265 bp_desc.nr_entries = nr_entries;
2266
2267 /*
2268 * Corresponds to the implicit memory barrier in try_get_desc() to
2269 * ensure reading a non-zero refcount provides up to date bp_desc data.
2270 */
2271 atomic_set_release(&bp_desc.refs, 1);
2272
2273 /*
2274 * Function tracing can enable thousands of places that need to be
2275 * updated. This can take quite some time, and with full kernel debugging
2276 * enabled, this could cause the softlockup watchdog to trigger.
2277 * This function gets called every 256 entries added to be patched.
2278 * Call cond_resched() here to make sure that other tasks can get scheduled
2279 * while processing all the functions being patched.
2280 */
2281 cond_resched();
2282
2283 /*
2284 * Corresponding read barrier in int3 notifier for making sure the
2285 * nr_entries and handler are correctly ordered wrt. patching.
2286 */
2287 smp_wmb();
2288
2289 /*
2290 * First step: add a int3 trap to the address that will be patched.
2291 */
2292 for (i = 0; i < nr_entries; i++) {
2293 tp[i].old = *(u8 *)text_poke_addr(&tp[i]);
2294 text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE);
2295 }
2296
2297 text_poke_sync();
2298
2299 /*
2300 * Second step: update all but the first byte of the patched range.
2301 */
2302 for (do_sync = 0, i = 0; i < nr_entries; i++) {
2303 u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, };
2304 u8 _new[POKE_MAX_OPCODE_SIZE+1];
2305 const u8 *new = tp[i].text;
2306 int len = tp[i].len;
2307
2308 if (len - INT3_INSN_SIZE > 0) {
2309 memcpy(old + INT3_INSN_SIZE,
2310 text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
2311 len - INT3_INSN_SIZE);
2312
2313 if (len == 6) {
2314 _new[0] = 0x0f;
2315 memcpy(_new + 1, new, 5);
2316 new = _new;
2317 }
2318
2319 text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
2320 new + INT3_INSN_SIZE,
2321 len - INT3_INSN_SIZE);
2322
2323 do_sync++;
2324 }
2325
2326 /*
2327 * Emit a perf event to record the text poke, primarily to
2328 * support Intel PT decoding which must walk the executable code
2329 * to reconstruct the trace. The flow up to here is:
2330 * - write INT3 byte
2331 * - IPI-SYNC
2332 * - write instruction tail
2333 * At this point the actual control flow will be through the
2334 * INT3 and handler and not hit the old or new instruction.
2335 * Intel PT outputs FUP/TIP packets for the INT3, so the flow
2336 * can still be decoded. Subsequently:
2337 * - emit RECORD_TEXT_POKE with the new instruction
2338 * - IPI-SYNC
2339 * - write first byte
2340 * - IPI-SYNC
2341 * So before the text poke event timestamp, the decoder will see
2342 * either the old instruction flow or FUP/TIP of INT3. After the
2343 * text poke event timestamp, the decoder will see either the
2344 * new instruction flow or FUP/TIP of INT3. Thus decoders can
2345 * use the timestamp as the point at which to modify the
2346 * executable code.
2347 * The old instruction is recorded so that the event can be
2348 * processed forwards or backwards.
2349 */
2350 perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len);
2351 }
2352
2353 if (do_sync) {
2354 /*
2355 * According to Intel, this core syncing is very likely
2356 * not necessary and we'd be safe even without it. But
2357 * better safe than sorry (plus there's not only Intel).
2358 */
2359 text_poke_sync();
2360 }
2361
2362 /*
2363 * Third step: replace the first byte (int3) by the first byte of
2364 * replacing opcode.
2365 */
2366 for (do_sync = 0, i = 0; i < nr_entries; i++) {
2367 u8 byte = tp[i].text[0];
2368
2369 if (tp[i].len == 6)
2370 byte = 0x0f;
2371
2372 if (byte == INT3_INSN_OPCODE)
2373 continue;
2374
2375 text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE);
2376 do_sync++;
2377 }
2378
2379 if (do_sync)
2380 text_poke_sync();
2381
2382 /*
2383 * Remove and wait for refs to be zero.
2384 */
2385 if (!atomic_dec_and_test(&bp_desc.refs))
2386 atomic_cond_read_acquire(&bp_desc.refs, !VAL);
2387}
2388
2389static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
2390 const void *opcode, size_t len, const void *emulate)
2391{
2392 struct insn insn;
2393 int ret, i = 0;
2394
2395 if (len == 6)
2396 i = 1;
2397 memcpy((void *)tp->text, opcode+i, len-i);
2398 if (!emulate)
2399 emulate = opcode;
2400
2401 ret = insn_decode_kernel(&insn, emulate);
2402 BUG_ON(ret < 0);
2403
2404 tp->rel_addr = addr - (void *)_stext;
2405 tp->len = len;
2406 tp->opcode = insn.opcode.bytes[0];
2407
2408 if (is_jcc32(&insn)) {
2409 /*
2410 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish.
2411 */
2412 tp->opcode = insn.opcode.bytes[1] - 0x10;
2413 }
2414
2415 switch (tp->opcode) {
2416 case RET_INSN_OPCODE:
2417 case JMP32_INSN_OPCODE:
2418 case JMP8_INSN_OPCODE:
2419 /*
2420 * Control flow instructions without implied execution of the
2421 * next instruction can be padded with INT3.
2422 */
2423 for (i = insn.length; i < len; i++)
2424 BUG_ON(tp->text[i] != INT3_INSN_OPCODE);
2425 break;
2426
2427 default:
2428 BUG_ON(len != insn.length);
2429 }
2430
2431 switch (tp->opcode) {
2432 case INT3_INSN_OPCODE:
2433 case RET_INSN_OPCODE:
2434 break;
2435
2436 case CALL_INSN_OPCODE:
2437 case JMP32_INSN_OPCODE:
2438 case JMP8_INSN_OPCODE:
2439 case 0x70 ... 0x7f: /* Jcc */
2440 tp->disp = insn.immediate.value;
2441 break;
2442
2443 default: /* assume NOP */
2444 switch (len) {
2445 case 2: /* NOP2 -- emulate as JMP8+0 */
2446 BUG_ON(memcmp(emulate, x86_nops[len], len));
2447 tp->opcode = JMP8_INSN_OPCODE;
2448 tp->disp = 0;
2449 break;
2450
2451 case 5: /* NOP5 -- emulate as JMP32+0 */
2452 BUG_ON(memcmp(emulate, x86_nops[len], len));
2453 tp->opcode = JMP32_INSN_OPCODE;
2454 tp->disp = 0;
2455 break;
2456
2457 default: /* unknown instruction */
2458 BUG();
2459 }
2460 break;
2461 }
2462}
2463
2464/*
2465 * We hard rely on the tp_vec being ordered; ensure this is so by flushing
2466 * early if needed.
2467 */
2468static bool tp_order_fail(void *addr)
2469{
2470 struct text_poke_loc *tp;
2471
2472 if (!tp_vec_nr)
2473 return false;
2474
2475 if (!addr) /* force */
2476 return true;
2477
2478 tp = &tp_vec[tp_vec_nr - 1];
2479 if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr)
2480 return true;
2481
2482 return false;
2483}
2484
2485static void text_poke_flush(void *addr)
2486{
2487 if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) {
2488 text_poke_bp_batch(tp_vec, tp_vec_nr);
2489 tp_vec_nr = 0;
2490 }
2491}
2492
2493void text_poke_finish(void)
2494{
2495 text_poke_flush(NULL);
2496}
2497
2498void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate)
2499{
2500 struct text_poke_loc *tp;
2501
2502 text_poke_flush(addr);
2503
2504 tp = &tp_vec[tp_vec_nr++];
2505 text_poke_loc_init(tp, addr, opcode, len, emulate);
2506}
2507
2508/**
2509 * text_poke_bp() -- update instructions on live kernel on SMP
2510 * @addr: address to patch
2511 * @opcode: opcode of new instruction
2512 * @len: length to copy
2513 * @emulate: instruction to be emulated
2514 *
2515 * Update a single instruction with the vector in the stack, avoiding
2516 * dynamically allocated memory. This function should be used when it is
2517 * not possible to allocate memory.
2518 */
2519void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
2520{
2521 struct text_poke_loc tp;
2522
2523 text_poke_loc_init(&tp, addr, opcode, len, emulate);
2524 text_poke_bp_batch(&tp, 1);
2525}