1#ifndef _ASM_X86_JUMP_LABEL_H
  2#define _ASM_X86_JUMP_LABEL_H
  3
  4#ifndef HAVE_JUMP_LABEL
  5/*
  6 * For better or for worse, if jump labels (the gcc extension) are missing,
  7 * then the entire static branch patching infrastructure is compiled out.
  8 * If that happens, the code in here will malfunction.  Raise a compiler
  9 * error instead.
 10 *
 11 * In theory, jump labels and the static branch patching infrastructure
 12 * could be decoupled to fix this.
 13 */
 14#error asm/jump_label.h included on a non-jump-label kernel
 15#endif
 16
 17#define JUMP_LABEL_NOP_SIZE 5
 18
 19#ifdef CONFIG_X86_64
 20# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
 21#else
 22# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
 23#endif
 24
 25#include <asm/asm.h>
 26#include <asm/nops.h>
 27
 28#ifndef __ASSEMBLY__
 29
 30#include <linux/stringify.h>
 31#include <linux/types.h>
 32
 
 
 
 
 
 
 
 
 
 
 33static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 34{
 35	asm_volatile_goto("1:"
 36		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
 37		".pushsection __jump_table,  \"aw\" \n\t"
 38		_ASM_ALIGN "\n\t"
 39		_ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
 40		".popsection \n\t"
 
 
 
 
 
 
 
 
 
 
 
 41		: :  "i" (key), "i" (branch) : : l_yes);
 42
 43	return false;
 44l_yes:
 45	return true;
 46}
 47
 48static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
 
 
 49{
 50	asm_volatile_goto("1:"
 51		".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
 52		"2:\n\t"
 53		".pushsection __jump_table,  \"aw\" \n\t"
 54		_ASM_ALIGN "\n\t"
 55		_ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
 56		".popsection \n\t"
 57		: :  "i" (key), "i" (branch) : : l_yes);
 58
 59	return false;
 60l_yes:
 61	return true;
 62}
 63
 64#ifdef CONFIG_X86_64
 65typedef u64 jump_label_t;
 66#else
 67typedef u32 jump_label_t;
 68#endif
 69
 70struct jump_entry {
 71	jump_label_t code;
 72	jump_label_t target;
 73	jump_label_t key;
 74};
 75
 76#else	/* __ASSEMBLY__ */
 77
 78.macro STATIC_JUMP_IF_TRUE target, key, def
 79.Lstatic_jump_\@:
 80	.if \def
 81	/* Equivalent to "jmp.d32 \target" */
 82	.byte		0xe9
 83	.long		\target - .Lstatic_jump_after_\@
 84.Lstatic_jump_after_\@:
 85	.else
 86	.byte		STATIC_KEY_INIT_NOP
 87	.endif
 88	.pushsection __jump_table, "aw"
 89	_ASM_ALIGN
 90	_ASM_PTR	.Lstatic_jump_\@, \target, \key
 91	.popsection
 92.endm
 93
 94.macro STATIC_JUMP_IF_FALSE target, key, def
 95.Lstatic_jump_\@:
 96	.if \def
 97	.byte		STATIC_KEY_INIT_NOP
 98	.else
 99	/* Equivalent to "jmp.d32 \target" */
100	.byte		0xe9
101	.long		\target - .Lstatic_jump_after_\@
102.Lstatic_jump_after_\@:
103	.endif
104	.pushsection __jump_table, "aw"
105	_ASM_ALIGN
106	_ASM_PTR	.Lstatic_jump_\@, \target, \key + 1
107	.popsection
108.endm
109
110#endif	/* __ASSEMBLY__ */
111
112#endif

 1/* SPDX-License-Identifier: GPL-2.0 */
 2#ifndef _ASM_X86_JUMP_LABEL_H
 3#define _ASM_X86_JUMP_LABEL_H
 4
 5#define HAVE_JUMP_LABEL_BATCH
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 6
 7#include <asm/asm.h>
 8#include <asm/nops.h>
 9
10#ifndef __ASSEMBLY__
11
12#include <linux/stringify.h>
13#include <linux/types.h>
14
15#define JUMP_TABLE_ENTRY				\
16	".pushsection __jump_table,  \"aw\" \n\t"	\
17	_ASM_ALIGN "\n\t"				\
18	".long 1b - . \n\t"				\
19	".long %l[l_yes] - . \n\t"			\
20	_ASM_PTR "%c0 + %c1 - .\n\t"			\
21	".popsection \n\t"
22
23#ifdef CONFIG_HAVE_JUMP_LABEL_HACK
24
25static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
26{
27	asm goto("1:"
28		"jmp %l[l_yes] # objtool NOPs this \n\t"
29		JUMP_TABLE_ENTRY
30		: :  "i" (key), "i" (2 | branch) : : l_yes);
31
32	return false;
33l_yes:
34	return true;
35}
36
37#else /* !CONFIG_HAVE_JUMP_LABEL_HACK */
38
39static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
40{
41	asm goto("1:"
42		".byte " __stringify(BYTES_NOP5) "\n\t"
43		JUMP_TABLE_ENTRY
44		: :  "i" (key), "i" (branch) : : l_yes);
45
46	return false;
47l_yes:
48	return true;
49}
50
51#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
52
53static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
54{
55	asm goto("1:"
56		"jmp %l[l_yes]\n\t"
57		JUMP_TABLE_ENTRY
 
 
 
 
58		: :  "i" (key), "i" (branch) : : l_yes);
59
60	return false;
61l_yes:
62	return true;
63}
64
65extern int arch_jump_entry_size(struct jump_entry *entry);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
67#endif	/* __ASSEMBLY__ */
68
69#endif