Loading...
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8#ifndef RSEQ_H
9#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
10#endif
11
12#include <stdint.h>
13
14/*
15 * RSEQ_SIG is used with the following reserved undefined instructions, which
16 * trap in user-space:
17 *
18 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
19 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
20 */
21#define RSEQ_SIG 0x53053053
22
23/*
24 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
25 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
26 * address through a "r" input operand.
27 */
28
29/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
30#define RSEQ_CPU_ID_OFFSET 4
31#define RSEQ_CS_OFFSET 8
32#define RSEQ_MM_CID_OFFSET 24
33
34#ifdef __x86_64__
35
36#define RSEQ_ASM_TP_SEGMENT %%fs
37
38#define rseq_smp_mb() \
39 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
40#define rseq_smp_rmb() rseq_barrier()
41#define rseq_smp_wmb() rseq_barrier()
42
43#define rseq_smp_load_acquire(p) \
44__extension__ ({ \
45 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
46 rseq_barrier(); \
47 ____p1; \
48})
49
50#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
51
52#define rseq_smp_store_release(p, v) \
53do { \
54 rseq_barrier(); \
55 RSEQ_WRITE_ONCE(*(p), v); \
56} while (0)
57
58#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
59 start_ip, post_commit_offset, abort_ip) \
60 ".pushsection __rseq_cs, \"aw\"\n\t" \
61 ".balign 32\n\t" \
62 __rseq_str(label) ":\n\t" \
63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
65 ".popsection\n\t" \
66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(label) "b\n\t" \
68 ".popsection\n\t"
69
70
71#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
72 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
73 (post_commit_ip - start_ip), abort_ip)
74
75/*
76 * Exit points of a rseq critical section consist of all instructions outside
77 * of the critical section where a critical section can either branch to or
78 * reach through the normal course of its execution. The abort IP and the
79 * post-commit IP are already part of the __rseq_cs section and should not be
80 * explicitly defined as additional exit points. Knowing all exit points is
81 * useful to assist debuggers stepping over the critical section.
82 */
83#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
84 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
85 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
86 ".popsection\n\t"
87
88#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
89 RSEQ_INJECT_ASM(1) \
90 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
91 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
92 __rseq_str(label) ":\n\t"
93
94#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
95 RSEQ_INJECT_ASM(2) \
96 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
97 "jnz " __rseq_str(label) "\n\t"
98
99#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
100 ".pushsection __rseq_failure, \"ax\"\n\t" \
101 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
102 ".byte 0x0f, 0xb9, 0x3d\n\t" \
103 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
104 __rseq_str(label) ":\n\t" \
105 teardown \
106 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
107 ".popsection\n\t"
108
109#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
110 ".pushsection __rseq_failure, \"ax\"\n\t" \
111 __rseq_str(label) ":\n\t" \
112 teardown \
113 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
114 ".popsection\n\t"
115
116#elif defined(__i386__)
117
118#define RSEQ_ASM_TP_SEGMENT %%gs
119
120#define rseq_smp_mb() \
121 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
122#define rseq_smp_rmb() \
123 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
124#define rseq_smp_wmb() \
125 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
126
127#define rseq_smp_load_acquire(p) \
128__extension__ ({ \
129 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
130 rseq_smp_mb(); \
131 ____p1; \
132})
133
134#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
135
136#define rseq_smp_store_release(p, v) \
137do { \
138 rseq_smp_mb(); \
139 RSEQ_WRITE_ONCE(*p, v); \
140} while (0)
141
142/*
143 * Use eax as scratch register and take memory operands as input to
144 * lessen register pressure. Especially needed when compiling in O0.
145 */
146#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
147 start_ip, post_commit_offset, abort_ip) \
148 ".pushsection __rseq_cs, \"aw\"\n\t" \
149 ".balign 32\n\t" \
150 __rseq_str(label) ":\n\t" \
151 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
152 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
153 ".popsection\n\t" \
154 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
155 ".long " __rseq_str(label) "b, 0x0\n\t" \
156 ".popsection\n\t"
157
158#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
159 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
160 (post_commit_ip - start_ip), abort_ip)
161
162/*
163 * Exit points of a rseq critical section consist of all instructions outside
164 * of the critical section where a critical section can either branch to or
165 * reach through the normal course of its execution. The abort IP and the
166 * post-commit IP are already part of the __rseq_cs section and should not be
167 * explicitly defined as additional exit points. Knowing all exit points is
168 * useful to assist debuggers stepping over the critical section.
169 */
170#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
171 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
172 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
173 ".popsection\n\t"
174
175#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
176 RSEQ_INJECT_ASM(1) \
177 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
178 __rseq_str(label) ":\n\t"
179
180#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
181 RSEQ_INJECT_ASM(2) \
182 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
183 "jnz " __rseq_str(label) "\n\t"
184
185#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
186 ".pushsection __rseq_failure, \"ax\"\n\t" \
187 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
188 ".byte 0x0f, 0xb9, 0x3d\n\t" \
189 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
190 __rseq_str(label) ":\n\t" \
191 teardown \
192 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
193 ".popsection\n\t"
194
195#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
196 ".pushsection __rseq_failure, \"ax\"\n\t" \
197 __rseq_str(label) ":\n\t" \
198 teardown \
199 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
200 ".popsection\n\t"
201
202#endif
203
204/* Per-cpu-id indexing. */
205
206#define RSEQ_TEMPLATE_CPU_ID
207#define RSEQ_TEMPLATE_MO_RELAXED
208#include "rseq-x86-bits.h"
209#undef RSEQ_TEMPLATE_MO_RELAXED
210
211#define RSEQ_TEMPLATE_MO_RELEASE
212#include "rseq-x86-bits.h"
213#undef RSEQ_TEMPLATE_MO_RELEASE
214#undef RSEQ_TEMPLATE_CPU_ID
215
216/* Per-mm-cid indexing. */
217
218#define RSEQ_TEMPLATE_MM_CID
219#define RSEQ_TEMPLATE_MO_RELAXED
220#include "rseq-x86-bits.h"
221#undef RSEQ_TEMPLATE_MO_RELAXED
222
223#define RSEQ_TEMPLATE_MO_RELEASE
224#include "rseq-x86-bits.h"
225#undef RSEQ_TEMPLATE_MO_RELEASE
226#undef RSEQ_TEMPLATE_MM_CID
227
228/* APIs which are not based on cpu ids. */
229
230#define RSEQ_TEMPLATE_CPU_ID_NONE
231#define RSEQ_TEMPLATE_MO_RELAXED
232#include "rseq-x86-bits.h"
233#undef RSEQ_TEMPLATE_MO_RELAXED
234#undef RSEQ_TEMPLATE_CPU_ID_NONE