Loading...
Note: File does not exist in v5.9.
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm64-bits.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9#include "rseq-bits-template.h"
10
11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
13
14static inline __attribute__((always_inline))
15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
16{
17 RSEQ_INJECT_C(9)
18
19 __asm__ __volatile__ goto (
20 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
22#ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
25#endif
26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
30 RSEQ_INJECT_ASM(4)
31#ifdef RSEQ_COMPARE_TWICE
32 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
33 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
34#endif
35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
36 RSEQ_INJECT_ASM(5)
37 RSEQ_ASM_DEFINE_ABORT(4, abort)
38 : /* gcc asm goto does not allow outputs */
39 : [cpu_id] "r" (cpu),
40 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
42 [v] "Qo" (*v),
43 [expect] "r" (expect),
44 [newv] "r" (newv)
45 RSEQ_INJECT_INPUT
46 : "memory", RSEQ_ASM_TMP_REG
47 : abort, cmpfail
48#ifdef RSEQ_COMPARE_TWICE
49 , error1, error2
50#endif
51 );
52 rseq_after_asm_goto();
53 return 0;
54abort:
55 rseq_after_asm_goto();
56 RSEQ_INJECT_FAILED
57 return -1;
58cmpfail:
59 rseq_after_asm_goto();
60 return 1;
61#ifdef RSEQ_COMPARE_TWICE
62error1:
63 rseq_after_asm_goto();
64 rseq_bug("cpu_id comparison failed");
65error2:
66 rseq_after_asm_goto();
67 rseq_bug("expected value comparison failed");
68#endif
69}
70
71static inline __attribute__((always_inline))
72int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
73 long voffp, intptr_t *load, int cpu)
74{
75 RSEQ_INJECT_C(9)
76
77 __asm__ __volatile__ goto (
78 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
79 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
80#ifdef RSEQ_COMPARE_TWICE
81 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
82 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
83#endif
84 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
85 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
86 RSEQ_INJECT_ASM(3)
87 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
88 RSEQ_INJECT_ASM(4)
89#ifdef RSEQ_COMPARE_TWICE
90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
91 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
92#endif
93 RSEQ_ASM_OP_R_LOAD(v)
94 RSEQ_ASM_OP_R_STORE(load)
95 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
96 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
97 RSEQ_INJECT_ASM(5)
98 RSEQ_ASM_DEFINE_ABORT(4, abort)
99 : /* gcc asm goto does not allow outputs */
100 : [cpu_id] "r" (cpu),
101 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
102 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
103 [v] "Qo" (*v),
104 [expectnot] "r" (expectnot),
105 [load] "Qo" (*load),
106 [voffp] "r" (voffp)
107 RSEQ_INJECT_INPUT
108 : "memory", RSEQ_ASM_TMP_REG
109 : abort, cmpfail
110#ifdef RSEQ_COMPARE_TWICE
111 , error1, error2
112#endif
113 );
114 rseq_after_asm_goto();
115 return 0;
116abort:
117 rseq_after_asm_goto();
118 RSEQ_INJECT_FAILED
119 return -1;
120cmpfail:
121 rseq_after_asm_goto();
122 return 1;
123#ifdef RSEQ_COMPARE_TWICE
124error1:
125 rseq_after_asm_goto();
126 rseq_bug("cpu_id comparison failed");
127error2:
128 rseq_after_asm_goto();
129 rseq_bug("expected value comparison failed");
130#endif
131}
132
133static inline __attribute__((always_inline))
134int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
135{
136 RSEQ_INJECT_C(9)
137
138 __asm__ __volatile__ goto (
139 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
140#ifdef RSEQ_COMPARE_TWICE
141 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
142#endif
143 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
144 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
145 RSEQ_INJECT_ASM(3)
146#ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
148#endif
149 RSEQ_ASM_OP_R_LOAD(v)
150 RSEQ_ASM_OP_R_ADD(count)
151 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
152 RSEQ_INJECT_ASM(4)
153 RSEQ_ASM_DEFINE_ABORT(4, abort)
154 : /* gcc asm goto does not allow outputs */
155 : [cpu_id] "r" (cpu),
156 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
157 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
158 [v] "Qo" (*v),
159 [count] "r" (count)
160 RSEQ_INJECT_INPUT
161 : "memory", RSEQ_ASM_TMP_REG
162 : abort
163#ifdef RSEQ_COMPARE_TWICE
164 , error1
165#endif
166 );
167 rseq_after_asm_goto();
168 return 0;
169abort:
170 rseq_after_asm_goto();
171 RSEQ_INJECT_FAILED
172 return -1;
173#ifdef RSEQ_COMPARE_TWICE
174error1:
175 rseq_after_asm_goto();
176 rseq_bug("cpu_id comparison failed");
177#endif
178}
179
180static inline __attribute__((always_inline))
181int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
182 intptr_t *v2, intptr_t expect2,
183 intptr_t newv, int cpu)
184{
185 RSEQ_INJECT_C(9)
186
187 __asm__ __volatile__ goto (
188 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
189 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
190#ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
192 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
193 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
194#endif
195 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
196 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
197 RSEQ_INJECT_ASM(3)
198 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
199 RSEQ_INJECT_ASM(4)
200 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
201 RSEQ_INJECT_ASM(5)
202#ifdef RSEQ_COMPARE_TWICE
203 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
204 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
205 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
206#endif
207 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
208 RSEQ_INJECT_ASM(6)
209 RSEQ_ASM_DEFINE_ABORT(4, abort)
210 : /* gcc asm goto does not allow outputs */
211 : [cpu_id] "r" (cpu),
212 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
213 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
214 [v] "Qo" (*v),
215 [expect] "r" (expect),
216 [v2] "Qo" (*v2),
217 [expect2] "r" (expect2),
218 [newv] "r" (newv)
219 RSEQ_INJECT_INPUT
220 : "memory", RSEQ_ASM_TMP_REG
221 : abort, cmpfail
222#ifdef RSEQ_COMPARE_TWICE
223 , error1, error2, error3
224#endif
225 );
226 rseq_after_asm_goto();
227 return 0;
228abort:
229 rseq_after_asm_goto();
230 RSEQ_INJECT_FAILED
231 return -1;
232cmpfail:
233 rseq_after_asm_goto();
234 return 1;
235#ifdef RSEQ_COMPARE_TWICE
236error1:
237 rseq_after_asm_goto();
238 rseq_bug("cpu_id comparison failed");
239error2:
240 rseq_after_asm_goto();
241 rseq_bug("expected value comparison failed");
242error3:
243 rseq_after_asm_goto();
244 rseq_bug("2nd expected value comparison failed");
245#endif
246}
247
248#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
249 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
250
251#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
252 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
253
254static inline __attribute__((always_inline))
255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
256 intptr_t *v2, intptr_t newv2,
257 intptr_t newv, int cpu)
258{
259 RSEQ_INJECT_C(9)
260
261 __asm__ __volatile__ goto (
262 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
263 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
264#ifdef RSEQ_COMPARE_TWICE
265 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
266 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
267#endif
268 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
269 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
270 RSEQ_INJECT_ASM(3)
271 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
272 RSEQ_INJECT_ASM(4)
273#ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
275 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
276#endif
277 RSEQ_ASM_OP_STORE(newv2, v2)
278 RSEQ_INJECT_ASM(5)
279#ifdef RSEQ_TEMPLATE_MO_RELEASE
280 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
281#else
282 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
283#endif
284 RSEQ_INJECT_ASM(6)
285 RSEQ_ASM_DEFINE_ABORT(4, abort)
286 : /* gcc asm goto does not allow outputs */
287 : [cpu_id] "r" (cpu),
288 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
289 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
290 [expect] "r" (expect),
291 [v] "Qo" (*v),
292 [newv] "r" (newv),
293 [v2] "Qo" (*v2),
294 [newv2] "r" (newv2)
295 RSEQ_INJECT_INPUT
296 : "memory", RSEQ_ASM_TMP_REG
297 : abort, cmpfail
298#ifdef RSEQ_COMPARE_TWICE
299 , error1, error2
300#endif
301 );
302 rseq_after_asm_goto();
303 return 0;
304abort:
305 rseq_after_asm_goto();
306 RSEQ_INJECT_FAILED
307 return -1;
308cmpfail:
309 rseq_after_asm_goto();
310 return 1;
311#ifdef RSEQ_COMPARE_TWICE
312error1:
313 rseq_after_asm_goto();
314 rseq_bug("cpu_id comparison failed");
315error2:
316 rseq_after_asm_goto();
317 rseq_bug("expected value comparison failed");
318#endif
319}
320
321static inline __attribute__((always_inline))
322int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
323 void *dst, void *src, size_t len,
324 intptr_t newv, int cpu)
325{
326 RSEQ_INJECT_C(9)
327
328 __asm__ __volatile__ goto (
329 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
330 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
331#ifdef RSEQ_COMPARE_TWICE
332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
333 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
334#endif
335 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
337 RSEQ_INJECT_ASM(3)
338 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
339 RSEQ_INJECT_ASM(4)
340#ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
342 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
343#endif
344 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
345 RSEQ_INJECT_ASM(5)
346#ifdef RSEQ_TEMPLATE_MO_RELEASE
347 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
348#else
349 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
350#endif
351 RSEQ_INJECT_ASM(6)
352 RSEQ_ASM_DEFINE_ABORT(4, abort)
353 : /* gcc asm goto does not allow outputs */
354 : [cpu_id] "r" (cpu),
355 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
356 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
357 [expect] "r" (expect),
358 [v] "Qo" (*v),
359 [newv] "r" (newv),
360 [dst] "r" (dst),
361 [src] "r" (src),
362 [len] "r" (len)
363 RSEQ_INJECT_INPUT
364 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
365 : abort, cmpfail
366#ifdef RSEQ_COMPARE_TWICE
367 , error1, error2
368#endif
369 );
370 rseq_after_asm_goto();
371 return 0;
372abort:
373 rseq_after_asm_goto();
374 RSEQ_INJECT_FAILED
375 return -1;
376cmpfail:
377 rseq_after_asm_goto();
378 return 1;
379#ifdef RSEQ_COMPARE_TWICE
380error1:
381 rseq_after_asm_goto();
382 rseq_bug("cpu_id comparison failed");
383error2:
384 rseq_after_asm_goto();
385 rseq_bug("expected value comparison failed");
386#endif
387}
388
389#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
390 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
391
392#include "rseq-bits-reset.h"