Loading...
Note: File does not exist in v3.15.
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3#include "rseq-bits-template.h"
4
5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8static inline __always_inline
9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10{
11 RSEQ_INJECT_C(9)
12
13 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
14 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
15#ifdef RSEQ_COMPARE_TWICE
16 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
18#endif
19 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
20 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
21 RSEQ_INJECT_ASM(3)
22 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
23 RSEQ_INJECT_ASM(4)
24#ifdef RSEQ_COMPARE_TWICE
25 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
26 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
27#endif
28 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
29 RSEQ_INJECT_ASM(5)
30 RSEQ_ASM_DEFINE_ABORT(4, abort)
31 : /* gcc asm goto does not allow outputs */
32 : [cpu_id] "r" (cpu),
33 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
34 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
35 [v] "m" (*v),
36 [expect] "r" (expect),
37 [newv] "r" (newv)
38 RSEQ_INJECT_INPUT
39 : "memory", RSEQ_ASM_TMP_REG_1
40 RSEQ_INJECT_CLOBBER
41 : abort, cmpfail
42#ifdef RSEQ_COMPARE_TWICE
43 , error1, error2
44#endif
45 );
46
47 return 0;
48abort:
49 RSEQ_INJECT_FAILED
50 return -1;
51cmpfail:
52 return 1;
53#ifdef RSEQ_COMPARE_TWICE
54error1:
55 rseq_bug("cpu_id comparison failed");
56error2:
57 rseq_bug("expected value comparison failed");
58#endif
59}
60
61static inline __always_inline
62int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
63 off_t voffp, intptr_t *load, int cpu)
64{
65 RSEQ_INJECT_C(9)
66
67 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
68 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
69#ifdef RSEQ_COMPARE_TWICE
70 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
72#endif
73 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
74 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
75 RSEQ_INJECT_ASM(3)
76 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
77 RSEQ_INJECT_ASM(4)
78#ifdef RSEQ_COMPARE_TWICE
79 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
80 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
81#endif
82 RSEQ_ASM_OP_R_LOAD(v)
83 RSEQ_ASM_OP_R_STORE(load)
84 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
85 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
86 RSEQ_INJECT_ASM(5)
87 RSEQ_ASM_DEFINE_ABORT(4, abort)
88 : /* gcc asm goto does not allow outputs */
89 : [cpu_id] "r" (cpu),
90 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
91 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
92 [v] "m" (*v),
93 [expectnot] "r" (expectnot),
94 [load] "m" (*load),
95 [voffp] "r" (voffp)
96 RSEQ_INJECT_INPUT
97 : "memory", RSEQ_ASM_TMP_REG_1
98 RSEQ_INJECT_CLOBBER
99 : abort, cmpfail
100#ifdef RSEQ_COMPARE_TWICE
101 , error1, error2
102#endif
103 );
104 return 0;
105abort:
106 RSEQ_INJECT_FAILED
107 return -1;
108cmpfail:
109 return 1;
110#ifdef RSEQ_COMPARE_TWICE
111error1:
112 rseq_bug("cpu_id comparison failed");
113error2:
114 rseq_bug("expected value comparison failed");
115#endif
116}
117
118static inline __always_inline
119int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
120{
121 RSEQ_INJECT_C(9)
122
123 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
124#ifdef RSEQ_COMPARE_TWICE
125 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
126#endif
127 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
128 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
129 RSEQ_INJECT_ASM(3)
130#ifdef RSEQ_COMPARE_TWICE
131 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
132#endif
133 RSEQ_ASM_OP_R_LOAD(v)
134 RSEQ_ASM_OP_R_ADD(count)
135 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
136 RSEQ_INJECT_ASM(4)
137 RSEQ_ASM_DEFINE_ABORT(4, abort)
138 : /* gcc asm goto does not allow outputs */
139 : [cpu_id] "r" (cpu),
140 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
141 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
142 [v] "m" (*v),
143 [count] "r" (count)
144 RSEQ_INJECT_INPUT
145 : "memory", RSEQ_ASM_TMP_REG_1
146 RSEQ_INJECT_CLOBBER
147 : abort
148#ifdef RSEQ_COMPARE_TWICE
149 , error1
150#endif
151 );
152 return 0;
153abort:
154 RSEQ_INJECT_FAILED
155 return -1;
156#ifdef RSEQ_COMPARE_TWICE
157error1:
158 rseq_bug("cpu_id comparison failed");
159#endif
160}
161
162static inline __always_inline
163int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
164 intptr_t *v2, intptr_t expect2,
165 intptr_t newv, int cpu)
166{
167 RSEQ_INJECT_C(9)
168
169 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
170 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
171#ifdef RSEQ_COMPARE_TWICE
172 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
175#endif
176 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
178 RSEQ_INJECT_ASM(3)
179 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
180 RSEQ_INJECT_ASM(4)
181 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
182 RSEQ_INJECT_ASM(5)
183#ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
185 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
186 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
187#endif
188 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
189 RSEQ_INJECT_ASM(6)
190 RSEQ_ASM_DEFINE_ABORT(4, abort)
191 : /* gcc asm goto does not allow outputs */
192 : [cpu_id] "r" (cpu),
193 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
194 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
195 [v] "m" (*v),
196 [expect] "r" (expect),
197 [v2] "m" (*v2),
198 [expect2] "r" (expect2),
199 [newv] "r" (newv)
200 RSEQ_INJECT_INPUT
201 : "memory", RSEQ_ASM_TMP_REG_1
202 RSEQ_INJECT_CLOBBER
203 : abort, cmpfail
204#ifdef RSEQ_COMPARE_TWICE
205 , error1, error2, error3
206#endif
207 );
208
209 return 0;
210abort:
211 RSEQ_INJECT_FAILED
212 return -1;
213cmpfail:
214 return 1;
215#ifdef RSEQ_COMPARE_TWICE
216error1:
217 rseq_bug("cpu_id comparison failed");
218error2:
219 rseq_bug("expected value comparison failed");
220error3:
221 rseq_bug("2nd expected value comparison failed");
222#endif
223}
224
225#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
226
227/*
228 * pval = *(ptr+off)
229 * *pval += inc;
230 */
231static inline __always_inline
232int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
233{
234 RSEQ_INJECT_C(9)
235
236 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
237#ifdef RSEQ_COMPARE_TWICE
238 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
239#endif
240 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
242 RSEQ_INJECT_ASM(3)
243#ifdef RSEQ_COMPARE_TWICE
244 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
245#endif
246 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
247 RSEQ_INJECT_ASM(4)
248 RSEQ_ASM_DEFINE_ABORT(4, abort)
249 : /* gcc asm goto does not allow outputs */
250 : [cpu_id] "r" (cpu),
251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
253 [ptr] "r" (ptr),
254 [off] "er" (off),
255 [inc] "er" (inc)
256 RSEQ_INJECT_INPUT
257 : "memory", RSEQ_ASM_TMP_REG_1
258 RSEQ_INJECT_CLOBBER
259 : abort
260#ifdef RSEQ_COMPARE_TWICE
261 , error1
262#endif
263 );
264 return 0;
265abort:
266 RSEQ_INJECT_FAILED
267 return -1;
268#ifdef RSEQ_COMPARE_TWICE
269error1:
270 rseq_bug("cpu_id comparison failed");
271#endif
272}
273
274#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
275 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
276
277#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
279
280static inline __always_inline
281int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
282 intptr_t *v2, intptr_t newv2,
283 intptr_t newv, int cpu)
284{
285 RSEQ_INJECT_C(9)
286
287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
289#ifdef RSEQ_COMPARE_TWICE
290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
292#endif
293 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295 RSEQ_INJECT_ASM(3)
296 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
297 RSEQ_INJECT_ASM(4)
298#ifdef RSEQ_COMPARE_TWICE
299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
300 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
301#endif
302 RSEQ_ASM_OP_STORE(newv2, v2)
303 RSEQ_INJECT_ASM(5)
304#ifdef RSEQ_TEMPLATE_MO_RELEASE
305 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
306#else
307 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
308#endif
309 RSEQ_INJECT_ASM(6)
310 RSEQ_ASM_DEFINE_ABORT(4, abort)
311 : /* gcc asm goto does not allow outputs */
312 : [cpu_id] "r" (cpu),
313 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
315 [expect] "r" (expect),
316 [v] "m" (*v),
317 [newv] "r" (newv),
318 [v2] "m" (*v2),
319 [newv2] "r" (newv2)
320 RSEQ_INJECT_INPUT
321 : "memory", RSEQ_ASM_TMP_REG_1
322 RSEQ_INJECT_CLOBBER
323 : abort, cmpfail
324#ifdef RSEQ_COMPARE_TWICE
325 , error1, error2
326#endif
327 );
328
329 return 0;
330abort:
331 RSEQ_INJECT_FAILED
332 return -1;
333cmpfail:
334 return 1;
335#ifdef RSEQ_COMPARE_TWICE
336error1:
337 rseq_bug("cpu_id comparison failed");
338error2:
339 rseq_bug("expected value comparison failed");
340#endif
341}
342
343static inline __always_inline
344int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
345 void *dst, void *src, size_t len,
346 intptr_t newv, int cpu)
347{
348 RSEQ_INJECT_C(9)
349 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
350 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
351#ifdef RSEQ_COMPARE_TWICE
352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
354#endif
355 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
356 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
357 RSEQ_INJECT_ASM(3)
358 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
359 RSEQ_INJECT_ASM(4)
360#ifdef RSEQ_COMPARE_TWICE
361 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
362 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
363#endif
364 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
365 RSEQ_INJECT_ASM(5)
366#ifdef RSEQ_TEMPLATE_MO_RELEASE
367 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
368#else
369 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
370#endif
371 RSEQ_INJECT_ASM(6)
372 RSEQ_ASM_DEFINE_ABORT(4, abort)
373 : /* gcc asm goto does not allow outputs */
374 : [cpu_id] "r" (cpu),
375 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
376 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
377 [expect] "r" (expect),
378 [v] "m" (*v),
379 [newv] "r" (newv),
380 [dst] "r" (dst),
381 [src] "r" (src),
382 [len] "r" (len)
383 RSEQ_INJECT_INPUT
384 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
385 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
386 RSEQ_INJECT_CLOBBER
387 : abort, cmpfail
388#ifdef RSEQ_COMPARE_TWICE
389 , error1, error2
390#endif
391 );
392
393 return 0;
394abort:
395 RSEQ_INJECT_FAILED
396 return -1;
397cmpfail:
398 return 1;
399#ifdef RSEQ_COMPARE_TWICE
400error1:
401 rseq_bug("cpu_id comparison failed");
402error2:
403 rseq_bug("expected value comparison failed");
404#endif
405}
406
407#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
408 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
409
410#include "rseq-bits-reset.h"