Loading...
1// SPDX-License-Identifier: LGPL-2.1
2#define _GNU_SOURCE
3#include <assert.h>
4#include <linux/membarrier.h>
5#include <pthread.h>
6#include <sched.h>
7#include <stdatomic.h>
8#include <stdint.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <syscall.h>
13#include <unistd.h>
14#include <poll.h>
15#include <sys/types.h>
16#include <signal.h>
17#include <errno.h>
18#include <stddef.h>
19#include <stdbool.h>
20
21static inline pid_t rseq_gettid(void)
22{
23 return syscall(__NR_gettid);
24}
25
26#define NR_INJECT 9
27static int loop_cnt[NR_INJECT + 1];
28
29static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
30static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
31static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
32static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
33static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
34static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
35
36static int opt_modulo, verbose;
37
38static int opt_yield, opt_signal, opt_sleep,
39 opt_disable_rseq, opt_threads = 200,
40 opt_disable_mod = 0, opt_test = 's';
41
42static long long opt_reps = 5000;
43
44static __thread __attribute__((tls_model("initial-exec")))
45unsigned int signals_delivered;
46
47#ifndef BENCHMARK
48
49static __thread __attribute__((tls_model("initial-exec"), unused))
50unsigned int yield_mod_cnt, nr_abort;
51
52#define printf_verbose(fmt, ...) \
53 do { \
54 if (verbose) \
55 printf(fmt, ## __VA_ARGS__); \
56 } while (0)
57
58#ifdef __i386__
59
60#define INJECT_ASM_REG "eax"
61
62#define RSEQ_INJECT_CLOBBER \
63 , INJECT_ASM_REG
64
65#define RSEQ_INJECT_ASM(n) \
66 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
67 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
68 "jz 333f\n\t" \
69 "222:\n\t" \
70 "dec %%" INJECT_ASM_REG "\n\t" \
71 "jnz 222b\n\t" \
72 "333:\n\t"
73
74#elif defined(__x86_64__)
75
76#define INJECT_ASM_REG_P "rax"
77#define INJECT_ASM_REG "eax"
78
79#define RSEQ_INJECT_CLOBBER \
80 , INJECT_ASM_REG_P \
81 , INJECT_ASM_REG
82
83#define RSEQ_INJECT_ASM(n) \
84 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
85 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
86 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
87 "jz 333f\n\t" \
88 "222:\n\t" \
89 "dec %%" INJECT_ASM_REG "\n\t" \
90 "jnz 222b\n\t" \
91 "333:\n\t"
92
93#elif defined(__s390__)
94
95#define RSEQ_INJECT_INPUT \
96 , [loop_cnt_1]"m"(loop_cnt[1]) \
97 , [loop_cnt_2]"m"(loop_cnt[2]) \
98 , [loop_cnt_3]"m"(loop_cnt[3]) \
99 , [loop_cnt_4]"m"(loop_cnt[4]) \
100 , [loop_cnt_5]"m"(loop_cnt[5]) \
101 , [loop_cnt_6]"m"(loop_cnt[6])
102
103#define INJECT_ASM_REG "r12"
104
105#define RSEQ_INJECT_CLOBBER \
106 , INJECT_ASM_REG
107
108#define RSEQ_INJECT_ASM(n) \
109 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
110 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
111 "je 333f\n\t" \
112 "222:\n\t" \
113 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
114 "jnz 222b\n\t" \
115 "333:\n\t"
116
117#elif defined(__ARMEL__)
118
119#define RSEQ_INJECT_INPUT \
120 , [loop_cnt_1]"m"(loop_cnt[1]) \
121 , [loop_cnt_2]"m"(loop_cnt[2]) \
122 , [loop_cnt_3]"m"(loop_cnt[3]) \
123 , [loop_cnt_4]"m"(loop_cnt[4]) \
124 , [loop_cnt_5]"m"(loop_cnt[5]) \
125 , [loop_cnt_6]"m"(loop_cnt[6])
126
127#define INJECT_ASM_REG "r4"
128
129#define RSEQ_INJECT_CLOBBER \
130 , INJECT_ASM_REG
131
132#define RSEQ_INJECT_ASM(n) \
133 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
134 "cmp " INJECT_ASM_REG ", #0\n\t" \
135 "beq 333f\n\t" \
136 "222:\n\t" \
137 "subs " INJECT_ASM_REG ", #1\n\t" \
138 "bne 222b\n\t" \
139 "333:\n\t"
140
141#elif defined(__AARCH64EL__)
142
143#define RSEQ_INJECT_INPUT \
144 , [loop_cnt_1] "Qo" (loop_cnt[1]) \
145 , [loop_cnt_2] "Qo" (loop_cnt[2]) \
146 , [loop_cnt_3] "Qo" (loop_cnt[3]) \
147 , [loop_cnt_4] "Qo" (loop_cnt[4]) \
148 , [loop_cnt_5] "Qo" (loop_cnt[5]) \
149 , [loop_cnt_6] "Qo" (loop_cnt[6])
150
151#define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
152
153#define RSEQ_INJECT_ASM(n) \
154 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
155 " cbz " INJECT_ASM_REG ", 333f\n" \
156 "222:\n" \
157 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
158 " cbnz " INJECT_ASM_REG ", 222b\n" \
159 "333:\n"
160
161#elif defined(__PPC__)
162
163#define RSEQ_INJECT_INPUT \
164 , [loop_cnt_1]"m"(loop_cnt[1]) \
165 , [loop_cnt_2]"m"(loop_cnt[2]) \
166 , [loop_cnt_3]"m"(loop_cnt[3]) \
167 , [loop_cnt_4]"m"(loop_cnt[4]) \
168 , [loop_cnt_5]"m"(loop_cnt[5]) \
169 , [loop_cnt_6]"m"(loop_cnt[6])
170
171#define INJECT_ASM_REG "r18"
172
173#define RSEQ_INJECT_CLOBBER \
174 , INJECT_ASM_REG
175
176#define RSEQ_INJECT_ASM(n) \
177 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
178 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
179 "beq 333f\n\t" \
180 "222:\n\t" \
181 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
182 "bne 222b\n\t" \
183 "333:\n\t"
184
185#elif defined(__mips__)
186
187#define RSEQ_INJECT_INPUT \
188 , [loop_cnt_1]"m"(loop_cnt[1]) \
189 , [loop_cnt_2]"m"(loop_cnt[2]) \
190 , [loop_cnt_3]"m"(loop_cnt[3]) \
191 , [loop_cnt_4]"m"(loop_cnt[4]) \
192 , [loop_cnt_5]"m"(loop_cnt[5]) \
193 , [loop_cnt_6]"m"(loop_cnt[6])
194
195#define INJECT_ASM_REG "$5"
196
197#define RSEQ_INJECT_CLOBBER \
198 , INJECT_ASM_REG
199
200#define RSEQ_INJECT_ASM(n) \
201 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
202 "beqz " INJECT_ASM_REG ", 333f\n\t" \
203 "222:\n\t" \
204 "addiu " INJECT_ASM_REG ", -1\n\t" \
205 "bnez " INJECT_ASM_REG ", 222b\n\t" \
206 "333:\n\t"
207#elif defined(__riscv)
208
209#define RSEQ_INJECT_INPUT \
210 , [loop_cnt_1]"m"(loop_cnt[1]) \
211 , [loop_cnt_2]"m"(loop_cnt[2]) \
212 , [loop_cnt_3]"m"(loop_cnt[3]) \
213 , [loop_cnt_4]"m"(loop_cnt[4]) \
214 , [loop_cnt_5]"m"(loop_cnt[5]) \
215 , [loop_cnt_6]"m"(loop_cnt[6])
216
217#define INJECT_ASM_REG "t1"
218
219#define RSEQ_INJECT_CLOBBER \
220 , INJECT_ASM_REG
221
222#define RSEQ_INJECT_ASM(n) \
223 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
224 "beqz " INJECT_ASM_REG ", 333f\n\t" \
225 "222:\n\t" \
226 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
227 "bnez " INJECT_ASM_REG ", 222b\n\t" \
228 "333:\n\t"
229
230
231#else
232#error unsupported target
233#endif
234
235#define RSEQ_INJECT_FAILED \
236 nr_abort++;
237
238#define RSEQ_INJECT_C(n) \
239{ \
240 int loc_i, loc_nr_loops = loop_cnt[n]; \
241 \
242 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
243 rseq_barrier(); \
244 } \
245 if (loc_nr_loops == -1 && opt_modulo) { \
246 if (yield_mod_cnt == opt_modulo - 1) { \
247 if (opt_sleep > 0) \
248 poll(NULL, 0, opt_sleep); \
249 if (opt_yield) \
250 sched_yield(); \
251 if (opt_signal) \
252 raise(SIGUSR1); \
253 yield_mod_cnt = 0; \
254 } else { \
255 yield_mod_cnt++; \
256 } \
257 } \
258}
259
260#else
261
262#define printf_verbose(fmt, ...)
263
264#endif /* BENCHMARK */
265
266#include "rseq.h"
267
268static enum rseq_mo opt_mo = RSEQ_MO_RELAXED;
269
270#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
271#define TEST_MEMBARRIER
272
273static int sys_membarrier(int cmd, int flags, int cpu_id)
274{
275 return syscall(__NR_membarrier, cmd, flags, cpu_id);
276}
277#endif
278
279#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID
280# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID
281static
282int get_current_cpu_id(void)
283{
284 return rseq_current_mm_cid();
285}
286static
287bool rseq_validate_cpu_id(void)
288{
289 return rseq_mm_cid_available();
290}
291static
292bool rseq_use_cpu_index(void)
293{
294 return false; /* Use mm_cid */
295}
296# ifdef TEST_MEMBARRIER
297/*
298 * Membarrier does not currently support targeting a mm_cid, so
299 * issue the barrier on all cpus.
300 */
301static
302int rseq_membarrier_expedited(int cpu)
303{
304 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
305 0, 0);
306}
307# endif /* TEST_MEMBARRIER */
308#else
309# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID
310static
311int get_current_cpu_id(void)
312{
313 return rseq_cpu_start();
314}
315static
316bool rseq_validate_cpu_id(void)
317{
318 return rseq_current_cpu_raw() >= 0;
319}
320static
321bool rseq_use_cpu_index(void)
322{
323 return true; /* Use cpu_id as index. */
324}
325# ifdef TEST_MEMBARRIER
326static
327int rseq_membarrier_expedited(int cpu)
328{
329 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
330 MEMBARRIER_CMD_FLAG_CPU, cpu);
331}
332# endif /* TEST_MEMBARRIER */
333#endif
334
335struct percpu_lock_entry {
336 intptr_t v;
337} __attribute__((aligned(128)));
338
339struct percpu_lock {
340 struct percpu_lock_entry c[CPU_SETSIZE];
341};
342
343struct test_data_entry {
344 intptr_t count;
345} __attribute__((aligned(128)));
346
347struct spinlock_test_data {
348 struct percpu_lock lock;
349 struct test_data_entry c[CPU_SETSIZE];
350};
351
352struct spinlock_thread_test_data {
353 struct spinlock_test_data *data;
354 long long reps;
355 int reg;
356};
357
358struct inc_test_data {
359 struct test_data_entry c[CPU_SETSIZE];
360};
361
362struct inc_thread_test_data {
363 struct inc_test_data *data;
364 long long reps;
365 int reg;
366};
367
368struct percpu_list_node {
369 intptr_t data;
370 struct percpu_list_node *next;
371};
372
373struct percpu_list_entry {
374 struct percpu_list_node *head;
375} __attribute__((aligned(128)));
376
377struct percpu_list {
378 struct percpu_list_entry c[CPU_SETSIZE];
379};
380
381#define BUFFER_ITEM_PER_CPU 100
382
383struct percpu_buffer_node {
384 intptr_t data;
385};
386
387struct percpu_buffer_entry {
388 intptr_t offset;
389 intptr_t buflen;
390 struct percpu_buffer_node **array;
391} __attribute__((aligned(128)));
392
393struct percpu_buffer {
394 struct percpu_buffer_entry c[CPU_SETSIZE];
395};
396
397#define MEMCPY_BUFFER_ITEM_PER_CPU 100
398
399struct percpu_memcpy_buffer_node {
400 intptr_t data1;
401 uint64_t data2;
402};
403
404struct percpu_memcpy_buffer_entry {
405 intptr_t offset;
406 intptr_t buflen;
407 struct percpu_memcpy_buffer_node *array;
408} __attribute__((aligned(128)));
409
410struct percpu_memcpy_buffer {
411 struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
412};
413
414/* A simple percpu spinlock. Grabs lock on current cpu. */
415static int rseq_this_cpu_lock(struct percpu_lock *lock)
416{
417 int cpu;
418
419 for (;;) {
420 int ret;
421
422 cpu = get_current_cpu_id();
423 if (cpu < 0) {
424 fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n",
425 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu);
426 abort();
427 }
428 ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
429 &lock->c[cpu].v,
430 0, 1, cpu);
431 if (rseq_likely(!ret))
432 break;
433 /* Retry if comparison fails or rseq aborts. */
434 }
435 /*
436 * Acquire semantic when taking lock after control dependency.
437 * Matches rseq_smp_store_release().
438 */
439 rseq_smp_acquire__after_ctrl_dep();
440 return cpu;
441}
442
443static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
444{
445 assert(lock->c[cpu].v == 1);
446 /*
447 * Release lock, with release semantic. Matches
448 * rseq_smp_acquire__after_ctrl_dep().
449 */
450 rseq_smp_store_release(&lock->c[cpu].v, 0);
451}
452
453void *test_percpu_spinlock_thread(void *arg)
454{
455 struct spinlock_thread_test_data *thread_data = arg;
456 struct spinlock_test_data *data = thread_data->data;
457 long long i, reps;
458
459 if (!opt_disable_rseq && thread_data->reg &&
460 rseq_register_current_thread())
461 abort();
462 reps = thread_data->reps;
463 for (i = 0; i < reps; i++) {
464 int cpu = rseq_this_cpu_lock(&data->lock);
465 data->c[cpu].count++;
466 rseq_percpu_unlock(&data->lock, cpu);
467#ifndef BENCHMARK
468 if (i != 0 && !(i % (reps / 10)))
469 printf_verbose("tid %d: count %lld\n",
470 (int) rseq_gettid(), i);
471#endif
472 }
473 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
474 (int) rseq_gettid(), nr_abort, signals_delivered);
475 if (!opt_disable_rseq && thread_data->reg &&
476 rseq_unregister_current_thread())
477 abort();
478 return NULL;
479}
480
481/*
482 * A simple test which implements a sharded counter using a per-cpu
483 * lock. Obviously real applications might prefer to simply use a
484 * per-cpu increment; however, this is reasonable for a test and the
485 * lock can be extended to synchronize more complicated operations.
486 */
487void test_percpu_spinlock(void)
488{
489 const int num_threads = opt_threads;
490 int i, ret;
491 uint64_t sum;
492 pthread_t test_threads[num_threads];
493 struct spinlock_test_data data;
494 struct spinlock_thread_test_data thread_data[num_threads];
495
496 memset(&data, 0, sizeof(data));
497 for (i = 0; i < num_threads; i++) {
498 thread_data[i].reps = opt_reps;
499 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
500 thread_data[i].reg = 1;
501 else
502 thread_data[i].reg = 0;
503 thread_data[i].data = &data;
504 ret = pthread_create(&test_threads[i], NULL,
505 test_percpu_spinlock_thread,
506 &thread_data[i]);
507 if (ret) {
508 errno = ret;
509 perror("pthread_create");
510 abort();
511 }
512 }
513
514 for (i = 0; i < num_threads; i++) {
515 ret = pthread_join(test_threads[i], NULL);
516 if (ret) {
517 errno = ret;
518 perror("pthread_join");
519 abort();
520 }
521 }
522
523 sum = 0;
524 for (i = 0; i < CPU_SETSIZE; i++)
525 sum += data.c[i].count;
526
527 assert(sum == (uint64_t)opt_reps * num_threads);
528}
529
530void *test_percpu_inc_thread(void *arg)
531{
532 struct inc_thread_test_data *thread_data = arg;
533 struct inc_test_data *data = thread_data->data;
534 long long i, reps;
535
536 if (!opt_disable_rseq && thread_data->reg &&
537 rseq_register_current_thread())
538 abort();
539 reps = thread_data->reps;
540 for (i = 0; i < reps; i++) {
541 int ret;
542
543 do {
544 int cpu;
545
546 cpu = get_current_cpu_id();
547 ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU,
548 &data->c[cpu].count, 1, cpu);
549 } while (rseq_unlikely(ret));
550#ifndef BENCHMARK
551 if (i != 0 && !(i % (reps / 10)))
552 printf_verbose("tid %d: count %lld\n",
553 (int) rseq_gettid(), i);
554#endif
555 }
556 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
557 (int) rseq_gettid(), nr_abort, signals_delivered);
558 if (!opt_disable_rseq && thread_data->reg &&
559 rseq_unregister_current_thread())
560 abort();
561 return NULL;
562}
563
564void test_percpu_inc(void)
565{
566 const int num_threads = opt_threads;
567 int i, ret;
568 uint64_t sum;
569 pthread_t test_threads[num_threads];
570 struct inc_test_data data;
571 struct inc_thread_test_data thread_data[num_threads];
572
573 memset(&data, 0, sizeof(data));
574 for (i = 0; i < num_threads; i++) {
575 thread_data[i].reps = opt_reps;
576 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
577 thread_data[i].reg = 1;
578 else
579 thread_data[i].reg = 0;
580 thread_data[i].data = &data;
581 ret = pthread_create(&test_threads[i], NULL,
582 test_percpu_inc_thread,
583 &thread_data[i]);
584 if (ret) {
585 errno = ret;
586 perror("pthread_create");
587 abort();
588 }
589 }
590
591 for (i = 0; i < num_threads; i++) {
592 ret = pthread_join(test_threads[i], NULL);
593 if (ret) {
594 errno = ret;
595 perror("pthread_join");
596 abort();
597 }
598 }
599
600 sum = 0;
601 for (i = 0; i < CPU_SETSIZE; i++)
602 sum += data.c[i].count;
603
604 assert(sum == (uint64_t)opt_reps * num_threads);
605}
606
607void this_cpu_list_push(struct percpu_list *list,
608 struct percpu_list_node *node,
609 int *_cpu)
610{
611 int cpu;
612
613 for (;;) {
614 intptr_t *targetptr, newval, expect;
615 int ret;
616
617 cpu = get_current_cpu_id();
618 /* Load list->c[cpu].head with single-copy atomicity. */
619 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
620 newval = (intptr_t)node;
621 targetptr = (intptr_t *)&list->c[cpu].head;
622 node->next = (struct percpu_list_node *)expect;
623 ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
624 targetptr, expect, newval, cpu);
625 if (rseq_likely(!ret))
626 break;
627 /* Retry if comparison fails or rseq aborts. */
628 }
629 if (_cpu)
630 *_cpu = cpu;
631}
632
633/*
634 * Unlike a traditional lock-less linked list; the availability of a
635 * rseq primitive allows us to implement pop without concerns over
636 * ABA-type races.
637 */
638struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
639 int *_cpu)
640{
641 struct percpu_list_node *node = NULL;
642 int cpu;
643
644 for (;;) {
645 struct percpu_list_node *head;
646 intptr_t *targetptr, expectnot, *load;
647 long offset;
648 int ret;
649
650 cpu = get_current_cpu_id();
651 targetptr = (intptr_t *)&list->c[cpu].head;
652 expectnot = (intptr_t)NULL;
653 offset = offsetof(struct percpu_list_node, next);
654 load = (intptr_t *)&head;
655 ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU,
656 targetptr, expectnot,
657 offset, load, cpu);
658 if (rseq_likely(!ret)) {
659 node = head;
660 break;
661 }
662 if (ret > 0)
663 break;
664 /* Retry if rseq aborts. */
665 }
666 if (_cpu)
667 *_cpu = cpu;
668 return node;
669}
670
671/*
672 * __percpu_list_pop is not safe against concurrent accesses. Should
673 * only be used on lists that are not concurrently modified.
674 */
675struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
676{
677 struct percpu_list_node *node;
678
679 node = list->c[cpu].head;
680 if (!node)
681 return NULL;
682 list->c[cpu].head = node->next;
683 return node;
684}
685
686void *test_percpu_list_thread(void *arg)
687{
688 long long i, reps;
689 struct percpu_list *list = (struct percpu_list *)arg;
690
691 if (!opt_disable_rseq && rseq_register_current_thread())
692 abort();
693
694 reps = opt_reps;
695 for (i = 0; i < reps; i++) {
696 struct percpu_list_node *node;
697
698 node = this_cpu_list_pop(list, NULL);
699 if (opt_yield)
700 sched_yield(); /* encourage shuffling */
701 if (node)
702 this_cpu_list_push(list, node, NULL);
703 }
704
705 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
706 (int) rseq_gettid(), nr_abort, signals_delivered);
707 if (!opt_disable_rseq && rseq_unregister_current_thread())
708 abort();
709
710 return NULL;
711}
712
713/* Simultaneous modification to a per-cpu linked list from many threads. */
714void test_percpu_list(void)
715{
716 const int num_threads = opt_threads;
717 int i, j, ret;
718 uint64_t sum = 0, expected_sum = 0;
719 struct percpu_list list;
720 pthread_t test_threads[num_threads];
721 cpu_set_t allowed_cpus;
722
723 memset(&list, 0, sizeof(list));
724
725 /* Generate list entries for every usable cpu. */
726 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
727 for (i = 0; i < CPU_SETSIZE; i++) {
728 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
729 continue;
730 for (j = 1; j <= 100; j++) {
731 struct percpu_list_node *node;
732
733 expected_sum += j;
734
735 node = malloc(sizeof(*node));
736 assert(node);
737 node->data = j;
738 node->next = list.c[i].head;
739 list.c[i].head = node;
740 }
741 }
742
743 for (i = 0; i < num_threads; i++) {
744 ret = pthread_create(&test_threads[i], NULL,
745 test_percpu_list_thread, &list);
746 if (ret) {
747 errno = ret;
748 perror("pthread_create");
749 abort();
750 }
751 }
752
753 for (i = 0; i < num_threads; i++) {
754 ret = pthread_join(test_threads[i], NULL);
755 if (ret) {
756 errno = ret;
757 perror("pthread_join");
758 abort();
759 }
760 }
761
762 for (i = 0; i < CPU_SETSIZE; i++) {
763 struct percpu_list_node *node;
764
765 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
766 continue;
767
768 while ((node = __percpu_list_pop(&list, i))) {
769 sum += node->data;
770 free(node);
771 }
772 }
773
774 /*
775 * All entries should now be accounted for (unless some external
776 * actor is interfering with our allowed affinity while this
777 * test is running).
778 */
779 assert(sum == expected_sum);
780}
781
782bool this_cpu_buffer_push(struct percpu_buffer *buffer,
783 struct percpu_buffer_node *node,
784 int *_cpu)
785{
786 bool result = false;
787 int cpu;
788
789 for (;;) {
790 intptr_t *targetptr_spec, newval_spec;
791 intptr_t *targetptr_final, newval_final;
792 intptr_t offset;
793 int ret;
794
795 cpu = get_current_cpu_id();
796 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
797 if (offset == buffer->c[cpu].buflen)
798 break;
799 newval_spec = (intptr_t)node;
800 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
801 newval_final = offset + 1;
802 targetptr_final = &buffer->c[cpu].offset;
803 ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU,
804 targetptr_final, offset, targetptr_spec,
805 newval_spec, newval_final, cpu);
806 if (rseq_likely(!ret)) {
807 result = true;
808 break;
809 }
810 /* Retry if comparison fails or rseq aborts. */
811 }
812 if (_cpu)
813 *_cpu = cpu;
814 return result;
815}
816
817struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
818 int *_cpu)
819{
820 struct percpu_buffer_node *head;
821 int cpu;
822
823 for (;;) {
824 intptr_t *targetptr, newval;
825 intptr_t offset;
826 int ret;
827
828 cpu = get_current_cpu_id();
829 /* Load offset with single-copy atomicity. */
830 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
831 if (offset == 0) {
832 head = NULL;
833 break;
834 }
835 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
836 newval = offset - 1;
837 targetptr = (intptr_t *)&buffer->c[cpu].offset;
838 ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
839 targetptr, offset,
840 (intptr_t *)&buffer->c[cpu].array[offset - 1],
841 (intptr_t)head, newval, cpu);
842 if (rseq_likely(!ret))
843 break;
844 /* Retry if comparison fails or rseq aborts. */
845 }
846 if (_cpu)
847 *_cpu = cpu;
848 return head;
849}
850
851/*
852 * __percpu_buffer_pop is not safe against concurrent accesses. Should
853 * only be used on buffers that are not concurrently modified.
854 */
855struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
856 int cpu)
857{
858 struct percpu_buffer_node *head;
859 intptr_t offset;
860
861 offset = buffer->c[cpu].offset;
862 if (offset == 0)
863 return NULL;
864 head = buffer->c[cpu].array[offset - 1];
865 buffer->c[cpu].offset = offset - 1;
866 return head;
867}
868
869void *test_percpu_buffer_thread(void *arg)
870{
871 long long i, reps;
872 struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
873
874 if (!opt_disable_rseq && rseq_register_current_thread())
875 abort();
876
877 reps = opt_reps;
878 for (i = 0; i < reps; i++) {
879 struct percpu_buffer_node *node;
880
881 node = this_cpu_buffer_pop(buffer, NULL);
882 if (opt_yield)
883 sched_yield(); /* encourage shuffling */
884 if (node) {
885 if (!this_cpu_buffer_push(buffer, node, NULL)) {
886 /* Should increase buffer size. */
887 abort();
888 }
889 }
890 }
891
892 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
893 (int) rseq_gettid(), nr_abort, signals_delivered);
894 if (!opt_disable_rseq && rseq_unregister_current_thread())
895 abort();
896
897 return NULL;
898}
899
900/* Simultaneous modification to a per-cpu buffer from many threads. */
901void test_percpu_buffer(void)
902{
903 const int num_threads = opt_threads;
904 int i, j, ret;
905 uint64_t sum = 0, expected_sum = 0;
906 struct percpu_buffer buffer;
907 pthread_t test_threads[num_threads];
908 cpu_set_t allowed_cpus;
909
910 memset(&buffer, 0, sizeof(buffer));
911
912 /* Generate list entries for every usable cpu. */
913 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
914 for (i = 0; i < CPU_SETSIZE; i++) {
915 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
916 continue;
917 /* Worse-case is every item in same CPU. */
918 buffer.c[i].array =
919 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
920 BUFFER_ITEM_PER_CPU);
921 assert(buffer.c[i].array);
922 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
923 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
924 struct percpu_buffer_node *node;
925
926 expected_sum += j;
927
928 /*
929 * We could theoretically put the word-sized
930 * "data" directly in the buffer. However, we
931 * want to model objects that would not fit
932 * within a single word, so allocate an object
933 * for each node.
934 */
935 node = malloc(sizeof(*node));
936 assert(node);
937 node->data = j;
938 buffer.c[i].array[j - 1] = node;
939 buffer.c[i].offset++;
940 }
941 }
942
943 for (i = 0; i < num_threads; i++) {
944 ret = pthread_create(&test_threads[i], NULL,
945 test_percpu_buffer_thread, &buffer);
946 if (ret) {
947 errno = ret;
948 perror("pthread_create");
949 abort();
950 }
951 }
952
953 for (i = 0; i < num_threads; i++) {
954 ret = pthread_join(test_threads[i], NULL);
955 if (ret) {
956 errno = ret;
957 perror("pthread_join");
958 abort();
959 }
960 }
961
962 for (i = 0; i < CPU_SETSIZE; i++) {
963 struct percpu_buffer_node *node;
964
965 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
966 continue;
967
968 while ((node = __percpu_buffer_pop(&buffer, i))) {
969 sum += node->data;
970 free(node);
971 }
972 free(buffer.c[i].array);
973 }
974
975 /*
976 * All entries should now be accounted for (unless some external
977 * actor is interfering with our allowed affinity while this
978 * test is running).
979 */
980 assert(sum == expected_sum);
981}
982
983bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
984 struct percpu_memcpy_buffer_node item,
985 int *_cpu)
986{
987 bool result = false;
988 int cpu;
989
990 for (;;) {
991 intptr_t *targetptr_final, newval_final, offset;
992 char *destptr, *srcptr;
993 size_t copylen;
994 int ret;
995
996 cpu = get_current_cpu_id();
997 /* Load offset with single-copy atomicity. */
998 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
999 if (offset == buffer->c[cpu].buflen)
1000 break;
1001 destptr = (char *)&buffer->c[cpu].array[offset];
1002 srcptr = (char *)&item;
1003 /* copylen must be <= 4kB. */
1004 copylen = sizeof(item);
1005 newval_final = offset + 1;
1006 targetptr_final = &buffer->c[cpu].offset;
1007 ret = rseq_cmpeqv_trymemcpy_storev(
1008 opt_mo, RSEQ_PERCPU,
1009 targetptr_final, offset,
1010 destptr, srcptr, copylen,
1011 newval_final, cpu);
1012 if (rseq_likely(!ret)) {
1013 result = true;
1014 break;
1015 }
1016 /* Retry if comparison fails or rseq aborts. */
1017 }
1018 if (_cpu)
1019 *_cpu = cpu;
1020 return result;
1021}
1022
1023bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
1024 struct percpu_memcpy_buffer_node *item,
1025 int *_cpu)
1026{
1027 bool result = false;
1028 int cpu;
1029
1030 for (;;) {
1031 intptr_t *targetptr_final, newval_final, offset;
1032 char *destptr, *srcptr;
1033 size_t copylen;
1034 int ret;
1035
1036 cpu = get_current_cpu_id();
1037 /* Load offset with single-copy atomicity. */
1038 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
1039 if (offset == 0)
1040 break;
1041 destptr = (char *)item;
1042 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
1043 /* copylen must be <= 4kB. */
1044 copylen = sizeof(*item);
1045 newval_final = offset - 1;
1046 targetptr_final = &buffer->c[cpu].offset;
1047 ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,
1048 targetptr_final, offset, destptr, srcptr, copylen,
1049 newval_final, cpu);
1050 if (rseq_likely(!ret)) {
1051 result = true;
1052 break;
1053 }
1054 /* Retry if comparison fails or rseq aborts. */
1055 }
1056 if (_cpu)
1057 *_cpu = cpu;
1058 return result;
1059}
1060
1061/*
1062 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
1063 * only be used on buffers that are not concurrently modified.
1064 */
1065bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
1066 struct percpu_memcpy_buffer_node *item,
1067 int cpu)
1068{
1069 intptr_t offset;
1070
1071 offset = buffer->c[cpu].offset;
1072 if (offset == 0)
1073 return false;
1074 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
1075 buffer->c[cpu].offset = offset - 1;
1076 return true;
1077}
1078
1079void *test_percpu_memcpy_buffer_thread(void *arg)
1080{
1081 long long i, reps;
1082 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
1083
1084 if (!opt_disable_rseq && rseq_register_current_thread())
1085 abort();
1086
1087 reps = opt_reps;
1088 for (i = 0; i < reps; i++) {
1089 struct percpu_memcpy_buffer_node item;
1090 bool result;
1091
1092 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
1093 if (opt_yield)
1094 sched_yield(); /* encourage shuffling */
1095 if (result) {
1096 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
1097 /* Should increase buffer size. */
1098 abort();
1099 }
1100 }
1101 }
1102
1103 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1104 (int) rseq_gettid(), nr_abort, signals_delivered);
1105 if (!opt_disable_rseq && rseq_unregister_current_thread())
1106 abort();
1107
1108 return NULL;
1109}
1110
1111/* Simultaneous modification to a per-cpu buffer from many threads. */
1112void test_percpu_memcpy_buffer(void)
1113{
1114 const int num_threads = opt_threads;
1115 int i, j, ret;
1116 uint64_t sum = 0, expected_sum = 0;
1117 struct percpu_memcpy_buffer buffer;
1118 pthread_t test_threads[num_threads];
1119 cpu_set_t allowed_cpus;
1120
1121 memset(&buffer, 0, sizeof(buffer));
1122
1123 /* Generate list entries for every usable cpu. */
1124 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
1125 for (i = 0; i < CPU_SETSIZE; i++) {
1126 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
1127 continue;
1128 /* Worse-case is every item in same CPU. */
1129 buffer.c[i].array =
1130 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
1131 MEMCPY_BUFFER_ITEM_PER_CPU);
1132 assert(buffer.c[i].array);
1133 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
1134 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1135 expected_sum += 2 * j + 1;
1136
1137 /*
1138 * We could theoretically put the word-sized
1139 * "data" directly in the buffer. However, we
1140 * want to model objects that would not fit
1141 * within a single word, so allocate an object
1142 * for each node.
1143 */
1144 buffer.c[i].array[j - 1].data1 = j;
1145 buffer.c[i].array[j - 1].data2 = j + 1;
1146 buffer.c[i].offset++;
1147 }
1148 }
1149
1150 for (i = 0; i < num_threads; i++) {
1151 ret = pthread_create(&test_threads[i], NULL,
1152 test_percpu_memcpy_buffer_thread,
1153 &buffer);
1154 if (ret) {
1155 errno = ret;
1156 perror("pthread_create");
1157 abort();
1158 }
1159 }
1160
1161 for (i = 0; i < num_threads; i++) {
1162 ret = pthread_join(test_threads[i], NULL);
1163 if (ret) {
1164 errno = ret;
1165 perror("pthread_join");
1166 abort();
1167 }
1168 }
1169
1170 for (i = 0; i < CPU_SETSIZE; i++) {
1171 struct percpu_memcpy_buffer_node item;
1172
1173 if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
1174 continue;
1175
1176 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1177 sum += item.data1;
1178 sum += item.data2;
1179 }
1180 free(buffer.c[i].array);
1181 }
1182
1183 /*
1184 * All entries should now be accounted for (unless some external
1185 * actor is interfering with our allowed affinity while this
1186 * test is running).
1187 */
1188 assert(sum == expected_sum);
1189}
1190
1191static void test_signal_interrupt_handler(int signo)
1192{
1193 signals_delivered++;
1194}
1195
1196static int set_signal_handler(void)
1197{
1198 int ret = 0;
1199 struct sigaction sa;
1200 sigset_t sigset;
1201
1202 ret = sigemptyset(&sigset);
1203 if (ret < 0) {
1204 perror("sigemptyset");
1205 return ret;
1206 }
1207
1208 sa.sa_handler = test_signal_interrupt_handler;
1209 sa.sa_mask = sigset;
1210 sa.sa_flags = 0;
1211 ret = sigaction(SIGUSR1, &sa, NULL);
1212 if (ret < 0) {
1213 perror("sigaction");
1214 return ret;
1215 }
1216
1217 printf_verbose("Signal handler set for SIGUSR1\n");
1218
1219 return ret;
1220}
1221
1222/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
1223#ifdef TEST_MEMBARRIER
1224struct test_membarrier_thread_args {
1225 int stop;
1226 intptr_t percpu_list_ptr;
1227};
1228
1229/* Worker threads modify data in their "active" percpu lists. */
1230void *test_membarrier_worker_thread(void *arg)
1231{
1232 struct test_membarrier_thread_args *args =
1233 (struct test_membarrier_thread_args *)arg;
1234 const int iters = opt_reps;
1235 int i;
1236
1237 if (rseq_register_current_thread()) {
1238 fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1239 errno, strerror(errno));
1240 abort();
1241 }
1242
1243 /* Wait for initialization. */
1244 while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {}
1245
1246 for (i = 0; i < iters; ++i) {
1247 int ret;
1248
1249 do {
1250 int cpu = get_current_cpu_id();
1251
1252 ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU,
1253 &args->percpu_list_ptr,
1254 sizeof(struct percpu_list_entry) * cpu, 1, cpu);
1255 } while (rseq_unlikely(ret));
1256 }
1257
1258 if (rseq_unregister_current_thread()) {
1259 fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1260 errno, strerror(errno));
1261 abort();
1262 }
1263 return NULL;
1264}
1265
1266void test_membarrier_init_percpu_list(struct percpu_list *list)
1267{
1268 int i;
1269
1270 memset(list, 0, sizeof(*list));
1271 for (i = 0; i < CPU_SETSIZE; i++) {
1272 struct percpu_list_node *node;
1273
1274 node = malloc(sizeof(*node));
1275 assert(node);
1276 node->data = 0;
1277 node->next = NULL;
1278 list->c[i].head = node;
1279 }
1280}
1281
1282void test_membarrier_free_percpu_list(struct percpu_list *list)
1283{
1284 int i;
1285
1286 for (i = 0; i < CPU_SETSIZE; i++)
1287 free(list->c[i].head);
1288}
1289
1290/*
1291 * The manager thread swaps per-cpu lists that worker threads see,
1292 * and validates that there are no unexpected modifications.
1293 */
1294void *test_membarrier_manager_thread(void *arg)
1295{
1296 struct test_membarrier_thread_args *args =
1297 (struct test_membarrier_thread_args *)arg;
1298 struct percpu_list list_a, list_b;
1299 intptr_t expect_a = 0, expect_b = 0;
1300 int cpu_a = 0, cpu_b = 0;
1301
1302 if (rseq_register_current_thread()) {
1303 fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1304 errno, strerror(errno));
1305 abort();
1306 }
1307
1308 /* Init lists. */
1309 test_membarrier_init_percpu_list(&list_a);
1310 test_membarrier_init_percpu_list(&list_b);
1311
1312 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);
1313
1314 while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) {
1315 /* list_a is "active". */
1316 cpu_a = rand() % CPU_SETSIZE;
1317 /*
1318 * As list_b is "inactive", we should never see changes
1319 * to list_b.
1320 */
1321 if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) {
1322 fprintf(stderr, "Membarrier test failed\n");
1323 abort();
1324 }
1325
1326 /* Make list_b "active". */
1327 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE);
1328 if (rseq_membarrier_expedited(cpu_a) &&
1329 errno != ENXIO /* missing CPU */) {
1330 perror("sys_membarrier");
1331 abort();
1332 }
1333 /*
1334 * Cpu A should now only modify list_b, so the values
1335 * in list_a should be stable.
1336 */
1337 expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE);
1338
1339 cpu_b = rand() % CPU_SETSIZE;
1340 /*
1341 * As list_a is "inactive", we should never see changes
1342 * to list_a.
1343 */
1344 if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) {
1345 fprintf(stderr, "Membarrier test failed\n");
1346 abort();
1347 }
1348
1349 /* Make list_a "active". */
1350 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);
1351 if (rseq_membarrier_expedited(cpu_b) &&
1352 errno != ENXIO /* missing CPU*/) {
1353 perror("sys_membarrier");
1354 abort();
1355 }
1356 /* Remember a value from list_b. */
1357 expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE);
1358 }
1359
1360 test_membarrier_free_percpu_list(&list_a);
1361 test_membarrier_free_percpu_list(&list_b);
1362
1363 if (rseq_unregister_current_thread()) {
1364 fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1365 errno, strerror(errno));
1366 abort();
1367 }
1368 return NULL;
1369}
1370
1371void test_membarrier(void)
1372{
1373 const int num_threads = opt_threads;
1374 struct test_membarrier_thread_args thread_args;
1375 pthread_t worker_threads[num_threads];
1376 pthread_t manager_thread;
1377 int i, ret;
1378
1379 if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0)) {
1380 perror("sys_membarrier");
1381 abort();
1382 }
1383
1384 thread_args.stop = 0;
1385 thread_args.percpu_list_ptr = 0;
1386 ret = pthread_create(&manager_thread, NULL,
1387 test_membarrier_manager_thread, &thread_args);
1388 if (ret) {
1389 errno = ret;
1390 perror("pthread_create");
1391 abort();
1392 }
1393
1394 for (i = 0; i < num_threads; i++) {
1395 ret = pthread_create(&worker_threads[i], NULL,
1396 test_membarrier_worker_thread, &thread_args);
1397 if (ret) {
1398 errno = ret;
1399 perror("pthread_create");
1400 abort();
1401 }
1402 }
1403
1404
1405 for (i = 0; i < num_threads; i++) {
1406 ret = pthread_join(worker_threads[i], NULL);
1407 if (ret) {
1408 errno = ret;
1409 perror("pthread_join");
1410 abort();
1411 }
1412 }
1413
1414 __atomic_store_n(&thread_args.stop, 1, __ATOMIC_RELEASE);
1415 ret = pthread_join(manager_thread, NULL);
1416 if (ret) {
1417 errno = ret;
1418 perror("pthread_join");
1419 abort();
1420 }
1421}
1422#else /* TEST_MEMBARRIER */
1423void test_membarrier(void)
1424{
1425 fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. "
1426 "Skipping membarrier test.\n");
1427}
1428#endif
1429
1430static void show_usage(int argc, char **argv)
1431{
1432 printf("Usage : %s <OPTIONS>\n",
1433 argv[0]);
1434 printf("OPTIONS:\n");
1435 printf(" [-1 loops] Number of loops for delay injection 1\n");
1436 printf(" [-2 loops] Number of loops for delay injection 2\n");
1437 printf(" [-3 loops] Number of loops for delay injection 3\n");
1438 printf(" [-4 loops] Number of loops for delay injection 4\n");
1439 printf(" [-5 loops] Number of loops for delay injection 5\n");
1440 printf(" [-6 loops] Number of loops for delay injection 6\n");
1441 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1442 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1443 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1444 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1445 printf(" [-y] Yield\n");
1446 printf(" [-k] Kill thread with signal\n");
1447 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1448 printf(" [-t N] Number of threads (default 200)\n");
1449 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1450 printf(" [-d] Disable rseq system call (no initialization)\n");
1451 printf(" [-D M] Disable rseq for each M threads\n");
1452 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
1453 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1454 printf(" [-v] Verbose output.\n");
1455 printf(" [-h] Show this help.\n");
1456 printf("\n");
1457}
1458
1459int main(int argc, char **argv)
1460{
1461 int i;
1462
1463 for (i = 1; i < argc; i++) {
1464 if (argv[i][0] != '-')
1465 continue;
1466 switch (argv[i][1]) {
1467 case '1':
1468 case '2':
1469 case '3':
1470 case '4':
1471 case '5':
1472 case '6':
1473 case '7':
1474 case '8':
1475 case '9':
1476 if (argc < i + 2) {
1477 show_usage(argc, argv);
1478 goto error;
1479 }
1480 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1481 i++;
1482 break;
1483 case 'm':
1484 if (argc < i + 2) {
1485 show_usage(argc, argv);
1486 goto error;
1487 }
1488 opt_modulo = atol(argv[i + 1]);
1489 if (opt_modulo < 0) {
1490 show_usage(argc, argv);
1491 goto error;
1492 }
1493 i++;
1494 break;
1495 case 's':
1496 if (argc < i + 2) {
1497 show_usage(argc, argv);
1498 goto error;
1499 }
1500 opt_sleep = atol(argv[i + 1]);
1501 if (opt_sleep < 0) {
1502 show_usage(argc, argv);
1503 goto error;
1504 }
1505 i++;
1506 break;
1507 case 'y':
1508 opt_yield = 1;
1509 break;
1510 case 'k':
1511 opt_signal = 1;
1512 break;
1513 case 'd':
1514 opt_disable_rseq = 1;
1515 break;
1516 case 'D':
1517 if (argc < i + 2) {
1518 show_usage(argc, argv);
1519 goto error;
1520 }
1521 opt_disable_mod = atol(argv[i + 1]);
1522 if (opt_disable_mod < 0) {
1523 show_usage(argc, argv);
1524 goto error;
1525 }
1526 i++;
1527 break;
1528 case 't':
1529 if (argc < i + 2) {
1530 show_usage(argc, argv);
1531 goto error;
1532 }
1533 opt_threads = atol(argv[i + 1]);
1534 if (opt_threads < 0) {
1535 show_usage(argc, argv);
1536 goto error;
1537 }
1538 i++;
1539 break;
1540 case 'r':
1541 if (argc < i + 2) {
1542 show_usage(argc, argv);
1543 goto error;
1544 }
1545 opt_reps = atoll(argv[i + 1]);
1546 if (opt_reps < 0) {
1547 show_usage(argc, argv);
1548 goto error;
1549 }
1550 i++;
1551 break;
1552 case 'h':
1553 show_usage(argc, argv);
1554 goto end;
1555 case 'T':
1556 if (argc < i + 2) {
1557 show_usage(argc, argv);
1558 goto error;
1559 }
1560 opt_test = *argv[i + 1];
1561 switch (opt_test) {
1562 case 's':
1563 case 'l':
1564 case 'i':
1565 case 'b':
1566 case 'm':
1567 case 'r':
1568 break;
1569 default:
1570 show_usage(argc, argv);
1571 goto error;
1572 }
1573 i++;
1574 break;
1575 case 'v':
1576 verbose = 1;
1577 break;
1578 case 'M':
1579 opt_mo = RSEQ_MO_RELEASE;
1580 break;
1581 default:
1582 show_usage(argc, argv);
1583 goto error;
1584 }
1585 }
1586
1587 loop_cnt_1 = loop_cnt[1];
1588 loop_cnt_2 = loop_cnt[2];
1589 loop_cnt_3 = loop_cnt[3];
1590 loop_cnt_4 = loop_cnt[4];
1591 loop_cnt_5 = loop_cnt[5];
1592 loop_cnt_6 = loop_cnt[6];
1593
1594 if (set_signal_handler())
1595 goto error;
1596
1597 if (!opt_disable_rseq && rseq_register_current_thread())
1598 goto error;
1599 if (!opt_disable_rseq && !rseq_validate_cpu_id()) {
1600 fprintf(stderr, "Error: cpu id getter unavailable\n");
1601 goto error;
1602 }
1603 switch (opt_test) {
1604 case 's':
1605 printf_verbose("spinlock\n");
1606 test_percpu_spinlock();
1607 break;
1608 case 'l':
1609 printf_verbose("linked list\n");
1610 test_percpu_list();
1611 break;
1612 case 'b':
1613 printf_verbose("buffer\n");
1614 test_percpu_buffer();
1615 break;
1616 case 'm':
1617 printf_verbose("memcpy buffer\n");
1618 test_percpu_memcpy_buffer();
1619 break;
1620 case 'i':
1621 printf_verbose("counter increment\n");
1622 test_percpu_inc();
1623 break;
1624 case 'r':
1625 printf_verbose("membarrier\n");
1626 test_membarrier();
1627 break;
1628 }
1629 if (!opt_disable_rseq && rseq_unregister_current_thread())
1630 abort();
1631end:
1632 return 0;
1633
1634error:
1635 return -1;
1636}
1// SPDX-License-Identifier: LGPL-2.1
2#define _GNU_SOURCE
3#include <assert.h>
4#include <pthread.h>
5#include <sched.h>
6#include <stdint.h>
7#include <stdio.h>
8#include <stdlib.h>
9#include <string.h>
10#include <syscall.h>
11#include <unistd.h>
12#include <poll.h>
13#include <sys/types.h>
14#include <signal.h>
15#include <errno.h>
16#include <stddef.h>
17
18static inline pid_t rseq_gettid(void)
19{
20 return syscall(__NR_gettid);
21}
22
23#define NR_INJECT 9
24static int loop_cnt[NR_INJECT + 1];
25
26static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
27static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
28static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
29static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
30static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
31static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
32
33static int opt_modulo, verbose;
34
35static int opt_yield, opt_signal, opt_sleep,
36 opt_disable_rseq, opt_threads = 200,
37 opt_disable_mod = 0, opt_test = 's', opt_mb = 0;
38
39#ifndef RSEQ_SKIP_FASTPATH
40static long long opt_reps = 5000;
41#else
42static long long opt_reps = 100;
43#endif
44
45static __thread __attribute__((tls_model("initial-exec")))
46unsigned int signals_delivered;
47
48#ifndef BENCHMARK
49
50static __thread __attribute__((tls_model("initial-exec"), unused))
51unsigned int yield_mod_cnt, nr_abort;
52
53#define printf_verbose(fmt, ...) \
54 do { \
55 if (verbose) \
56 printf(fmt, ## __VA_ARGS__); \
57 } while (0)
58
59#ifdef __i386__
60
61#define INJECT_ASM_REG "eax"
62
63#define RSEQ_INJECT_CLOBBER \
64 , INJECT_ASM_REG
65
66#define RSEQ_INJECT_ASM(n) \
67 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
68 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
69 "jz 333f\n\t" \
70 "222:\n\t" \
71 "dec %%" INJECT_ASM_REG "\n\t" \
72 "jnz 222b\n\t" \
73 "333:\n\t"
74
75#elif defined(__x86_64__)
76
77#define INJECT_ASM_REG_P "rax"
78#define INJECT_ASM_REG "eax"
79
80#define RSEQ_INJECT_CLOBBER \
81 , INJECT_ASM_REG_P \
82 , INJECT_ASM_REG
83
84#define RSEQ_INJECT_ASM(n) \
85 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
86 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
87 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
88 "jz 333f\n\t" \
89 "222:\n\t" \
90 "dec %%" INJECT_ASM_REG "\n\t" \
91 "jnz 222b\n\t" \
92 "333:\n\t"
93
94#elif defined(__s390__)
95
96#define RSEQ_INJECT_INPUT \
97 , [loop_cnt_1]"m"(loop_cnt[1]) \
98 , [loop_cnt_2]"m"(loop_cnt[2]) \
99 , [loop_cnt_3]"m"(loop_cnt[3]) \
100 , [loop_cnt_4]"m"(loop_cnt[4]) \
101 , [loop_cnt_5]"m"(loop_cnt[5]) \
102 , [loop_cnt_6]"m"(loop_cnt[6])
103
104#define INJECT_ASM_REG "r12"
105
106#define RSEQ_INJECT_CLOBBER \
107 , INJECT_ASM_REG
108
109#define RSEQ_INJECT_ASM(n) \
110 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
111 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
112 "je 333f\n\t" \
113 "222:\n\t" \
114 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
115 "jnz 222b\n\t" \
116 "333:\n\t"
117
118#elif defined(__ARMEL__)
119
120#define RSEQ_INJECT_INPUT \
121 , [loop_cnt_1]"m"(loop_cnt[1]) \
122 , [loop_cnt_2]"m"(loop_cnt[2]) \
123 , [loop_cnt_3]"m"(loop_cnt[3]) \
124 , [loop_cnt_4]"m"(loop_cnt[4]) \
125 , [loop_cnt_5]"m"(loop_cnt[5]) \
126 , [loop_cnt_6]"m"(loop_cnt[6])
127
128#define INJECT_ASM_REG "r4"
129
130#define RSEQ_INJECT_CLOBBER \
131 , INJECT_ASM_REG
132
133#define RSEQ_INJECT_ASM(n) \
134 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
135 "cmp " INJECT_ASM_REG ", #0\n\t" \
136 "beq 333f\n\t" \
137 "222:\n\t" \
138 "subs " INJECT_ASM_REG ", #1\n\t" \
139 "bne 222b\n\t" \
140 "333:\n\t"
141
142#elif defined(__AARCH64EL__)
143
144#define RSEQ_INJECT_INPUT \
145 , [loop_cnt_1] "Qo" (loop_cnt[1]) \
146 , [loop_cnt_2] "Qo" (loop_cnt[2]) \
147 , [loop_cnt_3] "Qo" (loop_cnt[3]) \
148 , [loop_cnt_4] "Qo" (loop_cnt[4]) \
149 , [loop_cnt_5] "Qo" (loop_cnt[5]) \
150 , [loop_cnt_6] "Qo" (loop_cnt[6])
151
152#define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
153
154#define RSEQ_INJECT_ASM(n) \
155 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
156 " cbz " INJECT_ASM_REG ", 333f\n" \
157 "222:\n" \
158 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
159 " cbnz " INJECT_ASM_REG ", 222b\n" \
160 "333:\n"
161
162#elif __PPC__
163
164#define RSEQ_INJECT_INPUT \
165 , [loop_cnt_1]"m"(loop_cnt[1]) \
166 , [loop_cnt_2]"m"(loop_cnt[2]) \
167 , [loop_cnt_3]"m"(loop_cnt[3]) \
168 , [loop_cnt_4]"m"(loop_cnt[4]) \
169 , [loop_cnt_5]"m"(loop_cnt[5]) \
170 , [loop_cnt_6]"m"(loop_cnt[6])
171
172#define INJECT_ASM_REG "r18"
173
174#define RSEQ_INJECT_CLOBBER \
175 , INJECT_ASM_REG
176
177#define RSEQ_INJECT_ASM(n) \
178 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
179 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
180 "beq 333f\n\t" \
181 "222:\n\t" \
182 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
183 "bne 222b\n\t" \
184 "333:\n\t"
185
186#elif defined(__mips__)
187
188#define RSEQ_INJECT_INPUT \
189 , [loop_cnt_1]"m"(loop_cnt[1]) \
190 , [loop_cnt_2]"m"(loop_cnt[2]) \
191 , [loop_cnt_3]"m"(loop_cnt[3]) \
192 , [loop_cnt_4]"m"(loop_cnt[4]) \
193 , [loop_cnt_5]"m"(loop_cnt[5]) \
194 , [loop_cnt_6]"m"(loop_cnt[6])
195
196#define INJECT_ASM_REG "$5"
197
198#define RSEQ_INJECT_CLOBBER \
199 , INJECT_ASM_REG
200
201#define RSEQ_INJECT_ASM(n) \
202 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
203 "beqz " INJECT_ASM_REG ", 333f\n\t" \
204 "222:\n\t" \
205 "addiu " INJECT_ASM_REG ", -1\n\t" \
206 "bnez " INJECT_ASM_REG ", 222b\n\t" \
207 "333:\n\t"
208
209#else
210#error unsupported target
211#endif
212
213#define RSEQ_INJECT_FAILED \
214 nr_abort++;
215
216#define RSEQ_INJECT_C(n) \
217{ \
218 int loc_i, loc_nr_loops = loop_cnt[n]; \
219 \
220 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
221 rseq_barrier(); \
222 } \
223 if (loc_nr_loops == -1 && opt_modulo) { \
224 if (yield_mod_cnt == opt_modulo - 1) { \
225 if (opt_sleep > 0) \
226 poll(NULL, 0, opt_sleep); \
227 if (opt_yield) \
228 sched_yield(); \
229 if (opt_signal) \
230 raise(SIGUSR1); \
231 yield_mod_cnt = 0; \
232 } else { \
233 yield_mod_cnt++; \
234 } \
235 } \
236}
237
238#else
239
240#define printf_verbose(fmt, ...)
241
242#endif /* BENCHMARK */
243
244#include "rseq.h"
245
246struct percpu_lock_entry {
247 intptr_t v;
248} __attribute__((aligned(128)));
249
250struct percpu_lock {
251 struct percpu_lock_entry c[CPU_SETSIZE];
252};
253
254struct test_data_entry {
255 intptr_t count;
256} __attribute__((aligned(128)));
257
258struct spinlock_test_data {
259 struct percpu_lock lock;
260 struct test_data_entry c[CPU_SETSIZE];
261};
262
263struct spinlock_thread_test_data {
264 struct spinlock_test_data *data;
265 long long reps;
266 int reg;
267};
268
269struct inc_test_data {
270 struct test_data_entry c[CPU_SETSIZE];
271};
272
273struct inc_thread_test_data {
274 struct inc_test_data *data;
275 long long reps;
276 int reg;
277};
278
279struct percpu_list_node {
280 intptr_t data;
281 struct percpu_list_node *next;
282};
283
284struct percpu_list_entry {
285 struct percpu_list_node *head;
286} __attribute__((aligned(128)));
287
288struct percpu_list {
289 struct percpu_list_entry c[CPU_SETSIZE];
290};
291
292#define BUFFER_ITEM_PER_CPU 100
293
294struct percpu_buffer_node {
295 intptr_t data;
296};
297
298struct percpu_buffer_entry {
299 intptr_t offset;
300 intptr_t buflen;
301 struct percpu_buffer_node **array;
302} __attribute__((aligned(128)));
303
304struct percpu_buffer {
305 struct percpu_buffer_entry c[CPU_SETSIZE];
306};
307
308#define MEMCPY_BUFFER_ITEM_PER_CPU 100
309
310struct percpu_memcpy_buffer_node {
311 intptr_t data1;
312 uint64_t data2;
313};
314
315struct percpu_memcpy_buffer_entry {
316 intptr_t offset;
317 intptr_t buflen;
318 struct percpu_memcpy_buffer_node *array;
319} __attribute__((aligned(128)));
320
321struct percpu_memcpy_buffer {
322 struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
323};
324
325/* A simple percpu spinlock. Grabs lock on current cpu. */
326static int rseq_this_cpu_lock(struct percpu_lock *lock)
327{
328 int cpu;
329
330 for (;;) {
331 int ret;
332
333 cpu = rseq_cpu_start();
334 ret = rseq_cmpeqv_storev(&lock->c[cpu].v,
335 0, 1, cpu);
336 if (rseq_likely(!ret))
337 break;
338 /* Retry if comparison fails or rseq aborts. */
339 }
340 /*
341 * Acquire semantic when taking lock after control dependency.
342 * Matches rseq_smp_store_release().
343 */
344 rseq_smp_acquire__after_ctrl_dep();
345 return cpu;
346}
347
348static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
349{
350 assert(lock->c[cpu].v == 1);
351 /*
352 * Release lock, with release semantic. Matches
353 * rseq_smp_acquire__after_ctrl_dep().
354 */
355 rseq_smp_store_release(&lock->c[cpu].v, 0);
356}
357
358void *test_percpu_spinlock_thread(void *arg)
359{
360 struct spinlock_thread_test_data *thread_data = arg;
361 struct spinlock_test_data *data = thread_data->data;
362 long long i, reps;
363
364 if (!opt_disable_rseq && thread_data->reg &&
365 rseq_register_current_thread())
366 abort();
367 reps = thread_data->reps;
368 for (i = 0; i < reps; i++) {
369 int cpu = rseq_cpu_start();
370
371 cpu = rseq_this_cpu_lock(&data->lock);
372 data->c[cpu].count++;
373 rseq_percpu_unlock(&data->lock, cpu);
374#ifndef BENCHMARK
375 if (i != 0 && !(i % (reps / 10)))
376 printf_verbose("tid %d: count %lld\n",
377 (int) rseq_gettid(), i);
378#endif
379 }
380 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
381 (int) rseq_gettid(), nr_abort, signals_delivered);
382 if (!opt_disable_rseq && thread_data->reg &&
383 rseq_unregister_current_thread())
384 abort();
385 return NULL;
386}
387
388/*
389 * A simple test which implements a sharded counter using a per-cpu
390 * lock. Obviously real applications might prefer to simply use a
391 * per-cpu increment; however, this is reasonable for a test and the
392 * lock can be extended to synchronize more complicated operations.
393 */
394void test_percpu_spinlock(void)
395{
396 const int num_threads = opt_threads;
397 int i, ret;
398 uint64_t sum;
399 pthread_t test_threads[num_threads];
400 struct spinlock_test_data data;
401 struct spinlock_thread_test_data thread_data[num_threads];
402
403 memset(&data, 0, sizeof(data));
404 for (i = 0; i < num_threads; i++) {
405 thread_data[i].reps = opt_reps;
406 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
407 thread_data[i].reg = 1;
408 else
409 thread_data[i].reg = 0;
410 thread_data[i].data = &data;
411 ret = pthread_create(&test_threads[i], NULL,
412 test_percpu_spinlock_thread,
413 &thread_data[i]);
414 if (ret) {
415 errno = ret;
416 perror("pthread_create");
417 abort();
418 }
419 }
420
421 for (i = 0; i < num_threads; i++) {
422 ret = pthread_join(test_threads[i], NULL);
423 if (ret) {
424 errno = ret;
425 perror("pthread_join");
426 abort();
427 }
428 }
429
430 sum = 0;
431 for (i = 0; i < CPU_SETSIZE; i++)
432 sum += data.c[i].count;
433
434 assert(sum == (uint64_t)opt_reps * num_threads);
435}
436
437void *test_percpu_inc_thread(void *arg)
438{
439 struct inc_thread_test_data *thread_data = arg;
440 struct inc_test_data *data = thread_data->data;
441 long long i, reps;
442
443 if (!opt_disable_rseq && thread_data->reg &&
444 rseq_register_current_thread())
445 abort();
446 reps = thread_data->reps;
447 for (i = 0; i < reps; i++) {
448 int ret;
449
450 do {
451 int cpu;
452
453 cpu = rseq_cpu_start();
454 ret = rseq_addv(&data->c[cpu].count, 1, cpu);
455 } while (rseq_unlikely(ret));
456#ifndef BENCHMARK
457 if (i != 0 && !(i % (reps / 10)))
458 printf_verbose("tid %d: count %lld\n",
459 (int) rseq_gettid(), i);
460#endif
461 }
462 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
463 (int) rseq_gettid(), nr_abort, signals_delivered);
464 if (!opt_disable_rseq && thread_data->reg &&
465 rseq_unregister_current_thread())
466 abort();
467 return NULL;
468}
469
470void test_percpu_inc(void)
471{
472 const int num_threads = opt_threads;
473 int i, ret;
474 uint64_t sum;
475 pthread_t test_threads[num_threads];
476 struct inc_test_data data;
477 struct inc_thread_test_data thread_data[num_threads];
478
479 memset(&data, 0, sizeof(data));
480 for (i = 0; i < num_threads; i++) {
481 thread_data[i].reps = opt_reps;
482 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
483 thread_data[i].reg = 1;
484 else
485 thread_data[i].reg = 0;
486 thread_data[i].data = &data;
487 ret = pthread_create(&test_threads[i], NULL,
488 test_percpu_inc_thread,
489 &thread_data[i]);
490 if (ret) {
491 errno = ret;
492 perror("pthread_create");
493 abort();
494 }
495 }
496
497 for (i = 0; i < num_threads; i++) {
498 ret = pthread_join(test_threads[i], NULL);
499 if (ret) {
500 errno = ret;
501 perror("pthread_join");
502 abort();
503 }
504 }
505
506 sum = 0;
507 for (i = 0; i < CPU_SETSIZE; i++)
508 sum += data.c[i].count;
509
510 assert(sum == (uint64_t)opt_reps * num_threads);
511}
512
513void this_cpu_list_push(struct percpu_list *list,
514 struct percpu_list_node *node,
515 int *_cpu)
516{
517 int cpu;
518
519 for (;;) {
520 intptr_t *targetptr, newval, expect;
521 int ret;
522
523 cpu = rseq_cpu_start();
524 /* Load list->c[cpu].head with single-copy atomicity. */
525 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
526 newval = (intptr_t)node;
527 targetptr = (intptr_t *)&list->c[cpu].head;
528 node->next = (struct percpu_list_node *)expect;
529 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu);
530 if (rseq_likely(!ret))
531 break;
532 /* Retry if comparison fails or rseq aborts. */
533 }
534 if (_cpu)
535 *_cpu = cpu;
536}
537
538/*
539 * Unlike a traditional lock-less linked list; the availability of a
540 * rseq primitive allows us to implement pop without concerns over
541 * ABA-type races.
542 */
543struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
544 int *_cpu)
545{
546 struct percpu_list_node *node = NULL;
547 int cpu;
548
549 for (;;) {
550 struct percpu_list_node *head;
551 intptr_t *targetptr, expectnot, *load;
552 off_t offset;
553 int ret;
554
555 cpu = rseq_cpu_start();
556 targetptr = (intptr_t *)&list->c[cpu].head;
557 expectnot = (intptr_t)NULL;
558 offset = offsetof(struct percpu_list_node, next);
559 load = (intptr_t *)&head;
560 ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot,
561 offset, load, cpu);
562 if (rseq_likely(!ret)) {
563 node = head;
564 break;
565 }
566 if (ret > 0)
567 break;
568 /* Retry if rseq aborts. */
569 }
570 if (_cpu)
571 *_cpu = cpu;
572 return node;
573}
574
575/*
576 * __percpu_list_pop is not safe against concurrent accesses. Should
577 * only be used on lists that are not concurrently modified.
578 */
579struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
580{
581 struct percpu_list_node *node;
582
583 node = list->c[cpu].head;
584 if (!node)
585 return NULL;
586 list->c[cpu].head = node->next;
587 return node;
588}
589
590void *test_percpu_list_thread(void *arg)
591{
592 long long i, reps;
593 struct percpu_list *list = (struct percpu_list *)arg;
594
595 if (!opt_disable_rseq && rseq_register_current_thread())
596 abort();
597
598 reps = opt_reps;
599 for (i = 0; i < reps; i++) {
600 struct percpu_list_node *node;
601
602 node = this_cpu_list_pop(list, NULL);
603 if (opt_yield)
604 sched_yield(); /* encourage shuffling */
605 if (node)
606 this_cpu_list_push(list, node, NULL);
607 }
608
609 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
610 (int) rseq_gettid(), nr_abort, signals_delivered);
611 if (!opt_disable_rseq && rseq_unregister_current_thread())
612 abort();
613
614 return NULL;
615}
616
617/* Simultaneous modification to a per-cpu linked list from many threads. */
618void test_percpu_list(void)
619{
620 const int num_threads = opt_threads;
621 int i, j, ret;
622 uint64_t sum = 0, expected_sum = 0;
623 struct percpu_list list;
624 pthread_t test_threads[num_threads];
625 cpu_set_t allowed_cpus;
626
627 memset(&list, 0, sizeof(list));
628
629 /* Generate list entries for every usable cpu. */
630 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
631 for (i = 0; i < CPU_SETSIZE; i++) {
632 if (!CPU_ISSET(i, &allowed_cpus))
633 continue;
634 for (j = 1; j <= 100; j++) {
635 struct percpu_list_node *node;
636
637 expected_sum += j;
638
639 node = malloc(sizeof(*node));
640 assert(node);
641 node->data = j;
642 node->next = list.c[i].head;
643 list.c[i].head = node;
644 }
645 }
646
647 for (i = 0; i < num_threads; i++) {
648 ret = pthread_create(&test_threads[i], NULL,
649 test_percpu_list_thread, &list);
650 if (ret) {
651 errno = ret;
652 perror("pthread_create");
653 abort();
654 }
655 }
656
657 for (i = 0; i < num_threads; i++) {
658 ret = pthread_join(test_threads[i], NULL);
659 if (ret) {
660 errno = ret;
661 perror("pthread_join");
662 abort();
663 }
664 }
665
666 for (i = 0; i < CPU_SETSIZE; i++) {
667 struct percpu_list_node *node;
668
669 if (!CPU_ISSET(i, &allowed_cpus))
670 continue;
671
672 while ((node = __percpu_list_pop(&list, i))) {
673 sum += node->data;
674 free(node);
675 }
676 }
677
678 /*
679 * All entries should now be accounted for (unless some external
680 * actor is interfering with our allowed affinity while this
681 * test is running).
682 */
683 assert(sum == expected_sum);
684}
685
686bool this_cpu_buffer_push(struct percpu_buffer *buffer,
687 struct percpu_buffer_node *node,
688 int *_cpu)
689{
690 bool result = false;
691 int cpu;
692
693 for (;;) {
694 intptr_t *targetptr_spec, newval_spec;
695 intptr_t *targetptr_final, newval_final;
696 intptr_t offset;
697 int ret;
698
699 cpu = rseq_cpu_start();
700 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
701 if (offset == buffer->c[cpu].buflen)
702 break;
703 newval_spec = (intptr_t)node;
704 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
705 newval_final = offset + 1;
706 targetptr_final = &buffer->c[cpu].offset;
707 if (opt_mb)
708 ret = rseq_cmpeqv_trystorev_storev_release(
709 targetptr_final, offset, targetptr_spec,
710 newval_spec, newval_final, cpu);
711 else
712 ret = rseq_cmpeqv_trystorev_storev(targetptr_final,
713 offset, targetptr_spec, newval_spec,
714 newval_final, cpu);
715 if (rseq_likely(!ret)) {
716 result = true;
717 break;
718 }
719 /* Retry if comparison fails or rseq aborts. */
720 }
721 if (_cpu)
722 *_cpu = cpu;
723 return result;
724}
725
726struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
727 int *_cpu)
728{
729 struct percpu_buffer_node *head;
730 int cpu;
731
732 for (;;) {
733 intptr_t *targetptr, newval;
734 intptr_t offset;
735 int ret;
736
737 cpu = rseq_cpu_start();
738 /* Load offset with single-copy atomicity. */
739 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
740 if (offset == 0) {
741 head = NULL;
742 break;
743 }
744 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
745 newval = offset - 1;
746 targetptr = (intptr_t *)&buffer->c[cpu].offset;
747 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset,
748 (intptr_t *)&buffer->c[cpu].array[offset - 1],
749 (intptr_t)head, newval, cpu);
750 if (rseq_likely(!ret))
751 break;
752 /* Retry if comparison fails or rseq aborts. */
753 }
754 if (_cpu)
755 *_cpu = cpu;
756 return head;
757}
758
759/*
760 * __percpu_buffer_pop is not safe against concurrent accesses. Should
761 * only be used on buffers that are not concurrently modified.
762 */
763struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
764 int cpu)
765{
766 struct percpu_buffer_node *head;
767 intptr_t offset;
768
769 offset = buffer->c[cpu].offset;
770 if (offset == 0)
771 return NULL;
772 head = buffer->c[cpu].array[offset - 1];
773 buffer->c[cpu].offset = offset - 1;
774 return head;
775}
776
777void *test_percpu_buffer_thread(void *arg)
778{
779 long long i, reps;
780 struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
781
782 if (!opt_disable_rseq && rseq_register_current_thread())
783 abort();
784
785 reps = opt_reps;
786 for (i = 0; i < reps; i++) {
787 struct percpu_buffer_node *node;
788
789 node = this_cpu_buffer_pop(buffer, NULL);
790 if (opt_yield)
791 sched_yield(); /* encourage shuffling */
792 if (node) {
793 if (!this_cpu_buffer_push(buffer, node, NULL)) {
794 /* Should increase buffer size. */
795 abort();
796 }
797 }
798 }
799
800 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
801 (int) rseq_gettid(), nr_abort, signals_delivered);
802 if (!opt_disable_rseq && rseq_unregister_current_thread())
803 abort();
804
805 return NULL;
806}
807
808/* Simultaneous modification to a per-cpu buffer from many threads. */
809void test_percpu_buffer(void)
810{
811 const int num_threads = opt_threads;
812 int i, j, ret;
813 uint64_t sum = 0, expected_sum = 0;
814 struct percpu_buffer buffer;
815 pthread_t test_threads[num_threads];
816 cpu_set_t allowed_cpus;
817
818 memset(&buffer, 0, sizeof(buffer));
819
820 /* Generate list entries for every usable cpu. */
821 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
822 for (i = 0; i < CPU_SETSIZE; i++) {
823 if (!CPU_ISSET(i, &allowed_cpus))
824 continue;
825 /* Worse-case is every item in same CPU. */
826 buffer.c[i].array =
827 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
828 BUFFER_ITEM_PER_CPU);
829 assert(buffer.c[i].array);
830 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
831 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
832 struct percpu_buffer_node *node;
833
834 expected_sum += j;
835
836 /*
837 * We could theoretically put the word-sized
838 * "data" directly in the buffer. However, we
839 * want to model objects that would not fit
840 * within a single word, so allocate an object
841 * for each node.
842 */
843 node = malloc(sizeof(*node));
844 assert(node);
845 node->data = j;
846 buffer.c[i].array[j - 1] = node;
847 buffer.c[i].offset++;
848 }
849 }
850
851 for (i = 0; i < num_threads; i++) {
852 ret = pthread_create(&test_threads[i], NULL,
853 test_percpu_buffer_thread, &buffer);
854 if (ret) {
855 errno = ret;
856 perror("pthread_create");
857 abort();
858 }
859 }
860
861 for (i = 0; i < num_threads; i++) {
862 ret = pthread_join(test_threads[i], NULL);
863 if (ret) {
864 errno = ret;
865 perror("pthread_join");
866 abort();
867 }
868 }
869
870 for (i = 0; i < CPU_SETSIZE; i++) {
871 struct percpu_buffer_node *node;
872
873 if (!CPU_ISSET(i, &allowed_cpus))
874 continue;
875
876 while ((node = __percpu_buffer_pop(&buffer, i))) {
877 sum += node->data;
878 free(node);
879 }
880 free(buffer.c[i].array);
881 }
882
883 /*
884 * All entries should now be accounted for (unless some external
885 * actor is interfering with our allowed affinity while this
886 * test is running).
887 */
888 assert(sum == expected_sum);
889}
890
891bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
892 struct percpu_memcpy_buffer_node item,
893 int *_cpu)
894{
895 bool result = false;
896 int cpu;
897
898 for (;;) {
899 intptr_t *targetptr_final, newval_final, offset;
900 char *destptr, *srcptr;
901 size_t copylen;
902 int ret;
903
904 cpu = rseq_cpu_start();
905 /* Load offset with single-copy atomicity. */
906 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
907 if (offset == buffer->c[cpu].buflen)
908 break;
909 destptr = (char *)&buffer->c[cpu].array[offset];
910 srcptr = (char *)&item;
911 /* copylen must be <= 4kB. */
912 copylen = sizeof(item);
913 newval_final = offset + 1;
914 targetptr_final = &buffer->c[cpu].offset;
915 if (opt_mb)
916 ret = rseq_cmpeqv_trymemcpy_storev_release(
917 targetptr_final, offset,
918 destptr, srcptr, copylen,
919 newval_final, cpu);
920 else
921 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
922 offset, destptr, srcptr, copylen,
923 newval_final, cpu);
924 if (rseq_likely(!ret)) {
925 result = true;
926 break;
927 }
928 /* Retry if comparison fails or rseq aborts. */
929 }
930 if (_cpu)
931 *_cpu = cpu;
932 return result;
933}
934
935bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
936 struct percpu_memcpy_buffer_node *item,
937 int *_cpu)
938{
939 bool result = false;
940 int cpu;
941
942 for (;;) {
943 intptr_t *targetptr_final, newval_final, offset;
944 char *destptr, *srcptr;
945 size_t copylen;
946 int ret;
947
948 cpu = rseq_cpu_start();
949 /* Load offset with single-copy atomicity. */
950 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
951 if (offset == 0)
952 break;
953 destptr = (char *)item;
954 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
955 /* copylen must be <= 4kB. */
956 copylen = sizeof(*item);
957 newval_final = offset - 1;
958 targetptr_final = &buffer->c[cpu].offset;
959 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
960 offset, destptr, srcptr, copylen,
961 newval_final, cpu);
962 if (rseq_likely(!ret)) {
963 result = true;
964 break;
965 }
966 /* Retry if comparison fails or rseq aborts. */
967 }
968 if (_cpu)
969 *_cpu = cpu;
970 return result;
971}
972
973/*
974 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
975 * only be used on buffers that are not concurrently modified.
976 */
977bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
978 struct percpu_memcpy_buffer_node *item,
979 int cpu)
980{
981 intptr_t offset;
982
983 offset = buffer->c[cpu].offset;
984 if (offset == 0)
985 return false;
986 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
987 buffer->c[cpu].offset = offset - 1;
988 return true;
989}
990
991void *test_percpu_memcpy_buffer_thread(void *arg)
992{
993 long long i, reps;
994 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
995
996 if (!opt_disable_rseq && rseq_register_current_thread())
997 abort();
998
999 reps = opt_reps;
1000 for (i = 0; i < reps; i++) {
1001 struct percpu_memcpy_buffer_node item;
1002 bool result;
1003
1004 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
1005 if (opt_yield)
1006 sched_yield(); /* encourage shuffling */
1007 if (result) {
1008 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
1009 /* Should increase buffer size. */
1010 abort();
1011 }
1012 }
1013 }
1014
1015 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1016 (int) rseq_gettid(), nr_abort, signals_delivered);
1017 if (!opt_disable_rseq && rseq_unregister_current_thread())
1018 abort();
1019
1020 return NULL;
1021}
1022
1023/* Simultaneous modification to a per-cpu buffer from many threads. */
1024void test_percpu_memcpy_buffer(void)
1025{
1026 const int num_threads = opt_threads;
1027 int i, j, ret;
1028 uint64_t sum = 0, expected_sum = 0;
1029 struct percpu_memcpy_buffer buffer;
1030 pthread_t test_threads[num_threads];
1031 cpu_set_t allowed_cpus;
1032
1033 memset(&buffer, 0, sizeof(buffer));
1034
1035 /* Generate list entries for every usable cpu. */
1036 sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
1037 for (i = 0; i < CPU_SETSIZE; i++) {
1038 if (!CPU_ISSET(i, &allowed_cpus))
1039 continue;
1040 /* Worse-case is every item in same CPU. */
1041 buffer.c[i].array =
1042 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
1043 MEMCPY_BUFFER_ITEM_PER_CPU);
1044 assert(buffer.c[i].array);
1045 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
1046 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1047 expected_sum += 2 * j + 1;
1048
1049 /*
1050 * We could theoretically put the word-sized
1051 * "data" directly in the buffer. However, we
1052 * want to model objects that would not fit
1053 * within a single word, so allocate an object
1054 * for each node.
1055 */
1056 buffer.c[i].array[j - 1].data1 = j;
1057 buffer.c[i].array[j - 1].data2 = j + 1;
1058 buffer.c[i].offset++;
1059 }
1060 }
1061
1062 for (i = 0; i < num_threads; i++) {
1063 ret = pthread_create(&test_threads[i], NULL,
1064 test_percpu_memcpy_buffer_thread,
1065 &buffer);
1066 if (ret) {
1067 errno = ret;
1068 perror("pthread_create");
1069 abort();
1070 }
1071 }
1072
1073 for (i = 0; i < num_threads; i++) {
1074 ret = pthread_join(test_threads[i], NULL);
1075 if (ret) {
1076 errno = ret;
1077 perror("pthread_join");
1078 abort();
1079 }
1080 }
1081
1082 for (i = 0; i < CPU_SETSIZE; i++) {
1083 struct percpu_memcpy_buffer_node item;
1084
1085 if (!CPU_ISSET(i, &allowed_cpus))
1086 continue;
1087
1088 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1089 sum += item.data1;
1090 sum += item.data2;
1091 }
1092 free(buffer.c[i].array);
1093 }
1094
1095 /*
1096 * All entries should now be accounted for (unless some external
1097 * actor is interfering with our allowed affinity while this
1098 * test is running).
1099 */
1100 assert(sum == expected_sum);
1101}
1102
1103static void test_signal_interrupt_handler(int signo)
1104{
1105 signals_delivered++;
1106}
1107
1108static int set_signal_handler(void)
1109{
1110 int ret = 0;
1111 struct sigaction sa;
1112 sigset_t sigset;
1113
1114 ret = sigemptyset(&sigset);
1115 if (ret < 0) {
1116 perror("sigemptyset");
1117 return ret;
1118 }
1119
1120 sa.sa_handler = test_signal_interrupt_handler;
1121 sa.sa_mask = sigset;
1122 sa.sa_flags = 0;
1123 ret = sigaction(SIGUSR1, &sa, NULL);
1124 if (ret < 0) {
1125 perror("sigaction");
1126 return ret;
1127 }
1128
1129 printf_verbose("Signal handler set for SIGUSR1\n");
1130
1131 return ret;
1132}
1133
1134static void show_usage(int argc, char **argv)
1135{
1136 printf("Usage : %s <OPTIONS>\n",
1137 argv[0]);
1138 printf("OPTIONS:\n");
1139 printf(" [-1 loops] Number of loops for delay injection 1\n");
1140 printf(" [-2 loops] Number of loops for delay injection 2\n");
1141 printf(" [-3 loops] Number of loops for delay injection 3\n");
1142 printf(" [-4 loops] Number of loops for delay injection 4\n");
1143 printf(" [-5 loops] Number of loops for delay injection 5\n");
1144 printf(" [-6 loops] Number of loops for delay injection 6\n");
1145 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1146 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1147 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1148 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1149 printf(" [-y] Yield\n");
1150 printf(" [-k] Kill thread with signal\n");
1151 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1152 printf(" [-t N] Number of threads (default 200)\n");
1153 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1154 printf(" [-d] Disable rseq system call (no initialization)\n");
1155 printf(" [-D M] Disable rseq for each M threads\n");
1156 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n");
1157 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1158 printf(" [-v] Verbose output.\n");
1159 printf(" [-h] Show this help.\n");
1160 printf("\n");
1161}
1162
1163int main(int argc, char **argv)
1164{
1165 int i;
1166
1167 for (i = 1; i < argc; i++) {
1168 if (argv[i][0] != '-')
1169 continue;
1170 switch (argv[i][1]) {
1171 case '1':
1172 case '2':
1173 case '3':
1174 case '4':
1175 case '5':
1176 case '6':
1177 case '7':
1178 case '8':
1179 case '9':
1180 if (argc < i + 2) {
1181 show_usage(argc, argv);
1182 goto error;
1183 }
1184 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1185 i++;
1186 break;
1187 case 'm':
1188 if (argc < i + 2) {
1189 show_usage(argc, argv);
1190 goto error;
1191 }
1192 opt_modulo = atol(argv[i + 1]);
1193 if (opt_modulo < 0) {
1194 show_usage(argc, argv);
1195 goto error;
1196 }
1197 i++;
1198 break;
1199 case 's':
1200 if (argc < i + 2) {
1201 show_usage(argc, argv);
1202 goto error;
1203 }
1204 opt_sleep = atol(argv[i + 1]);
1205 if (opt_sleep < 0) {
1206 show_usage(argc, argv);
1207 goto error;
1208 }
1209 i++;
1210 break;
1211 case 'y':
1212 opt_yield = 1;
1213 break;
1214 case 'k':
1215 opt_signal = 1;
1216 break;
1217 case 'd':
1218 opt_disable_rseq = 1;
1219 break;
1220 case 'D':
1221 if (argc < i + 2) {
1222 show_usage(argc, argv);
1223 goto error;
1224 }
1225 opt_disable_mod = atol(argv[i + 1]);
1226 if (opt_disable_mod < 0) {
1227 show_usage(argc, argv);
1228 goto error;
1229 }
1230 i++;
1231 break;
1232 case 't':
1233 if (argc < i + 2) {
1234 show_usage(argc, argv);
1235 goto error;
1236 }
1237 opt_threads = atol(argv[i + 1]);
1238 if (opt_threads < 0) {
1239 show_usage(argc, argv);
1240 goto error;
1241 }
1242 i++;
1243 break;
1244 case 'r':
1245 if (argc < i + 2) {
1246 show_usage(argc, argv);
1247 goto error;
1248 }
1249 opt_reps = atoll(argv[i + 1]);
1250 if (opt_reps < 0) {
1251 show_usage(argc, argv);
1252 goto error;
1253 }
1254 i++;
1255 break;
1256 case 'h':
1257 show_usage(argc, argv);
1258 goto end;
1259 case 'T':
1260 if (argc < i + 2) {
1261 show_usage(argc, argv);
1262 goto error;
1263 }
1264 opt_test = *argv[i + 1];
1265 switch (opt_test) {
1266 case 's':
1267 case 'l':
1268 case 'i':
1269 case 'b':
1270 case 'm':
1271 break;
1272 default:
1273 show_usage(argc, argv);
1274 goto error;
1275 }
1276 i++;
1277 break;
1278 case 'v':
1279 verbose = 1;
1280 break;
1281 case 'M':
1282 opt_mb = 1;
1283 break;
1284 default:
1285 show_usage(argc, argv);
1286 goto error;
1287 }
1288 }
1289
1290 loop_cnt_1 = loop_cnt[1];
1291 loop_cnt_2 = loop_cnt[2];
1292 loop_cnt_3 = loop_cnt[3];
1293 loop_cnt_4 = loop_cnt[4];
1294 loop_cnt_5 = loop_cnt[5];
1295 loop_cnt_6 = loop_cnt[6];
1296
1297 if (set_signal_handler())
1298 goto error;
1299
1300 if (!opt_disable_rseq && rseq_register_current_thread())
1301 goto error;
1302 switch (opt_test) {
1303 case 's':
1304 printf_verbose("spinlock\n");
1305 test_percpu_spinlock();
1306 break;
1307 case 'l':
1308 printf_verbose("linked list\n");
1309 test_percpu_list();
1310 break;
1311 case 'b':
1312 printf_verbose("buffer\n");
1313 test_percpu_buffer();
1314 break;
1315 case 'm':
1316 printf_verbose("memcpy buffer\n");
1317 test_percpu_memcpy_buffer();
1318 break;
1319 case 'i':
1320 printf_verbose("counter increment\n");
1321 test_percpu_inc();
1322 break;
1323 }
1324 if (!opt_disable_rseq && rseq_unregister_current_thread())
1325 abort();
1326end:
1327 return 0;
1328
1329error:
1330 return -1;
1331}