Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ /* * rseq.h * * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ #ifndef RSEQ_H #define RSEQ_H #include <stdint.h> #include <stdbool.h> #include <pthread.h> #include <signal.h> #include <sched.h> #include <errno.h> #include <stdio.h> #include <stdlib.h> #include <linux/rseq.h> /* * Empty code injection macros, override when testing. * It is important to consider that the ASM injection macros need to be * fully reentrant (e.g. do not modify the stack). */ #ifndef RSEQ_INJECT_ASM #define RSEQ_INJECT_ASM(n) #endif #ifndef RSEQ_INJECT_C #define RSEQ_INJECT_C(n) #endif #ifndef RSEQ_INJECT_INPUT #define RSEQ_INJECT_INPUT #endif #ifndef RSEQ_INJECT_CLOBBER #define RSEQ_INJECT_CLOBBER #endif #ifndef RSEQ_INJECT_FAILED #define RSEQ_INJECT_FAILED #endif extern __thread volatile struct rseq __rseq_abi; extern int __rseq_handled; #define rseq_likely(x) __builtin_expect(!!(x), 1) #define rseq_unlikely(x) __builtin_expect(!!(x), 0) #define rseq_barrier() __asm__ __volatile__("" : : : "memory") #define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) #define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); }) #define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x) #define __rseq_str_1(x) #x #define __rseq_str(x) __rseq_str_1(x) #define rseq_log(fmt, args...) \ fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \ ## args, __func__) #define rseq_bug(fmt, args...) \ do { \ rseq_log(fmt, ##args); \ abort(); \ } while (0) #if defined(__x86_64__) || defined(__i386__) #include <rseq-x86.h> #elif defined(__ARMEL__) #include <rseq-arm.h> #elif defined (__AARCH64EL__) #include <rseq-arm64.h> #elif defined(__PPC__) #include <rseq-ppc.h> #elif defined(__mips__) #include <rseq-mips.h> #elif defined(__s390__) #include <rseq-s390.h> #else #error unsupported target #endif /* * Register rseq for the current thread. This needs to be called once * by any thread which uses restartable sequences, before they start * using restartable sequences, to ensure restartable sequences * succeed. A restartable sequence executed from a non-registered * thread will always fail. */ int rseq_register_current_thread(void); /* * Unregister rseq for current thread. */ int rseq_unregister_current_thread(void); /* * Restartable sequence fallback for reading the current CPU number. */ int32_t rseq_fallback_current_cpu(void); /* * Values returned can be either the current CPU number, -1 (rseq is * uninitialized), or -2 (rseq initialization has failed). */ static inline int32_t rseq_current_cpu_raw(void) { return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id); } /* * Returns a possible CPU number, which is typically the current CPU. * The returned CPU number can be used to prepare for an rseq critical * section, which will confirm whether the cpu number is indeed the * current one, and whether rseq is initialized. * * The CPU number returned by rseq_cpu_start should always be validated * by passing it to a rseq asm sequence, or by comparing it to the * return value of rseq_current_cpu_raw() if the rseq asm sequence * does not need to be invoked. */ static inline uint32_t rseq_cpu_start(void) { return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start); } static inline uint32_t rseq_current_cpu(void) { int32_t cpu; cpu = rseq_current_cpu_raw(); if (rseq_unlikely(cpu < 0)) cpu = rseq_fallback_current_cpu(); return cpu; } static inline void rseq_clear_rseq_cs(void) { #ifdef __LP64__ __rseq_abi.rseq_cs.ptr = 0; #else __rseq_abi.rseq_cs.ptr.ptr32 = 0; #endif } /* * rseq_prepare_unload() should be invoked by each thread executing a rseq * critical section at least once between their last critical section and * library unload of the library defining the rseq critical section (struct * rseq_cs) or the code referred to by the struct rseq_cs start_ip and * post_commit_offset fields. This also applies to use of rseq in code * generated by JIT: rseq_prepare_unload() should be invoked at least once by * each thread executing a rseq critical section before reclaim of the memory * holding the struct rseq_cs or reclaim of the code pointed to by struct * rseq_cs start_ip and post_commit_offset fields. */ static inline void rseq_prepare_unload(void) { rseq_clear_rseq_cs(); } #endif /* RSEQ_H_ */ |