1// SPDX-License-Identifier: GPL-2.0+
  2//
  3// An earlier version of this file appears in the companion webpage for
  4// "Frightening small children and disconcerting grown-ups: Concurrency
  5// in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern,
  6// which is to appear in ASPLOS 2018.
  7
  8// ONCE
  9READ_ONCE(X) __load{once}(X)
 10WRITE_ONCE(X,V) { __store{once}(X,V); }
 11
 12// Release Acquire and friends
 13smp_store_release(X,V) { __store{release}(*X,V); }
 14smp_load_acquire(X) __load{acquire}(*X)
 15rcu_assign_pointer(X,V) { __store{release}(X,V); }
 16rcu_dereference(X) __load{once}(X)
 17
 18// Fences
 19smp_mb() { __fence{mb} ; }
 20smp_rmb() { __fence{rmb} ; }
 21smp_wmb() { __fence{wmb} ; }
 22smp_mb__before_atomic() { __fence{before-atomic} ; }
 23smp_mb__after_atomic() { __fence{after-atomic} ; }
 24smp_mb__after_spinlock() { __fence{after-spinlock} ; }
 25
 26// Exchange
 27xchg(X,V)  __xchg{mb}(X,V)
 28xchg_relaxed(X,V) __xchg{once}(X,V)
 29xchg_release(X,V) __xchg{release}(X,V)
 30xchg_acquire(X,V) __xchg{acquire}(X,V)
 31cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
 32cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
 33cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
 34cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
 35
 36// Spinlocks
 37spin_lock(X) { __lock(X) ; }
 38spin_unlock(X) { __unlock(X) ; }
 39spin_trylock(X) __trylock(X)
 40
 41// RCU
 42rcu_read_lock() { __fence{rcu-lock}; }
 43rcu_read_unlock() { __fence{rcu-unlock};}
 44synchronize_rcu() { __fence{sync-rcu}; }
 45synchronize_rcu_expedited() { __fence{sync-rcu}; }
 46
 47// Atomic
 48atomic_read(X) READ_ONCE(*X)
 49atomic_set(X,V) { WRITE_ONCE(*X,V) ; }
 50atomic_read_acquire(X) smp_load_acquire(X)
 51atomic_set_release(X,V) { smp_store_release(X,V); }
 52
 53atomic_add(V,X) { __atomic_op(X,+,V) ; }
 54atomic_sub(V,X) { __atomic_op(X,-,V) ; }
 55atomic_inc(X)   { __atomic_op(X,+,1) ; }
 56atomic_dec(X)   { __atomic_op(X,-,1) ; }
 57
 58atomic_add_return(V,X) __atomic_op_return{mb}(X,+,V)
 59atomic_add_return_relaxed(V,X) __atomic_op_return{once}(X,+,V)
 60atomic_add_return_acquire(V,X) __atomic_op_return{acquire}(X,+,V)
 61atomic_add_return_release(V,X) __atomic_op_return{release}(X,+,V)
 62atomic_fetch_add(V,X) __atomic_fetch_op{mb}(X,+,V)
 63atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{once}(X,+,V)
 64atomic_fetch_add_acquire(V,X) __atomic_fetch_op{acquire}(X,+,V)
 65atomic_fetch_add_release(V,X) __atomic_fetch_op{release}(X,+,V)
 66
 67atomic_inc_return(X) __atomic_op_return{mb}(X,+,1)
 68atomic_inc_return_relaxed(X) __atomic_op_return{once}(X,+,1)
 69atomic_inc_return_acquire(X) __atomic_op_return{acquire}(X,+,1)
 70atomic_inc_return_release(X) __atomic_op_return{release}(X,+,1)
 71atomic_fetch_inc(X) __atomic_fetch_op{mb}(X,+,1)
 72atomic_fetch_inc_relaxed(X) __atomic_fetch_op{once}(X,+,1)
 73atomic_fetch_inc_acquire(X) __atomic_fetch_op{acquire}(X,+,1)
 74atomic_fetch_inc_release(X) __atomic_fetch_op{release}(X,+,1)
 75
 76atomic_sub_return(V,X) __atomic_op_return{mb}(X,-,V)
 77atomic_sub_return_relaxed(V,X) __atomic_op_return{once}(X,-,V)
 78atomic_sub_return_acquire(V,X) __atomic_op_return{acquire}(X,-,V)
 79atomic_sub_return_release(V,X) __atomic_op_return{release}(X,-,V)
 80atomic_fetch_sub(V,X) __atomic_fetch_op{mb}(X,-,V)
 81atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{once}(X,-,V)
 82atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{acquire}(X,-,V)
 83atomic_fetch_sub_release(V,X) __atomic_fetch_op{release}(X,-,V)
 84
 85atomic_dec_return(X) __atomic_op_return{mb}(X,-,1)
 86atomic_dec_return_relaxed(X) __atomic_op_return{once}(X,-,1)
 87atomic_dec_return_acquire(X) __atomic_op_return{acquire}(X,-,1)
 88atomic_dec_return_release(X) __atomic_op_return{release}(X,-,1)
 89atomic_fetch_dec(X) __atomic_fetch_op{mb}(X,-,1)
 90atomic_fetch_dec_relaxed(X) __atomic_fetch_op{once}(X,-,1)
 91atomic_fetch_dec_acquire(X) __atomic_fetch_op{acquire}(X,-,1)
 92atomic_fetch_dec_release(X) __atomic_fetch_op{release}(X,-,1)
 93
 94atomic_xchg(X,V) __xchg{mb}(X,V)
 95atomic_xchg_relaxed(X,V) __xchg{once}(X,V)
 96atomic_xchg_release(X,V) __xchg{release}(X,V)
 97atomic_xchg_acquire(X,V) __xchg{acquire}(X,V)
 98atomic_cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
 99atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
100atomic_cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
101atomic_cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
102
103atomic_sub_and_test(V,X) __atomic_op_return{mb}(X,-,V) == 0
104atomic_dec_and_test(X)  __atomic_op_return{mb}(X,-,1) == 0
105atomic_inc_and_test(X)  __atomic_op_return{mb}(X,+,1) == 0
106atomic_add_negative(V,X) __atomic_op_return{mb}(X,+,V) < 0