Linux Audio

Check our new training course

Loading...
v5.14.15
  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Generic implementation of 64-bit atomics using spinlocks,
  4 * useful on processors that don't have 64-bit atomic instructions.
  5 *
  6 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  7 */
  8#include <linux/types.h>
  9#include <linux/cache.h>
 10#include <linux/spinlock.h>
 11#include <linux/init.h>
 12#include <linux/export.h>
 13#include <linux/atomic.h>
 14
 15/*
 16 * We use a hashed array of spinlocks to provide exclusive access
 17 * to each atomic64_t variable.  Since this is expected to used on
 18 * systems with small numbers of CPUs (<= 4 or so), we use a
 19 * relatively small array of 16 spinlocks to avoid wasting too much
 20 * memory on the spinlock array.
 21 */
 22#define NR_LOCKS	16
 23
 24/*
 25 * Ensure each lock is in a separate cacheline.
 26 */
 27static union {
 28	raw_spinlock_t lock;
 29	char pad[L1_CACHE_BYTES];
 30} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
 31	[0 ... (NR_LOCKS - 1)] = {
 32		.lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
 33	},
 34};
 35
 36static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
 37{
 38	unsigned long addr = (unsigned long) v;
 39
 40	addr >>= L1_CACHE_SHIFT;
 41	addr ^= (addr >> 8) ^ (addr >> 16);
 42	return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
 43}
 44
 45s64 generic_atomic64_read(const atomic64_t *v)
 46{
 47	unsigned long flags;
 48	raw_spinlock_t *lock = lock_addr(v);
 49	s64 val;
 50
 51	raw_spin_lock_irqsave(lock, flags);
 
 52	val = v->counter;
 53	raw_spin_unlock_irqrestore(lock, flags);
 
 54	return val;
 55}
 56EXPORT_SYMBOL(generic_atomic64_read);
 57
 58void generic_atomic64_set(atomic64_t *v, s64 i)
 59{
 60	unsigned long flags;
 61	raw_spinlock_t *lock = lock_addr(v);
 62
 63	raw_spin_lock_irqsave(lock, flags);
 
 64	v->counter = i;
 65	raw_spin_unlock_irqrestore(lock, flags);
 
 66}
 67EXPORT_SYMBOL(generic_atomic64_set);
 68
 69#define ATOMIC64_OP(op, c_op)						\
 70void generic_atomic64_##op(s64 a, atomic64_t *v)			\
 71{									\
 72	unsigned long flags;						\
 73	raw_spinlock_t *lock = lock_addr(v);				\
 74									\
 75	raw_spin_lock_irqsave(lock, flags);				\
 
 76	v->counter c_op a;						\
 77	raw_spin_unlock_irqrestore(lock, flags);			\
 
 78}									\
 79EXPORT_SYMBOL(generic_atomic64_##op);
 80
 81#define ATOMIC64_OP_RETURN(op, c_op)					\
 82s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v)		\
 83{									\
 84	unsigned long flags;						\
 85	raw_spinlock_t *lock = lock_addr(v);				\
 86	s64 val;							\
 87									\
 88	raw_spin_lock_irqsave(lock, flags);				\
 
 89	val = (v->counter c_op a);					\
 90	raw_spin_unlock_irqrestore(lock, flags);			\
 
 91	return val;							\
 92}									\
 93EXPORT_SYMBOL(generic_atomic64_##op##_return);
 94
 95#define ATOMIC64_FETCH_OP(op, c_op)					\
 96s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v)			\
 97{									\
 98	unsigned long flags;						\
 99	raw_spinlock_t *lock = lock_addr(v);				\
100	s64 val;							\
101									\
102	raw_spin_lock_irqsave(lock, flags);				\
 
103	val = v->counter;						\
104	v->counter c_op a;						\
105	raw_spin_unlock_irqrestore(lock, flags);			\
 
106	return val;							\
107}									\
108EXPORT_SYMBOL(generic_atomic64_fetch_##op);
109
110#define ATOMIC64_OPS(op, c_op)						\
111	ATOMIC64_OP(op, c_op)						\
112	ATOMIC64_OP_RETURN(op, c_op)					\
113	ATOMIC64_FETCH_OP(op, c_op)
114
115ATOMIC64_OPS(add, +=)
116ATOMIC64_OPS(sub, -=)
117
118#undef ATOMIC64_OPS
119#define ATOMIC64_OPS(op, c_op)						\
120	ATOMIC64_OP(op, c_op)						\
121	ATOMIC64_OP_RETURN(op, c_op)					\
122	ATOMIC64_FETCH_OP(op, c_op)
123
124ATOMIC64_OPS(and, &=)
125ATOMIC64_OPS(or, |=)
126ATOMIC64_OPS(xor, ^=)
127
128#undef ATOMIC64_OPS
129#undef ATOMIC64_FETCH_OP
130#undef ATOMIC64_OP_RETURN
131#undef ATOMIC64_OP
132
133s64 generic_atomic64_dec_if_positive(atomic64_t *v)
134{
135	unsigned long flags;
136	raw_spinlock_t *lock = lock_addr(v);
137	s64 val;
138
139	raw_spin_lock_irqsave(lock, flags);
 
140	val = v->counter - 1;
141	if (val >= 0)
142		v->counter = val;
143	raw_spin_unlock_irqrestore(lock, flags);
 
144	return val;
145}
146EXPORT_SYMBOL(generic_atomic64_dec_if_positive);
147
148s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
149{
150	unsigned long flags;
151	raw_spinlock_t *lock = lock_addr(v);
152	s64 val;
153
154	raw_spin_lock_irqsave(lock, flags);
 
155	val = v->counter;
156	if (val == o)
157		v->counter = n;
158	raw_spin_unlock_irqrestore(lock, flags);
 
159	return val;
160}
161EXPORT_SYMBOL(generic_atomic64_cmpxchg);
162
163s64 generic_atomic64_xchg(atomic64_t *v, s64 new)
164{
165	unsigned long flags;
166	raw_spinlock_t *lock = lock_addr(v);
167	s64 val;
168
169	raw_spin_lock_irqsave(lock, flags);
 
170	val = v->counter;
171	v->counter = new;
172	raw_spin_unlock_irqrestore(lock, flags);
 
173	return val;
174}
175EXPORT_SYMBOL(generic_atomic64_xchg);
176
177s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
178{
179	unsigned long flags;
180	raw_spinlock_t *lock = lock_addr(v);
181	s64 val;
182
183	raw_spin_lock_irqsave(lock, flags);
 
184	val = v->counter;
185	if (val != u)
186		v->counter += a;
187	raw_spin_unlock_irqrestore(lock, flags);
 
188
189	return val;
190}
191EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);
v6.13.7
  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Generic implementation of 64-bit atomics using spinlocks,
  4 * useful on processors that don't have 64-bit atomic instructions.
  5 *
  6 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  7 */
  8#include <linux/types.h>
  9#include <linux/cache.h>
 10#include <linux/spinlock.h>
 11#include <linux/init.h>
 12#include <linux/export.h>
 13#include <linux/atomic.h>
 14
 15/*
 16 * We use a hashed array of spinlocks to provide exclusive access
 17 * to each atomic64_t variable.  Since this is expected to used on
 18 * systems with small numbers of CPUs (<= 4 or so), we use a
 19 * relatively small array of 16 spinlocks to avoid wasting too much
 20 * memory on the spinlock array.
 21 */
 22#define NR_LOCKS	16
 23
 24/*
 25 * Ensure each lock is in a separate cacheline.
 26 */
 27static union {
 28	arch_spinlock_t lock;
 29	char pad[L1_CACHE_BYTES];
 30} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
 31	[0 ... (NR_LOCKS - 1)] = {
 32		.lock =  __ARCH_SPIN_LOCK_UNLOCKED,
 33	},
 34};
 35
 36static inline arch_spinlock_t *lock_addr(const atomic64_t *v)
 37{
 38	unsigned long addr = (unsigned long) v;
 39
 40	addr >>= L1_CACHE_SHIFT;
 41	addr ^= (addr >> 8) ^ (addr >> 16);
 42	return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
 43}
 44
 45s64 generic_atomic64_read(const atomic64_t *v)
 46{
 47	unsigned long flags;
 48	arch_spinlock_t *lock = lock_addr(v);
 49	s64 val;
 50
 51	local_irq_save(flags);
 52	arch_spin_lock(lock);
 53	val = v->counter;
 54	arch_spin_unlock(lock);
 55	local_irq_restore(flags);
 56	return val;
 57}
 58EXPORT_SYMBOL(generic_atomic64_read);
 59
 60void generic_atomic64_set(atomic64_t *v, s64 i)
 61{
 62	unsigned long flags;
 63	arch_spinlock_t *lock = lock_addr(v);
 64
 65	local_irq_save(flags);
 66	arch_spin_lock(lock);
 67	v->counter = i;
 68	arch_spin_unlock(lock);
 69	local_irq_restore(flags);
 70}
 71EXPORT_SYMBOL(generic_atomic64_set);
 72
 73#define ATOMIC64_OP(op, c_op)						\
 74void generic_atomic64_##op(s64 a, atomic64_t *v)			\
 75{									\
 76	unsigned long flags;						\
 77	arch_spinlock_t *lock = lock_addr(v);				\
 78									\
 79	local_irq_save(flags);						\
 80	arch_spin_lock(lock);						\
 81	v->counter c_op a;						\
 82	arch_spin_unlock(lock);						\
 83	local_irq_restore(flags);					\
 84}									\
 85EXPORT_SYMBOL(generic_atomic64_##op);
 86
 87#define ATOMIC64_OP_RETURN(op, c_op)					\
 88s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v)		\
 89{									\
 90	unsigned long flags;						\
 91	arch_spinlock_t *lock = lock_addr(v);				\
 92	s64 val;							\
 93									\
 94	local_irq_save(flags);						\
 95	arch_spin_lock(lock);						\
 96	val = (v->counter c_op a);					\
 97	arch_spin_unlock(lock);						\
 98	local_irq_restore(flags);					\
 99	return val;							\
100}									\
101EXPORT_SYMBOL(generic_atomic64_##op##_return);
102
103#define ATOMIC64_FETCH_OP(op, c_op)					\
104s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v)			\
105{									\
106	unsigned long flags;						\
107	arch_spinlock_t *lock = lock_addr(v);				\
108	s64 val;							\
109									\
110	local_irq_save(flags);						\
111	arch_spin_lock(lock);						\
112	val = v->counter;						\
113	v->counter c_op a;						\
114	arch_spin_unlock(lock);						\
115	local_irq_restore(flags);					\
116	return val;							\
117}									\
118EXPORT_SYMBOL(generic_atomic64_fetch_##op);
119
120#define ATOMIC64_OPS(op, c_op)						\
121	ATOMIC64_OP(op, c_op)						\
122	ATOMIC64_OP_RETURN(op, c_op)					\
123	ATOMIC64_FETCH_OP(op, c_op)
124
125ATOMIC64_OPS(add, +=)
126ATOMIC64_OPS(sub, -=)
127
128#undef ATOMIC64_OPS
129#define ATOMIC64_OPS(op, c_op)						\
130	ATOMIC64_OP(op, c_op)						\
 
131	ATOMIC64_FETCH_OP(op, c_op)
132
133ATOMIC64_OPS(and, &=)
134ATOMIC64_OPS(or, |=)
135ATOMIC64_OPS(xor, ^=)
136
137#undef ATOMIC64_OPS
138#undef ATOMIC64_FETCH_OP
 
139#undef ATOMIC64_OP
140
141s64 generic_atomic64_dec_if_positive(atomic64_t *v)
142{
143	unsigned long flags;
144	arch_spinlock_t *lock = lock_addr(v);
145	s64 val;
146
147	local_irq_save(flags);
148	arch_spin_lock(lock);
149	val = v->counter - 1;
150	if (val >= 0)
151		v->counter = val;
152	arch_spin_unlock(lock);
153	local_irq_restore(flags);
154	return val;
155}
156EXPORT_SYMBOL(generic_atomic64_dec_if_positive);
157
158s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
159{
160	unsigned long flags;
161	arch_spinlock_t *lock = lock_addr(v);
162	s64 val;
163
164	local_irq_save(flags);
165	arch_spin_lock(lock);
166	val = v->counter;
167	if (val == o)
168		v->counter = n;
169	arch_spin_unlock(lock);
170	local_irq_restore(flags);
171	return val;
172}
173EXPORT_SYMBOL(generic_atomic64_cmpxchg);
174
175s64 generic_atomic64_xchg(atomic64_t *v, s64 new)
176{
177	unsigned long flags;
178	arch_spinlock_t *lock = lock_addr(v);
179	s64 val;
180
181	local_irq_save(flags);
182	arch_spin_lock(lock);
183	val = v->counter;
184	v->counter = new;
185	arch_spin_unlock(lock);
186	local_irq_restore(flags);
187	return val;
188}
189EXPORT_SYMBOL(generic_atomic64_xchg);
190
191s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
192{
193	unsigned long flags;
194	arch_spinlock_t *lock = lock_addr(v);
195	s64 val;
196
197	local_irq_save(flags);
198	arch_spin_lock(lock);
199	val = v->counter;
200	if (val != u)
201		v->counter += a;
202	arch_spin_unlock(lock);
203	local_irq_restore(flags);
204
205	return val;
206}
207EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);