1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * PREEMPT_RT substitution for spin/rw_locks
  4 *
  5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
  6 * resemble the non RT semantics:
  7 *
  8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
  9 *   preserving. The task state is saved before blocking on the underlying
 10 *   rtmutex, and restored when the lock has been acquired. Regular wakeups
 11 *   during that time are redirected to the saved state so no wake up is
 12 *   missed.
 13 *
 14 * - Non RT spin/rwlocks disable preemption and eventually interrupts.
 15 *   Disabling preemption has the side effect of disabling migration and
 16 *   preventing RCU grace periods.
 17 *
 18 *   The RT substitutions explicitly disable migration and take
 19 *   rcu_read_lock() across the lock held section.
 20 */
 21#include <linux/spinlock.h>
 22#include <linux/export.h>
 23
 24#define RT_MUTEX_BUILD_SPINLOCKS
 25#include "rtmutex.c"
 26
 27/*
 28 * __might_resched() skips the state check as rtlocks are state
 29 * preserving. Take RCU nesting into account as spin/read/write_lock() can
 30 * legitimately nest into an RCU read side critical section.
 31 */
 32#define RTLOCK_RESCHED_OFFSETS						\
 33	(rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
 34
 35#define rtlock_might_resched()						\
 36	__might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
 37
 38static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
 39{
 40	lockdep_assert(!current->pi_blocked_on);
 41
 42	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
 43		rtlock_slowlock(rtm);
 44}
 45
 46static __always_inline void __rt_spin_lock(spinlock_t *lock)
 47{
 48	rtlock_might_resched();
 49	rtlock_lock(&lock->lock);
 50	rcu_read_lock();
 51	migrate_disable();
 52}
 53
 54void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU)
 55{
 56	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 57	__rt_spin_lock(lock);
 58}
 59EXPORT_SYMBOL(rt_spin_lock);
 60
 61#ifdef CONFIG_DEBUG_LOCK_ALLOC
 62void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
 63{
 64	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 65	__rt_spin_lock(lock);
 66}
 67EXPORT_SYMBOL(rt_spin_lock_nested);
 68
 69void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
 70				    struct lockdep_map *nest_lock)
 71{
 72	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 73	__rt_spin_lock(lock);
 74}
 75EXPORT_SYMBOL(rt_spin_lock_nest_lock);
 76#endif
 77
 78void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU)
 79{
 80	spin_release(&lock->dep_map, _RET_IP_);
 81	migrate_enable();
 82	rcu_read_unlock();
 83
 84	if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
 85		rt_mutex_slowunlock(&lock->lock);
 86}
 87EXPORT_SYMBOL(rt_spin_unlock);
 88
 89/*
 90 * Wait for the lock to get unlocked: instead of polling for an unlock
 91 * (like raw spinlocks do), lock and unlock, to force the kernel to
 92 * schedule if there's contention:
 93 */
 94void __sched rt_spin_lock_unlock(spinlock_t *lock)
 95{
 96	spin_lock(lock);
 97	spin_unlock(lock);
 98}
 99EXPORT_SYMBOL(rt_spin_lock_unlock);
100
101static __always_inline int __rt_spin_trylock(spinlock_t *lock)
102{
103	int ret = 1;
104
105	if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
106		ret = rt_mutex_slowtrylock(&lock->lock);
107
108	if (ret) {
109		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
110		rcu_read_lock();
111		migrate_disable();
112	}
113	return ret;
114}
115
116int __sched rt_spin_trylock(spinlock_t *lock)
117{
118	return __rt_spin_trylock(lock);
119}
120EXPORT_SYMBOL(rt_spin_trylock);
121
122int __sched rt_spin_trylock_bh(spinlock_t *lock)
123{
124	int ret;
125
126	local_bh_disable();
127	ret = __rt_spin_trylock(lock);
128	if (!ret)
129		local_bh_enable();
130	return ret;
131}
132EXPORT_SYMBOL(rt_spin_trylock_bh);
133
134#ifdef CONFIG_DEBUG_LOCK_ALLOC
135void __rt_spin_lock_init(spinlock_t *lock, const char *name,
136			 struct lock_class_key *key, bool percpu)
137{
138	u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
139
140	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
141	lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
142			      LD_WAIT_INV, type);
143}
144EXPORT_SYMBOL(__rt_spin_lock_init);
145#endif
146
147/*
148 * RT-specific reader/writer locks
149 */
150#define rwbase_set_and_save_current_state(state)	\
151	current_save_and_set_rtlock_wait_state()
152
153#define rwbase_restore_current_state()			\
154	current_restore_rtlock_saved_state()
155
156static __always_inline int
157rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
158{
159	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
160		rtlock_slowlock(rtm);
161	return 0;
162}
163
164static __always_inline int
165rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state,
166			       struct wake_q_head *wake_q)
167{
168	rtlock_slowlock_locked(rtm, wake_q);
169	return 0;
170}
171
172static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
173{
174	if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
175		return;
176
177	rt_mutex_slowunlock(rtm);
178}
179
180static __always_inline int  rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
181{
182	if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
183		return 1;
184
185	return rt_mutex_slowtrylock(rtm);
186}
187
188#define rwbase_signal_pending_state(state, current)	(0)
189
190#define rwbase_pre_schedule()
191
192#define rwbase_schedule()				\
193	schedule_rtlock()
194
195#define rwbase_post_schedule()
196
197#include "rwbase_rt.c"
198/*
199 * The common functions which get wrapped into the rwlock API.
200 */
201int __sched rt_read_trylock(rwlock_t *rwlock)
202{
203	int ret;
204
205	ret = rwbase_read_trylock(&rwlock->rwbase);
206	if (ret) {
207		rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
208		rcu_read_lock();
209		migrate_disable();
210	}
211	return ret;
212}
213EXPORT_SYMBOL(rt_read_trylock);
214
215int __sched rt_write_trylock(rwlock_t *rwlock)
216{
217	int ret;
218
219	ret = rwbase_write_trylock(&rwlock->rwbase);
220	if (ret) {
221		rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
222		rcu_read_lock();
223		migrate_disable();
224	}
225	return ret;
226}
227EXPORT_SYMBOL(rt_write_trylock);
228
229void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU)
230{
231	rtlock_might_resched();
232	rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
233	rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
234	rcu_read_lock();
235	migrate_disable();
236}
237EXPORT_SYMBOL(rt_read_lock);
238
239void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU)
240{
241	rtlock_might_resched();
242	rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
243	rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
244	rcu_read_lock();
245	migrate_disable();
246}
247EXPORT_SYMBOL(rt_write_lock);
248
249#ifdef CONFIG_DEBUG_LOCK_ALLOC
250void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(RCU)
251{
252	rtlock_might_resched();
253	rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
254	rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
255	rcu_read_lock();
256	migrate_disable();
257}
258EXPORT_SYMBOL(rt_write_lock_nested);
259#endif
260
261void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU)
262{
263	rwlock_release(&rwlock->dep_map, _RET_IP_);
264	migrate_enable();
265	rcu_read_unlock();
266	rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
267}
268EXPORT_SYMBOL(rt_read_unlock);
269
270void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU)
271{
272	rwlock_release(&rwlock->dep_map, _RET_IP_);
273	rcu_read_unlock();
274	migrate_enable();
275	rwbase_write_unlock(&rwlock->rwbase);
276}
277EXPORT_SYMBOL(rt_write_unlock);
278
279#ifdef CONFIG_DEBUG_LOCK_ALLOC
280void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
281		      struct lock_class_key *key)
282{
283	debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
284	lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
285}
286EXPORT_SYMBOL(__rt_rwlock_init);
287#endif