1// SPDX-License-Identifier: GPL-2.0
  2/* kernel/rwsem.c: R/W semaphores, public implementation
  3 *
  4 * Written by David Howells (dhowells@redhat.com).
  5 * Derived from asm-i386/semaphore.h
  6 */
  7
  8#include <linux/types.h>
  9#include <linux/kernel.h>
 10#include <linux/sched.h>
 11#include <linux/sched/debug.h>
 12#include <linux/export.h>
 13#include <linux/rwsem.h>
 14#include <linux/atomic.h>
 15
 16#include "rwsem.h"
 17
 18/*
 19 * lock for reading
 20 */
 21void __sched down_read(struct rw_semaphore *sem)
 22{
 23	might_sleep();
 24	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
 25
 26	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
 27	rwsem_set_reader_owned(sem);
 28}
 29
 30EXPORT_SYMBOL(down_read);
 31
 32int __sched down_read_killable(struct rw_semaphore *sem)
 33{
 34	might_sleep();
 35	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
 36
 37	if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) {
 38		rwsem_release(&sem->dep_map, 1, _RET_IP_);
 39		return -EINTR;
 40	}
 41
 42	rwsem_set_reader_owned(sem);
 43	return 0;
 44}
 45
 46EXPORT_SYMBOL(down_read_killable);
 47
 48/*
 49 * trylock for reading -- returns 1 if successful, 0 if contention
 50 */
 51int down_read_trylock(struct rw_semaphore *sem)
 52{
 53	int ret = __down_read_trylock(sem);
 54
 55	if (ret == 1) {
 56		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
 57		rwsem_set_reader_owned(sem);
 58	}
 59	return ret;
 60}
 61
 62EXPORT_SYMBOL(down_read_trylock);
 63
 64/*
 65 * lock for writing
 66 */
 67void __sched down_write(struct rw_semaphore *sem)
 68{
 69	might_sleep();
 70	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
 71
 72	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 73	rwsem_set_owner(sem);
 74}
 75
 76EXPORT_SYMBOL(down_write);
 77
 78/*
 79 * lock for writing
 80 */
 81int __sched down_write_killable(struct rw_semaphore *sem)
 82{
 83	might_sleep();
 84	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
 85
 86	if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
 87		rwsem_release(&sem->dep_map, 1, _RET_IP_);
 88		return -EINTR;
 89	}
 90
 91	rwsem_set_owner(sem);
 92	return 0;
 93}
 94
 95EXPORT_SYMBOL(down_write_killable);
 96
 97/*
 98 * trylock for writing -- returns 1 if successful, 0 if contention
 99 */
100int down_write_trylock(struct rw_semaphore *sem)
101{
102	int ret = __down_write_trylock(sem);
103
104	if (ret == 1) {
105		rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
106		rwsem_set_owner(sem);
107	}
108
109	return ret;
110}
111
112EXPORT_SYMBOL(down_write_trylock);
113
114/*
115 * release a read lock
116 */
117void up_read(struct rw_semaphore *sem)
118{
119	rwsem_release(&sem->dep_map, 1, _RET_IP_);
120	DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
121
122	__up_read(sem);
123}
124
125EXPORT_SYMBOL(up_read);
126
127/*
128 * release a write lock
129 */
130void up_write(struct rw_semaphore *sem)
131{
132	rwsem_release(&sem->dep_map, 1, _RET_IP_);
133	DEBUG_RWSEMS_WARN_ON(sem->owner != current);
134
135	rwsem_clear_owner(sem);
136	__up_write(sem);
137}
138
139EXPORT_SYMBOL(up_write);
140
141/*
142 * downgrade write lock to read lock
143 */
144void downgrade_write(struct rw_semaphore *sem)
145{
146	lock_downgrade(&sem->dep_map, _RET_IP_);
147	DEBUG_RWSEMS_WARN_ON(sem->owner != current);
148
149	rwsem_set_reader_owned(sem);
150	__downgrade_write(sem);
151}
152
153EXPORT_SYMBOL(downgrade_write);
154
155#ifdef CONFIG_DEBUG_LOCK_ALLOC
156
157void down_read_nested(struct rw_semaphore *sem, int subclass)
158{
159	might_sleep();
160	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
161
162	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
163	rwsem_set_reader_owned(sem);
164}
165
166EXPORT_SYMBOL(down_read_nested);
167
168void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
169{
170	might_sleep();
171	rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
172
173	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
174	rwsem_set_owner(sem);
175}
176
177EXPORT_SYMBOL(_down_write_nest_lock);
178
179void down_read_non_owner(struct rw_semaphore *sem)
180{
181	might_sleep();
182
183	__down_read(sem);
184}
185
186EXPORT_SYMBOL(down_read_non_owner);
187
188void down_write_nested(struct rw_semaphore *sem, int subclass)
189{
190	might_sleep();
191	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
192
193	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
194	rwsem_set_owner(sem);
195}
196
197EXPORT_SYMBOL(down_write_nested);
198
199int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
200{
201	might_sleep();
202	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
203
204	if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
205		rwsem_release(&sem->dep_map, 1, _RET_IP_);
206		return -EINTR;
207	}
208
209	rwsem_set_owner(sem);
210	return 0;
211}
212
213EXPORT_SYMBOL(down_write_killable_nested);
214
215void up_read_non_owner(struct rw_semaphore *sem)
216{
217	DEBUG_RWSEMS_WARN_ON(sem->owner != RWSEM_READER_OWNED);
218	__up_read(sem);
219}
220
221EXPORT_SYMBOL(up_read_non_owner);
222
223#endif