Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Queue read/write lock
4 *
5 * These use generic atomic and locking routines, but depend on a fair spinlock
6 * implementation in order to be fair themselves. The implementation in
7 * asm-generic/spinlock.h meets these requirements.
8 *
9 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
10 *
11 * Authors: Waiman Long <waiman.long@hp.com>
12 */
13#ifndef __ASM_GENERIC_QRWLOCK_H
14#define __ASM_GENERIC_QRWLOCK_H
15
16#include <linux/atomic.h>
17#include <asm/barrier.h>
18#include <asm/processor.h>
19
20#include <asm-generic/qrwlock_types.h>
21
22/* Must be included from asm/spinlock.h after defining arch_spin_is_locked. */
23
24/*
25 * Writer states & reader shift and bias.
26 */
27#define _QW_WAITING 0x100 /* A writer is waiting */
28#define _QW_LOCKED 0x0ff /* A writer holds the lock */
29#define _QW_WMASK 0x1ff /* Writer mask */
30#define _QR_SHIFT 9 /* Reader count shift */
31#define _QR_BIAS (1U << _QR_SHIFT)
32
33/*
34 * External function declarations
35 */
36extern void queued_read_lock_slowpath(struct qrwlock *lock);
37extern void queued_write_lock_slowpath(struct qrwlock *lock);
38
39/**
40 * queued_read_trylock - try to acquire read lock of a queued rwlock
41 * @lock : Pointer to queued rwlock structure
42 * Return: 1 if lock acquired, 0 if failed
43 */
44static inline int queued_read_trylock(struct qrwlock *lock)
45{
46 int cnts;
47
48 cnts = atomic_read(&lock->cnts);
49 if (likely(!(cnts & _QW_WMASK))) {
50 cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
51 if (likely(!(cnts & _QW_WMASK)))
52 return 1;
53 atomic_sub(_QR_BIAS, &lock->cnts);
54 }
55 return 0;
56}
57
58/**
59 * queued_write_trylock - try to acquire write lock of a queued rwlock
60 * @lock : Pointer to queued rwlock structure
61 * Return: 1 if lock acquired, 0 if failed
62 */
63static inline int queued_write_trylock(struct qrwlock *lock)
64{
65 int cnts;
66
67 cnts = atomic_read(&lock->cnts);
68 if (unlikely(cnts))
69 return 0;
70
71 return likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts,
72 _QW_LOCKED));
73}
74/**
75 * queued_read_lock - acquire read lock of a queued rwlock
76 * @lock: Pointer to queued rwlock structure
77 */
78static inline void queued_read_lock(struct qrwlock *lock)
79{
80 int cnts;
81
82 cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
83 if (likely(!(cnts & _QW_WMASK)))
84 return;
85
86 /* The slowpath will decrement the reader count, if necessary. */
87 queued_read_lock_slowpath(lock);
88}
89
90/**
91 * queued_write_lock - acquire write lock of a queued rwlock
92 * @lock : Pointer to queued rwlock structure
93 */
94static inline void queued_write_lock(struct qrwlock *lock)
95{
96 int cnts = 0;
97 /* Optimize for the unfair lock case where the fair flag is 0. */
98 if (likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED)))
99 return;
100
101 queued_write_lock_slowpath(lock);
102}
103
104/**
105 * queued_read_unlock - release read lock of a queued rwlock
106 * @lock : Pointer to queued rwlock structure
107 */
108static inline void queued_read_unlock(struct qrwlock *lock)
109{
110 /*
111 * Atomically decrement the reader count
112 */
113 (void)atomic_sub_return_release(_QR_BIAS, &lock->cnts);
114}
115
116/**
117 * queued_write_unlock - release write lock of a queued rwlock
118 * @lock : Pointer to queued rwlock structure
119 */
120static inline void queued_write_unlock(struct qrwlock *lock)
121{
122 smp_store_release(&lock->wlocked, 0);
123}
124
125/**
126 * queued_rwlock_is_contended - check if the lock is contended
127 * @lock : Pointer to queued rwlock structure
128 * Return: 1 if lock contended, 0 otherwise
129 */
130static inline int queued_rwlock_is_contended(struct qrwlock *lock)
131{
132 return arch_spin_is_locked(&lock->wait_lock);
133}
134
135/*
136 * Remapping rwlock architecture specific functions to the corresponding
137 * queued rwlock functions.
138 */
139#define arch_read_lock(l) queued_read_lock(l)
140#define arch_write_lock(l) queued_write_lock(l)
141#define arch_read_trylock(l) queued_read_trylock(l)
142#define arch_write_trylock(l) queued_write_trylock(l)
143#define arch_read_unlock(l) queued_read_unlock(l)
144#define arch_write_unlock(l) queued_write_unlock(l)
145#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l)
146
147#endif /* __ASM_GENERIC_QRWLOCK_H */
1/*
2 * Queue read/write lock
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
15 *
16 * Authors: Waiman Long <waiman.long@hp.com>
17 */
18#ifndef __ASM_GENERIC_QRWLOCK_H
19#define __ASM_GENERIC_QRWLOCK_H
20
21#include <linux/atomic.h>
22#include <asm/barrier.h>
23#include <asm/processor.h>
24
25#include <asm-generic/qrwlock_types.h>
26
27/*
28 * Writer states & reader shift and bias
29 */
30#define _QW_WAITING 1 /* A writer is waiting */
31#define _QW_LOCKED 0xff /* A writer holds the lock */
32#define _QW_WMASK 0xff /* Writer mask */
33#define _QR_SHIFT 8 /* Reader count shift */
34#define _QR_BIAS (1U << _QR_SHIFT)
35
36/*
37 * External function declarations
38 */
39extern void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts);
40extern void queued_write_lock_slowpath(struct qrwlock *lock);
41
42/**
43 * queued_read_can_lock- would read_trylock() succeed?
44 * @lock: Pointer to queue rwlock structure
45 */
46static inline int queued_read_can_lock(struct qrwlock *lock)
47{
48 return !(atomic_read(&lock->cnts) & _QW_WMASK);
49}
50
51/**
52 * queued_write_can_lock- would write_trylock() succeed?
53 * @lock: Pointer to queue rwlock structure
54 */
55static inline int queued_write_can_lock(struct qrwlock *lock)
56{
57 return !atomic_read(&lock->cnts);
58}
59
60/**
61 * queued_read_trylock - try to acquire read lock of a queue rwlock
62 * @lock : Pointer to queue rwlock structure
63 * Return: 1 if lock acquired, 0 if failed
64 */
65static inline int queued_read_trylock(struct qrwlock *lock)
66{
67 u32 cnts;
68
69 cnts = atomic_read(&lock->cnts);
70 if (likely(!(cnts & _QW_WMASK))) {
71 cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
72 if (likely(!(cnts & _QW_WMASK)))
73 return 1;
74 atomic_sub(_QR_BIAS, &lock->cnts);
75 }
76 return 0;
77}
78
79/**
80 * queued_write_trylock - try to acquire write lock of a queue rwlock
81 * @lock : Pointer to queue rwlock structure
82 * Return: 1 if lock acquired, 0 if failed
83 */
84static inline int queued_write_trylock(struct qrwlock *lock)
85{
86 u32 cnts;
87
88 cnts = atomic_read(&lock->cnts);
89 if (unlikely(cnts))
90 return 0;
91
92 return likely(atomic_cmpxchg_acquire(&lock->cnts,
93 cnts, cnts | _QW_LOCKED) == cnts);
94}
95/**
96 * queued_read_lock - acquire read lock of a queue rwlock
97 * @lock: Pointer to queue rwlock structure
98 */
99static inline void queued_read_lock(struct qrwlock *lock)
100{
101 u32 cnts;
102
103 cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
104 if (likely(!(cnts & _QW_WMASK)))
105 return;
106
107 /* The slowpath will decrement the reader count, if necessary. */
108 queued_read_lock_slowpath(lock, cnts);
109}
110
111/**
112 * queued_write_lock - acquire write lock of a queue rwlock
113 * @lock : Pointer to queue rwlock structure
114 */
115static inline void queued_write_lock(struct qrwlock *lock)
116{
117 /* Optimize for the unfair lock case where the fair flag is 0. */
118 if (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)
119 return;
120
121 queued_write_lock_slowpath(lock);
122}
123
124/**
125 * queued_read_unlock - release read lock of a queue rwlock
126 * @lock : Pointer to queue rwlock structure
127 */
128static inline void queued_read_unlock(struct qrwlock *lock)
129{
130 /*
131 * Atomically decrement the reader count
132 */
133 (void)atomic_sub_return_release(_QR_BIAS, &lock->cnts);
134}
135
136/**
137 * queued_write_unlock - release write lock of a queue rwlock
138 * @lock : Pointer to queue rwlock structure
139 */
140static inline void queued_write_unlock(struct qrwlock *lock)
141{
142 smp_store_release((u8 *)&lock->cnts, 0);
143}
144
145/*
146 * Remapping rwlock architecture specific functions to the corresponding
147 * queue rwlock functions.
148 */
149#define arch_read_can_lock(l) queued_read_can_lock(l)
150#define arch_write_can_lock(l) queued_write_can_lock(l)
151#define arch_read_lock(l) queued_read_lock(l)
152#define arch_write_lock(l) queued_write_lock(l)
153#define arch_read_trylock(l) queued_read_trylock(l)
154#define arch_write_trylock(l) queued_write_trylock(l)
155#define arch_read_unlock(l) queued_read_unlock(l)
156#define arch_write_unlock(l) queued_write_unlock(l)
157
158#endif /* __ASM_GENERIC_QRWLOCK_H */