Loading...
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/lockref.h>
4
5#if USE_CMPXCHG_LOCKREF
6
7/*
8 * Note that the "cmpxchg()" reloads the "old" value for the
9 * failure case.
10 */
11#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
12 int retry = 100; \
13 struct lockref old; \
14 BUILD_BUG_ON(sizeof(old) != 8); \
15 old.lock_count = READ_ONCE(lockref->lock_count); \
16 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
17 struct lockref new = old, prev = old; \
18 CODE \
19 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
20 old.lock_count, \
21 new.lock_count); \
22 if (likely(old.lock_count == prev.lock_count)) { \
23 SUCCESS; \
24 } \
25 if (!--retry) \
26 break; \
27 cpu_relax(); \
28 } \
29} while (0)
30
31#else
32
33#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
34
35#endif
36
37/**
38 * lockref_get - Increments reference count unconditionally
39 * @lockref: pointer to lockref structure
40 *
41 * This operation is only valid if you already hold a reference
42 * to the object, so you know the count cannot be zero.
43 */
44void lockref_get(struct lockref *lockref)
45{
46 CMPXCHG_LOOP(
47 new.count++;
48 ,
49 return;
50 );
51
52 spin_lock(&lockref->lock);
53 lockref->count++;
54 spin_unlock(&lockref->lock);
55}
56EXPORT_SYMBOL(lockref_get);
57
58/**
59 * lockref_get_not_zero - Increments count unless the count is 0 or dead
60 * @lockref: pointer to lockref structure
61 * Return: 1 if count updated successfully or 0 if count was zero
62 */
63int lockref_get_not_zero(struct lockref *lockref)
64{
65 int retval;
66
67 CMPXCHG_LOOP(
68 new.count++;
69 if (old.count <= 0)
70 return 0;
71 ,
72 return 1;
73 );
74
75 spin_lock(&lockref->lock);
76 retval = 0;
77 if (lockref->count > 0) {
78 lockref->count++;
79 retval = 1;
80 }
81 spin_unlock(&lockref->lock);
82 return retval;
83}
84EXPORT_SYMBOL(lockref_get_not_zero);
85
86/**
87 * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
88 * @lockref: pointer to lockref structure
89 * Return: 1 if count updated successfully or 0 if count would become zero
90 */
91int lockref_put_not_zero(struct lockref *lockref)
92{
93 int retval;
94
95 CMPXCHG_LOOP(
96 new.count--;
97 if (old.count <= 1)
98 return 0;
99 ,
100 return 1;
101 );
102
103 spin_lock(&lockref->lock);
104 retval = 0;
105 if (lockref->count > 1) {
106 lockref->count--;
107 retval = 1;
108 }
109 spin_unlock(&lockref->lock);
110 return retval;
111}
112EXPORT_SYMBOL(lockref_put_not_zero);
113
114/**
115 * lockref_get_or_lock - Increments count unless the count is 0 or dead
116 * @lockref: pointer to lockref structure
117 * Return: 1 if count updated successfully or 0 if count was zero
118 * and we got the lock instead.
119 */
120int lockref_get_or_lock(struct lockref *lockref)
121{
122 CMPXCHG_LOOP(
123 new.count++;
124 if (old.count <= 0)
125 break;
126 ,
127 return 1;
128 );
129
130 spin_lock(&lockref->lock);
131 if (lockref->count <= 0)
132 return 0;
133 lockref->count++;
134 spin_unlock(&lockref->lock);
135 return 1;
136}
137EXPORT_SYMBOL(lockref_get_or_lock);
138
139/**
140 * lockref_put_return - Decrement reference count if possible
141 * @lockref: pointer to lockref structure
142 *
143 * Decrement the reference count and return the new value.
144 * If the lockref was dead or locked, return an error.
145 */
146int lockref_put_return(struct lockref *lockref)
147{
148 CMPXCHG_LOOP(
149 new.count--;
150 if (old.count <= 0)
151 return -1;
152 ,
153 return new.count;
154 );
155 return -1;
156}
157EXPORT_SYMBOL(lockref_put_return);
158
159/**
160 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
161 * @lockref: pointer to lockref structure
162 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
163 */
164int lockref_put_or_lock(struct lockref *lockref)
165{
166 CMPXCHG_LOOP(
167 new.count--;
168 if (old.count <= 1)
169 break;
170 ,
171 return 1;
172 );
173
174 spin_lock(&lockref->lock);
175 if (lockref->count <= 1)
176 return 0;
177 lockref->count--;
178 spin_unlock(&lockref->lock);
179 return 1;
180}
181EXPORT_SYMBOL(lockref_put_or_lock);
182
183/**
184 * lockref_mark_dead - mark lockref dead
185 * @lockref: pointer to lockref structure
186 */
187void lockref_mark_dead(struct lockref *lockref)
188{
189 assert_spin_locked(&lockref->lock);
190 lockref->count = -128;
191}
192EXPORT_SYMBOL(lockref_mark_dead);
193
194/**
195 * lockref_get_not_dead - Increments count unless the ref is dead
196 * @lockref: pointer to lockref structure
197 * Return: 1 if count updated successfully or 0 if lockref was dead
198 */
199int lockref_get_not_dead(struct lockref *lockref)
200{
201 int retval;
202
203 CMPXCHG_LOOP(
204 new.count++;
205 if (old.count < 0)
206 return 0;
207 ,
208 return 1;
209 );
210
211 spin_lock(&lockref->lock);
212 retval = 0;
213 if (lockref->count >= 0) {
214 lockref->count++;
215 retval = 1;
216 }
217 spin_unlock(&lockref->lock);
218 return retval;
219}
220EXPORT_SYMBOL(lockref_get_not_dead);
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/lockref.h>
4
5#if USE_CMPXCHG_LOCKREF
6
7/*
8 * Note that the "cmpxchg()" reloads the "old" value for the
9 * failure case.
10 */
11#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
12 struct lockref old; \
13 BUILD_BUG_ON(sizeof(old) != 8); \
14 old.lock_count = READ_ONCE(lockref->lock_count); \
15 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
16 struct lockref new = old, prev = old; \
17 CODE \
18 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
19 old.lock_count, \
20 new.lock_count); \
21 if (likely(old.lock_count == prev.lock_count)) { \
22 SUCCESS; \
23 } \
24 cpu_relax(); \
25 } \
26} while (0)
27
28#else
29
30#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
31
32#endif
33
34/**
35 * lockref_get - Increments reference count unconditionally
36 * @lockref: pointer to lockref structure
37 *
38 * This operation is only valid if you already hold a reference
39 * to the object, so you know the count cannot be zero.
40 */
41void lockref_get(struct lockref *lockref)
42{
43 CMPXCHG_LOOP(
44 new.count++;
45 ,
46 return;
47 );
48
49 spin_lock(&lockref->lock);
50 lockref->count++;
51 spin_unlock(&lockref->lock);
52}
53EXPORT_SYMBOL(lockref_get);
54
55/**
56 * lockref_get_not_zero - Increments count unless the count is 0 or dead
57 * @lockref: pointer to lockref structure
58 * Return: 1 if count updated successfully or 0 if count was zero
59 */
60int lockref_get_not_zero(struct lockref *lockref)
61{
62 int retval;
63
64 CMPXCHG_LOOP(
65 new.count++;
66 if (old.count <= 0)
67 return 0;
68 ,
69 return 1;
70 );
71
72 spin_lock(&lockref->lock);
73 retval = 0;
74 if (lockref->count > 0) {
75 lockref->count++;
76 retval = 1;
77 }
78 spin_unlock(&lockref->lock);
79 return retval;
80}
81EXPORT_SYMBOL(lockref_get_not_zero);
82
83/**
84 * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
85 * @lockref: pointer to lockref structure
86 * Return: 1 if count updated successfully or 0 if count would become zero
87 */
88int lockref_put_not_zero(struct lockref *lockref)
89{
90 int retval;
91
92 CMPXCHG_LOOP(
93 new.count--;
94 if (old.count <= 1)
95 return 0;
96 ,
97 return 1;
98 );
99
100 spin_lock(&lockref->lock);
101 retval = 0;
102 if (lockref->count > 1) {
103 lockref->count--;
104 retval = 1;
105 }
106 spin_unlock(&lockref->lock);
107 return retval;
108}
109EXPORT_SYMBOL(lockref_put_not_zero);
110
111/**
112 * lockref_get_or_lock - Increments count unless the count is 0 or dead
113 * @lockref: pointer to lockref structure
114 * Return: 1 if count updated successfully or 0 if count was zero
115 * and we got the lock instead.
116 */
117int lockref_get_or_lock(struct lockref *lockref)
118{
119 CMPXCHG_LOOP(
120 new.count++;
121 if (old.count <= 0)
122 break;
123 ,
124 return 1;
125 );
126
127 spin_lock(&lockref->lock);
128 if (lockref->count <= 0)
129 return 0;
130 lockref->count++;
131 spin_unlock(&lockref->lock);
132 return 1;
133}
134EXPORT_SYMBOL(lockref_get_or_lock);
135
136/**
137 * lockref_put_return - Decrement reference count if possible
138 * @lockref: pointer to lockref structure
139 *
140 * Decrement the reference count and return the new value.
141 * If the lockref was dead or locked, return an error.
142 */
143int lockref_put_return(struct lockref *lockref)
144{
145 CMPXCHG_LOOP(
146 new.count--;
147 if (old.count <= 0)
148 return -1;
149 ,
150 return new.count;
151 );
152 return -1;
153}
154EXPORT_SYMBOL(lockref_put_return);
155
156/**
157 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
158 * @lockref: pointer to lockref structure
159 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
160 */
161int lockref_put_or_lock(struct lockref *lockref)
162{
163 CMPXCHG_LOOP(
164 new.count--;
165 if (old.count <= 1)
166 break;
167 ,
168 return 1;
169 );
170
171 spin_lock(&lockref->lock);
172 if (lockref->count <= 1)
173 return 0;
174 lockref->count--;
175 spin_unlock(&lockref->lock);
176 return 1;
177}
178EXPORT_SYMBOL(lockref_put_or_lock);
179
180/**
181 * lockref_mark_dead - mark lockref dead
182 * @lockref: pointer to lockref structure
183 */
184void lockref_mark_dead(struct lockref *lockref)
185{
186 assert_spin_locked(&lockref->lock);
187 lockref->count = -128;
188}
189EXPORT_SYMBOL(lockref_mark_dead);
190
191/**
192 * lockref_get_not_dead - Increments count unless the ref is dead
193 * @lockref: pointer to lockref structure
194 * Return: 1 if count updated successfully or 0 if lockref was dead
195 */
196int lockref_get_not_dead(struct lockref *lockref)
197{
198 int retval;
199
200 CMPXCHG_LOOP(
201 new.count++;
202 if (old.count < 0)
203 return 0;
204 ,
205 return 1;
206 );
207
208 spin_lock(&lockref->lock);
209 retval = 0;
210 if (lockref->count >= 0) {
211 lockref->count++;
212 retval = 1;
213 }
214 spin_unlock(&lockref->lock);
215 return retval;
216}
217EXPORT_SYMBOL(lockref_get_not_dead);