Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Variant of atomic_t specialized for reference counts.
4 *
5 * The interface matches the atomic_t interface (to aid in porting) but only
6 * provides the few functions one should use for reference counting.
7 *
8 * It differs in that the counter saturates at UINT_MAX and will not move once
9 * there. This avoids wrapping the counter and causing 'spurious'
10 * use-after-free issues.
11 *
12 * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
13 * and provide only what is strictly required for refcounts.
14 *
15 * The increments are fully relaxed; these will not provide ordering. The
16 * rationale is that whatever is used to obtain the object we're increasing the
17 * reference count on will provide the ordering. For locked data structures,
18 * its the lock acquire, for RCU/lockless data structures its the dependent
19 * load.
20 *
21 * Do note that inc_not_zero() provides a control dependency which will order
22 * future stores against the inc, this ensures we'll never modify the object
23 * if we did not in fact acquire a reference.
24 *
25 * The decrements will provide release order, such that all the prior loads and
26 * stores will be issued before, it also provides a control dependency, which
27 * will order us against the subsequent free().
28 *
29 * The control dependency is against the load of the cmpxchg (ll/sc) that
30 * succeeded. This means the stores aren't fully ordered, but this is fine
31 * because the 1->0 transition indicates no concurrency.
32 *
33 * Note that the allocator is responsible for ordering things between free()
34 * and alloc().
35 *
36 */
37
38#include <linux/refcount.h>
39#include <linux/bug.h>
40
41#ifdef CONFIG_REFCOUNT_FULL
42
43/**
44 * refcount_add_not_zero - add a value to a refcount unless it is 0
45 * @i: the value to add to the refcount
46 * @r: the refcount
47 *
48 * Will saturate at UINT_MAX and WARN.
49 *
50 * Provides no memory ordering, it is assumed the caller has guaranteed the
51 * object memory to be stable (RCU, etc.). It does provide a control dependency
52 * and thereby orders future stores. See the comment on top.
53 *
54 * Use of this function is not recommended for the normal reference counting
55 * use case in which references are taken and released one at a time. In these
56 * cases, refcount_inc(), or one of its variants, should instead be used to
57 * increment a reference count.
58 *
59 * Return: false if the passed refcount is 0, true otherwise
60 */
61bool refcount_add_not_zero(unsigned int i, refcount_t *r)
62{
63 unsigned int new, val = atomic_read(&r->refs);
64
65 do {
66 if (!val)
67 return false;
68
69 if (unlikely(val == UINT_MAX))
70 return true;
71
72 new = val + i;
73 if (new < val)
74 new = UINT_MAX;
75
76 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
77
78 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
79
80 return true;
81}
82EXPORT_SYMBOL(refcount_add_not_zero);
83
84/**
85 * refcount_add - add a value to a refcount
86 * @i: the value to add to the refcount
87 * @r: the refcount
88 *
89 * Similar to atomic_add(), but will saturate at UINT_MAX and WARN.
90 *
91 * Provides no memory ordering, it is assumed the caller has guaranteed the
92 * object memory to be stable (RCU, etc.). It does provide a control dependency
93 * and thereby orders future stores. See the comment on top.
94 *
95 * Use of this function is not recommended for the normal reference counting
96 * use case in which references are taken and released one at a time. In these
97 * cases, refcount_inc(), or one of its variants, should instead be used to
98 * increment a reference count.
99 */
100void refcount_add(unsigned int i, refcount_t *r)
101{
102 WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
103}
104EXPORT_SYMBOL(refcount_add);
105
106/**
107 * refcount_inc_not_zero - increment a refcount unless it is 0
108 * @r: the refcount to increment
109 *
110 * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN.
111 *
112 * Provides no memory ordering, it is assumed the caller has guaranteed the
113 * object memory to be stable (RCU, etc.). It does provide a control dependency
114 * and thereby orders future stores. See the comment on top.
115 *
116 * Return: true if the increment was successful, false otherwise
117 */
118bool refcount_inc_not_zero(refcount_t *r)
119{
120 unsigned int new, val = atomic_read(&r->refs);
121
122 do {
123 new = val + 1;
124
125 if (!val)
126 return false;
127
128 if (unlikely(!new))
129 return true;
130
131 } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
132
133 WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
134
135 return true;
136}
137EXPORT_SYMBOL(refcount_inc_not_zero);
138
139/**
140 * refcount_inc - increment a refcount
141 * @r: the refcount to increment
142 *
143 * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN.
144 *
145 * Provides no memory ordering, it is assumed the caller already has a
146 * reference on the object.
147 *
148 * Will WARN if the refcount is 0, as this represents a possible use-after-free
149 * condition.
150 */
151void refcount_inc(refcount_t *r)
152{
153 WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
154}
155EXPORT_SYMBOL(refcount_inc);
156
157/**
158 * refcount_sub_and_test - subtract from a refcount and test if it is 0
159 * @i: amount to subtract from the refcount
160 * @r: the refcount
161 *
162 * Similar to atomic_dec_and_test(), but it will WARN, return false and
163 * ultimately leak on underflow and will fail to decrement when saturated
164 * at UINT_MAX.
165 *
166 * Provides release memory ordering, such that prior loads and stores are done
167 * before, and provides a control dependency such that free() must come after.
168 * See the comment on top.
169 *
170 * Use of this function is not recommended for the normal reference counting
171 * use case in which references are taken and released one at a time. In these
172 * cases, refcount_dec(), or one of its variants, should instead be used to
173 * decrement a reference count.
174 *
175 * Return: true if the resulting refcount is 0, false otherwise
176 */
177bool refcount_sub_and_test(unsigned int i, refcount_t *r)
178{
179 unsigned int new, val = atomic_read(&r->refs);
180
181 do {
182 if (unlikely(val == UINT_MAX))
183 return false;
184
185 new = val - i;
186 if (new > val) {
187 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
188 return false;
189 }
190
191 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
192
193 return !new;
194}
195EXPORT_SYMBOL(refcount_sub_and_test);
196
197/**
198 * refcount_dec_and_test - decrement a refcount and test if it is 0
199 * @r: the refcount
200 *
201 * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
202 * decrement when saturated at UINT_MAX.
203 *
204 * Provides release memory ordering, such that prior loads and stores are done
205 * before, and provides a control dependency such that free() must come after.
206 * See the comment on top.
207 *
208 * Return: true if the resulting refcount is 0, false otherwise
209 */
210bool refcount_dec_and_test(refcount_t *r)
211{
212 return refcount_sub_and_test(1, r);
213}
214EXPORT_SYMBOL(refcount_dec_and_test);
215
216/**
217 * refcount_dec - decrement a refcount
218 * @r: the refcount
219 *
220 * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
221 * when saturated at UINT_MAX.
222 *
223 * Provides release memory ordering, such that prior loads and stores are done
224 * before.
225 */
226void refcount_dec(refcount_t *r)
227{
228 WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
229}
230EXPORT_SYMBOL(refcount_dec);
231#endif /* CONFIG_REFCOUNT_FULL */
232
233/**
234 * refcount_dec_if_one - decrement a refcount if it is 1
235 * @r: the refcount
236 *
237 * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the
238 * success thereof.
239 *
240 * Like all decrement operations, it provides release memory order and provides
241 * a control dependency.
242 *
243 * It can be used like a try-delete operator; this explicit case is provided
244 * and not cmpxchg in generic, because that would allow implementing unsafe
245 * operations.
246 *
247 * Return: true if the resulting refcount is 0, false otherwise
248 */
249bool refcount_dec_if_one(refcount_t *r)
250{
251 int val = 1;
252
253 return atomic_try_cmpxchg_release(&r->refs, &val, 0);
254}
255EXPORT_SYMBOL(refcount_dec_if_one);
256
257/**
258 * refcount_dec_not_one - decrement a refcount if it is not 1
259 * @r: the refcount
260 *
261 * No atomic_t counterpart, it decrements unless the value is 1, in which case
262 * it will return false.
263 *
264 * Was often done like: atomic_add_unless(&var, -1, 1)
265 *
266 * Return: true if the decrement operation was successful, false otherwise
267 */
268bool refcount_dec_not_one(refcount_t *r)
269{
270 unsigned int new, val = atomic_read(&r->refs);
271
272 do {
273 if (unlikely(val == UINT_MAX))
274 return true;
275
276 if (val == 1)
277 return false;
278
279 new = val - 1;
280 if (new > val) {
281 WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
282 return true;
283 }
284
285 } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
286
287 return true;
288}
289EXPORT_SYMBOL(refcount_dec_not_one);
290
291/**
292 * refcount_dec_and_mutex_lock - return holding mutex if able to decrement
293 * refcount to 0
294 * @r: the refcount
295 * @lock: the mutex to be locked
296 *
297 * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
298 * to decrement when saturated at UINT_MAX.
299 *
300 * Provides release memory ordering, such that prior loads and stores are done
301 * before, and provides a control dependency such that free() must come after.
302 * See the comment on top.
303 *
304 * Return: true and hold mutex if able to decrement refcount to 0, false
305 * otherwise
306 */
307bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock)
308{
309 if (refcount_dec_not_one(r))
310 return false;
311
312 mutex_lock(lock);
313 if (!refcount_dec_and_test(r)) {
314 mutex_unlock(lock);
315 return false;
316 }
317
318 return true;
319}
320EXPORT_SYMBOL(refcount_dec_and_mutex_lock);
321
322/**
323 * refcount_dec_and_lock - return holding spinlock if able to decrement
324 * refcount to 0
325 * @r: the refcount
326 * @lock: the spinlock to be locked
327 *
328 * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
329 * decrement when saturated at UINT_MAX.
330 *
331 * Provides release memory ordering, such that prior loads and stores are done
332 * before, and provides a control dependency such that free() must come after.
333 * See the comment on top.
334 *
335 * Return: true and hold spinlock if able to decrement refcount to 0, false
336 * otherwise
337 */
338bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
339{
340 if (refcount_dec_not_one(r))
341 return false;
342
343 spin_lock(lock);
344 if (!refcount_dec_and_test(r)) {
345 spin_unlock(lock);
346 return false;
347 }
348
349 return true;
350}
351EXPORT_SYMBOL(refcount_dec_and_lock);
352