Loading...
1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright IBM Corporation, 2001
6 *
7 * Author: Dipankar Sarma <dipankar@in.ibm.com>
8 *
9 * Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
10 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
11 * Papers:
12 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
13 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
14 *
15 * For detailed explanation of Read-Copy Update mechanism see -
16 * http://lse.sourceforge.net/locking/rcupdate.html
17 *
18 */
19
20#ifndef __LINUX_RCUPDATE_H
21#define __LINUX_RCUPDATE_H
22
23#include <linux/types.h>
24#include <linux/compiler.h>
25#include <linux/atomic.h>
26#include <linux/irqflags.h>
27#include <linux/preempt.h>
28#include <linux/bottom_half.h>
29#include <linux/lockdep.h>
30#include <asm/processor.h>
31#include <linux/cpumask.h>
32
33#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
34#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
35#define ulong2long(a) (*(long *)(&(a)))
36
37/* Exported common interfaces */
38void call_rcu(struct rcu_head *head, rcu_callback_t func);
39void rcu_barrier_tasks(void);
40void rcu_barrier_tasks_rude(void);
41void synchronize_rcu(void);
42
43#ifdef CONFIG_PREEMPT_RCU
44
45void __rcu_read_lock(void);
46void __rcu_read_unlock(void);
47
48/*
49 * Defined as a macro as it is a very low level header included from
50 * areas that don't even know about current. This gives the rcu_read_lock()
51 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
52 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
53 */
54#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
55
56#else /* #ifdef CONFIG_PREEMPT_RCU */
57
58static inline void __rcu_read_lock(void)
59{
60 preempt_disable();
61}
62
63static inline void __rcu_read_unlock(void)
64{
65 preempt_enable();
66}
67
68static inline int rcu_preempt_depth(void)
69{
70 return 0;
71}
72
73#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
74
75/* Internal to kernel */
76void rcu_init(void);
77extern int rcu_scheduler_active __read_mostly;
78void rcu_sched_clock_irq(int user);
79void rcu_report_dead(unsigned int cpu);
80void rcutree_migrate_callbacks(int cpu);
81
82#ifdef CONFIG_RCU_STALL_COMMON
83void rcu_sysrq_start(void);
84void rcu_sysrq_end(void);
85#else /* #ifdef CONFIG_RCU_STALL_COMMON */
86static inline void rcu_sysrq_start(void) { }
87static inline void rcu_sysrq_end(void) { }
88#endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */
89
90#ifdef CONFIG_NO_HZ_FULL
91void rcu_user_enter(void);
92void rcu_user_exit(void);
93#else
94static inline void rcu_user_enter(void) { }
95static inline void rcu_user_exit(void) { }
96#endif /* CONFIG_NO_HZ_FULL */
97
98#ifdef CONFIG_RCU_NOCB_CPU
99void rcu_init_nohz(void);
100#else /* #ifdef CONFIG_RCU_NOCB_CPU */
101static inline void rcu_init_nohz(void) { }
102#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
103
104/**
105 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
106 * @a: Code that RCU needs to pay attention to.
107 *
108 * RCU read-side critical sections are forbidden in the inner idle loop,
109 * that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU
110 * will happily ignore any such read-side critical sections. However,
111 * things like powertop need tracepoints in the inner idle loop.
112 *
113 * This macro provides the way out: RCU_NONIDLE(do_something_with_RCU())
114 * will tell RCU that it needs to pay attention, invoke its argument
115 * (in this example, calling the do_something_with_RCU() function),
116 * and then tell RCU to go back to ignoring this CPU. It is permissible
117 * to nest RCU_NONIDLE() wrappers, but not indefinitely (but the limit is
118 * on the order of a million or so, even on 32-bit systems). It is
119 * not legal to block within RCU_NONIDLE(), nor is it permissible to
120 * transfer control either into or out of RCU_NONIDLE()'s statement.
121 */
122#define RCU_NONIDLE(a) \
123 do { \
124 rcu_irq_enter_irqson(); \
125 do { a; } while (0); \
126 rcu_irq_exit_irqson(); \
127 } while (0)
128
129/*
130 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
131 * This is a macro rather than an inline function to avoid #include hell.
132 */
133#ifdef CONFIG_TASKS_RCU_GENERIC
134
135# ifdef CONFIG_TASKS_RCU
136# define rcu_tasks_classic_qs(t, preempt) \
137 do { \
138 if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
139 WRITE_ONCE((t)->rcu_tasks_holdout, false); \
140 } while (0)
141void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
142void synchronize_rcu_tasks(void);
143# else
144# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
145# define call_rcu_tasks call_rcu
146# define synchronize_rcu_tasks synchronize_rcu
147# endif
148
149# ifdef CONFIG_TASKS_RCU_TRACE
150# define rcu_tasks_trace_qs(t) \
151 do { \
152 if (!likely(READ_ONCE((t)->trc_reader_checked)) && \
153 !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \
154 smp_store_release(&(t)->trc_reader_checked, true); \
155 smp_mb(); /* Readers partitioned by store. */ \
156 } \
157 } while (0)
158# else
159# define rcu_tasks_trace_qs(t) do { } while (0)
160# endif
161
162#define rcu_tasks_qs(t, preempt) \
163do { \
164 rcu_tasks_classic_qs((t), (preempt)); \
165 rcu_tasks_trace_qs((t)); \
166} while (0)
167
168# ifdef CONFIG_TASKS_RUDE_RCU
169void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
170void synchronize_rcu_tasks_rude(void);
171# endif
172
173#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
174void exit_tasks_rcu_start(void);
175void exit_tasks_rcu_finish(void);
176#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
177#define rcu_tasks_qs(t, preempt) do { } while (0)
178#define rcu_note_voluntary_context_switch(t) do { } while (0)
179#define call_rcu_tasks call_rcu
180#define synchronize_rcu_tasks synchronize_rcu
181static inline void exit_tasks_rcu_start(void) { }
182static inline void exit_tasks_rcu_finish(void) { }
183#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
184
185/**
186 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
187 *
188 * This macro resembles cond_resched(), except that it is defined to
189 * report potential quiescent states to RCU-tasks even if the cond_resched()
190 * machinery were to be shut off, as some advocate for PREEMPTION kernels.
191 */
192#define cond_resched_tasks_rcu_qs() \
193do { \
194 rcu_tasks_qs(current, false); \
195 cond_resched(); \
196} while (0)
197
198/*
199 * Infrastructure to implement the synchronize_() primitives in
200 * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
201 */
202
203#if defined(CONFIG_TREE_RCU)
204#include <linux/rcutree.h>
205#elif defined(CONFIG_TINY_RCU)
206#include <linux/rcutiny.h>
207#else
208#error "Unknown RCU implementation specified to kernel configuration"
209#endif
210
211/*
212 * The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
213 * are needed for dynamic initialization and destruction of rcu_head
214 * on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
215 * dynamic initialization and destruction of statically allocated rcu_head
216 * structures. However, rcu_head structures allocated dynamically in the
217 * heap don't need any initialization.
218 */
219#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
220void init_rcu_head(struct rcu_head *head);
221void destroy_rcu_head(struct rcu_head *head);
222void init_rcu_head_on_stack(struct rcu_head *head);
223void destroy_rcu_head_on_stack(struct rcu_head *head);
224#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
225static inline void init_rcu_head(struct rcu_head *head) { }
226static inline void destroy_rcu_head(struct rcu_head *head) { }
227static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
228static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
229#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
230
231#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
232bool rcu_lockdep_current_cpu_online(void);
233#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
234static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
235#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
236
237#ifdef CONFIG_DEBUG_LOCK_ALLOC
238
239static inline void rcu_lock_acquire(struct lockdep_map *map)
240{
241 lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
242}
243
244static inline void rcu_lock_release(struct lockdep_map *map)
245{
246 lock_release(map, _THIS_IP_);
247}
248
249extern struct lockdep_map rcu_lock_map;
250extern struct lockdep_map rcu_bh_lock_map;
251extern struct lockdep_map rcu_sched_lock_map;
252extern struct lockdep_map rcu_callback_map;
253int debug_lockdep_rcu_enabled(void);
254int rcu_read_lock_held(void);
255int rcu_read_lock_bh_held(void);
256int rcu_read_lock_sched_held(void);
257int rcu_read_lock_any_held(void);
258
259#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
260
261# define rcu_lock_acquire(a) do { } while (0)
262# define rcu_lock_release(a) do { } while (0)
263
264static inline int rcu_read_lock_held(void)
265{
266 return 1;
267}
268
269static inline int rcu_read_lock_bh_held(void)
270{
271 return 1;
272}
273
274static inline int rcu_read_lock_sched_held(void)
275{
276 return !preemptible();
277}
278
279static inline int rcu_read_lock_any_held(void)
280{
281 return !preemptible();
282}
283
284#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
285
286#ifdef CONFIG_PROVE_RCU
287
288/**
289 * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
290 * @c: condition to check
291 * @s: informative message
292 */
293#define RCU_LOCKDEP_WARN(c, s) \
294 do { \
295 static bool __section(.data.unlikely) __warned; \
296 if (debug_lockdep_rcu_enabled() && !__warned && (c)) { \
297 __warned = true; \
298 lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
299 } \
300 } while (0)
301
302#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
303static inline void rcu_preempt_sleep_check(void)
304{
305 RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
306 "Illegal context switch in RCU read-side critical section");
307}
308#else /* #ifdef CONFIG_PROVE_RCU */
309static inline void rcu_preempt_sleep_check(void) { }
310#endif /* #else #ifdef CONFIG_PROVE_RCU */
311
312#define rcu_sleep_check() \
313 do { \
314 rcu_preempt_sleep_check(); \
315 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map), \
316 "Illegal context switch in RCU-bh read-side critical section"); \
317 RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map), \
318 "Illegal context switch in RCU-sched read-side critical section"); \
319 } while (0)
320
321#else /* #ifdef CONFIG_PROVE_RCU */
322
323#define RCU_LOCKDEP_WARN(c, s) do { } while (0)
324#define rcu_sleep_check() do { } while (0)
325
326#endif /* #else #ifdef CONFIG_PROVE_RCU */
327
328/*
329 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
330 * and rcu_assign_pointer(). Some of these could be folded into their
331 * callers, but they are left separate in order to ease introduction of
332 * multiple pointers markings to match different RCU implementations
333 * (e.g., __srcu), should this make sense in the future.
334 */
335
336#ifdef __CHECKER__
337#define rcu_check_sparse(p, space) \
338 ((void)(((typeof(*p) space *)p) == p))
339#else /* #ifdef __CHECKER__ */
340#define rcu_check_sparse(p, space)
341#endif /* #else #ifdef __CHECKER__ */
342
343#define __rcu_access_pointer(p, space) \
344({ \
345 typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
346 rcu_check_sparse(p, space); \
347 ((typeof(*p) __force __kernel *)(_________p1)); \
348})
349#define __rcu_dereference_check(p, c, space) \
350({ \
351 /* Dependency order vs. p above. */ \
352 typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \
353 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
354 rcu_check_sparse(p, space); \
355 ((typeof(*p) __force __kernel *)(________p1)); \
356})
357#define __rcu_dereference_protected(p, c, space) \
358({ \
359 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
360 rcu_check_sparse(p, space); \
361 ((typeof(*p) __force __kernel *)(p)); \
362})
363#define rcu_dereference_raw(p) \
364({ \
365 /* Dependency order vs. p above. */ \
366 typeof(p) ________p1 = READ_ONCE(p); \
367 ((typeof(*p) __force __kernel *)(________p1)); \
368})
369
370/**
371 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
372 * @v: The value to statically initialize with.
373 */
374#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
375
376/**
377 * rcu_assign_pointer() - assign to RCU-protected pointer
378 * @p: pointer to assign to
379 * @v: value to assign (publish)
380 *
381 * Assigns the specified value to the specified RCU-protected
382 * pointer, ensuring that any concurrent RCU readers will see
383 * any prior initialization.
384 *
385 * Inserts memory barriers on architectures that require them
386 * (which is most of them), and also prevents the compiler from
387 * reordering the code that initializes the structure after the pointer
388 * assignment. More importantly, this call documents which pointers
389 * will be dereferenced by RCU read-side code.
390 *
391 * In some special cases, you may use RCU_INIT_POINTER() instead
392 * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
393 * to the fact that it does not constrain either the CPU or the compiler.
394 * That said, using RCU_INIT_POINTER() when you should have used
395 * rcu_assign_pointer() is a very bad thing that results in
396 * impossible-to-diagnose memory corruption. So please be careful.
397 * See the RCU_INIT_POINTER() comment header for details.
398 *
399 * Note that rcu_assign_pointer() evaluates each of its arguments only
400 * once, appearances notwithstanding. One of the "extra" evaluations
401 * is in typeof() and the other visible only to sparse (__CHECKER__),
402 * neither of which actually execute the argument. As with most cpp
403 * macros, this execute-arguments-only-once property is important, so
404 * please be careful when making changes to rcu_assign_pointer() and the
405 * other macros that it invokes.
406 */
407#define rcu_assign_pointer(p, v) \
408do { \
409 uintptr_t _r_a_p__v = (uintptr_t)(v); \
410 rcu_check_sparse(p, __rcu); \
411 \
412 if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
413 WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
414 else \
415 smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
416} while (0)
417
418/**
419 * rcu_replace_pointer() - replace an RCU pointer, returning its old value
420 * @rcu_ptr: RCU pointer, whose old value is returned
421 * @ptr: regular pointer
422 * @c: the lockdep conditions under which the dereference will take place
423 *
424 * Perform a replacement, where @rcu_ptr is an RCU-annotated
425 * pointer and @c is the lockdep argument that is passed to the
426 * rcu_dereference_protected() call used to read that pointer. The old
427 * value of @rcu_ptr is returned, and @rcu_ptr is set to @ptr.
428 */
429#define rcu_replace_pointer(rcu_ptr, ptr, c) \
430({ \
431 typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c)); \
432 rcu_assign_pointer((rcu_ptr), (ptr)); \
433 __tmp; \
434})
435
436/**
437 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
438 * @p: The pointer to read
439 *
440 * Return the value of the specified RCU-protected pointer, but omit the
441 * lockdep checks for being in an RCU read-side critical section. This is
442 * useful when the value of this pointer is accessed, but the pointer is
443 * not dereferenced, for example, when testing an RCU-protected pointer
444 * against NULL. Although rcu_access_pointer() may also be used in cases
445 * where update-side locks prevent the value of the pointer from changing,
446 * you should instead use rcu_dereference_protected() for this use case.
447 *
448 * It is also permissible to use rcu_access_pointer() when read-side
449 * access to the pointer was removed at least one grace period ago, as
450 * is the case in the context of the RCU callback that is freeing up
451 * the data, or after a synchronize_rcu() returns. This can be useful
452 * when tearing down multi-linked structures after a grace period
453 * has elapsed.
454 */
455#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
456
457/**
458 * rcu_dereference_check() - rcu_dereference with debug checking
459 * @p: The pointer to read, prior to dereferencing
460 * @c: The conditions under which the dereference will take place
461 *
462 * Do an rcu_dereference(), but check that the conditions under which the
463 * dereference will take place are correct. Typically the conditions
464 * indicate the various locking conditions that should be held at that
465 * point. The check should return true if the conditions are satisfied.
466 * An implicit check for being in an RCU read-side critical section
467 * (rcu_read_lock()) is included.
468 *
469 * For example:
470 *
471 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
472 *
473 * could be used to indicate to lockdep that foo->bar may only be dereferenced
474 * if either rcu_read_lock() is held, or that the lock required to replace
475 * the bar struct at foo->bar is held.
476 *
477 * Note that the list of conditions may also include indications of when a lock
478 * need not be held, for example during initialisation or destruction of the
479 * target struct:
480 *
481 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
482 * atomic_read(&foo->usage) == 0);
483 *
484 * Inserts memory barriers on architectures that require them
485 * (currently only the Alpha), prevents the compiler from refetching
486 * (and from merging fetches), and, more importantly, documents exactly
487 * which pointers are protected by RCU and checks that the pointer is
488 * annotated as __rcu.
489 */
490#define rcu_dereference_check(p, c) \
491 __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)
492
493/**
494 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
495 * @p: The pointer to read, prior to dereferencing
496 * @c: The conditions under which the dereference will take place
497 *
498 * This is the RCU-bh counterpart to rcu_dereference_check().
499 */
500#define rcu_dereference_bh_check(p, c) \
501 __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)
502
503/**
504 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
505 * @p: The pointer to read, prior to dereferencing
506 * @c: The conditions under which the dereference will take place
507 *
508 * This is the RCU-sched counterpart to rcu_dereference_check().
509 */
510#define rcu_dereference_sched_check(p, c) \
511 __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \
512 __rcu)
513
514/*
515 * The tracing infrastructure traces RCU (we want that), but unfortunately
516 * some of the RCU checks causes tracing to lock up the system.
517 *
518 * The no-tracing version of rcu_dereference_raw() must not call
519 * rcu_read_lock_held().
520 */
521#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu)
522
523/**
524 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
525 * @p: The pointer to read, prior to dereferencing
526 * @c: The conditions under which the dereference will take place
527 *
528 * Return the value of the specified RCU-protected pointer, but omit
529 * the READ_ONCE(). This is useful in cases where update-side locks
530 * prevent the value of the pointer from changing. Please note that this
531 * primitive does *not* prevent the compiler from repeating this reference
532 * or combining it with other references, so it should not be used without
533 * protection of appropriate locks.
534 *
535 * This function is only for update-side use. Using this function
536 * when protected only by rcu_read_lock() will result in infrequent
537 * but very ugly failures.
538 */
539#define rcu_dereference_protected(p, c) \
540 __rcu_dereference_protected((p), (c), __rcu)
541
542
543/**
544 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
545 * @p: The pointer to read, prior to dereferencing
546 *
547 * This is a simple wrapper around rcu_dereference_check().
548 */
549#define rcu_dereference(p) rcu_dereference_check(p, 0)
550
551/**
552 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
553 * @p: The pointer to read, prior to dereferencing
554 *
555 * Makes rcu_dereference_check() do the dirty work.
556 */
557#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
558
559/**
560 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
561 * @p: The pointer to read, prior to dereferencing
562 *
563 * Makes rcu_dereference_check() do the dirty work.
564 */
565#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
566
567/**
568 * rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
569 * @p: The pointer to hand off
570 *
571 * This is simply an identity function, but it documents where a pointer
572 * is handed off from RCU to some other synchronization mechanism, for
573 * example, reference counting or locking. In C11, it would map to
574 * kill_dependency(). It could be used as follows::
575 *
576 * rcu_read_lock();
577 * p = rcu_dereference(gp);
578 * long_lived = is_long_lived(p);
579 * if (long_lived) {
580 * if (!atomic_inc_not_zero(p->refcnt))
581 * long_lived = false;
582 * else
583 * p = rcu_pointer_handoff(p);
584 * }
585 * rcu_read_unlock();
586 */
587#define rcu_pointer_handoff(p) (p)
588
589/**
590 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
591 *
592 * When synchronize_rcu() is invoked on one CPU while other CPUs
593 * are within RCU read-side critical sections, then the
594 * synchronize_rcu() is guaranteed to block until after all the other
595 * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
596 * on one CPU while other CPUs are within RCU read-side critical
597 * sections, invocation of the corresponding RCU callback is deferred
598 * until after the all the other CPUs exit their critical sections.
599 *
600 * Note, however, that RCU callbacks are permitted to run concurrently
601 * with new RCU read-side critical sections. One way that this can happen
602 * is via the following sequence of events: (1) CPU 0 enters an RCU
603 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
604 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
605 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
606 * callback is invoked. This is legal, because the RCU read-side critical
607 * section that was running concurrently with the call_rcu() (and which
608 * therefore might be referencing something that the corresponding RCU
609 * callback would free up) has completed before the corresponding
610 * RCU callback is invoked.
611 *
612 * RCU read-side critical sections may be nested. Any deferred actions
613 * will be deferred until the outermost RCU read-side critical section
614 * completes.
615 *
616 * You can avoid reading and understanding the next paragraph by
617 * following this rule: don't put anything in an rcu_read_lock() RCU
618 * read-side critical section that would block in a !PREEMPTION kernel.
619 * But if you want the full story, read on!
620 *
621 * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
622 * it is illegal to block while in an RCU read-side critical section.
623 * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
624 * kernel builds, RCU read-side critical sections may be preempted,
625 * but explicit blocking is illegal. Finally, in preemptible RCU
626 * implementations in real-time (with -rt patchset) kernel builds, RCU
627 * read-side critical sections may be preempted and they may also block, but
628 * only when acquiring spinlocks that are subject to priority inheritance.
629 */
630static __always_inline void rcu_read_lock(void)
631{
632 __rcu_read_lock();
633 __acquire(RCU);
634 rcu_lock_acquire(&rcu_lock_map);
635 RCU_LOCKDEP_WARN(!rcu_is_watching(),
636 "rcu_read_lock() used illegally while idle");
637}
638
639/*
640 * So where is rcu_write_lock()? It does not exist, as there is no
641 * way for writers to lock out RCU readers. This is a feature, not
642 * a bug -- this property is what provides RCU's performance benefits.
643 * Of course, writers must coordinate with each other. The normal
644 * spinlock primitives work well for this, but any other technique may be
645 * used as well. RCU does not care how the writers keep out of each
646 * others' way, as long as they do so.
647 */
648
649/**
650 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
651 *
652 * In most situations, rcu_read_unlock() is immune from deadlock.
653 * However, in kernels built with CONFIG_RCU_BOOST, rcu_read_unlock()
654 * is responsible for deboosting, which it does via rt_mutex_unlock().
655 * Unfortunately, this function acquires the scheduler's runqueue and
656 * priority-inheritance spinlocks. This means that deadlock could result
657 * if the caller of rcu_read_unlock() already holds one of these locks or
658 * any lock that is ever acquired while holding them.
659 *
660 * That said, RCU readers are never priority boosted unless they were
661 * preempted. Therefore, one way to avoid deadlock is to make sure
662 * that preemption never happens within any RCU read-side critical
663 * section whose outermost rcu_read_unlock() is called with one of
664 * rt_mutex_unlock()'s locks held. Such preemption can be avoided in
665 * a number of ways, for example, by invoking preempt_disable() before
666 * critical section's outermost rcu_read_lock().
667 *
668 * Given that the set of locks acquired by rt_mutex_unlock() might change
669 * at any time, a somewhat more future-proofed approach is to make sure
670 * that that preemption never happens within any RCU read-side critical
671 * section whose outermost rcu_read_unlock() is called with irqs disabled.
672 * This approach relies on the fact that rt_mutex_unlock() currently only
673 * acquires irq-disabled locks.
674 *
675 * The second of these two approaches is best in most situations,
676 * however, the first approach can also be useful, at least to those
677 * developers willing to keep abreast of the set of locks acquired by
678 * rt_mutex_unlock().
679 *
680 * See rcu_read_lock() for more information.
681 */
682static inline void rcu_read_unlock(void)
683{
684 RCU_LOCKDEP_WARN(!rcu_is_watching(),
685 "rcu_read_unlock() used illegally while idle");
686 __release(RCU);
687 __rcu_read_unlock();
688 rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
689}
690
691/**
692 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
693 *
694 * This is equivalent of rcu_read_lock(), but also disables softirqs.
695 * Note that anything else that disables softirqs can also serve as
696 * an RCU read-side critical section.
697 *
698 * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
699 * must occur in the same context, for example, it is illegal to invoke
700 * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
701 * was invoked from some other task.
702 */
703static inline void rcu_read_lock_bh(void)
704{
705 local_bh_disable();
706 __acquire(RCU_BH);
707 rcu_lock_acquire(&rcu_bh_lock_map);
708 RCU_LOCKDEP_WARN(!rcu_is_watching(),
709 "rcu_read_lock_bh() used illegally while idle");
710}
711
712/*
713 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
714 *
715 * See rcu_read_lock_bh() for more information.
716 */
717static inline void rcu_read_unlock_bh(void)
718{
719 RCU_LOCKDEP_WARN(!rcu_is_watching(),
720 "rcu_read_unlock_bh() used illegally while idle");
721 rcu_lock_release(&rcu_bh_lock_map);
722 __release(RCU_BH);
723 local_bh_enable();
724}
725
726/**
727 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
728 *
729 * This is equivalent of rcu_read_lock(), but disables preemption.
730 * Read-side critical sections can also be introduced by anything else
731 * that disables preemption, including local_irq_disable() and friends.
732 *
733 * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
734 * must occur in the same context, for example, it is illegal to invoke
735 * rcu_read_unlock_sched() from process context if the matching
736 * rcu_read_lock_sched() was invoked from an NMI handler.
737 */
738static inline void rcu_read_lock_sched(void)
739{
740 preempt_disable();
741 __acquire(RCU_SCHED);
742 rcu_lock_acquire(&rcu_sched_lock_map);
743 RCU_LOCKDEP_WARN(!rcu_is_watching(),
744 "rcu_read_lock_sched() used illegally while idle");
745}
746
747/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
748static inline notrace void rcu_read_lock_sched_notrace(void)
749{
750 preempt_disable_notrace();
751 __acquire(RCU_SCHED);
752}
753
754/*
755 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
756 *
757 * See rcu_read_lock_sched for more information.
758 */
759static inline void rcu_read_unlock_sched(void)
760{
761 RCU_LOCKDEP_WARN(!rcu_is_watching(),
762 "rcu_read_unlock_sched() used illegally while idle");
763 rcu_lock_release(&rcu_sched_lock_map);
764 __release(RCU_SCHED);
765 preempt_enable();
766}
767
768/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
769static inline notrace void rcu_read_unlock_sched_notrace(void)
770{
771 __release(RCU_SCHED);
772 preempt_enable_notrace();
773}
774
775/**
776 * RCU_INIT_POINTER() - initialize an RCU protected pointer
777 * @p: The pointer to be initialized.
778 * @v: The value to initialized the pointer to.
779 *
780 * Initialize an RCU-protected pointer in special cases where readers
781 * do not need ordering constraints on the CPU or the compiler. These
782 * special cases are:
783 *
784 * 1. This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
785 * 2. The caller has taken whatever steps are required to prevent
786 * RCU readers from concurrently accessing this pointer *or*
787 * 3. The referenced data structure has already been exposed to
788 * readers either at compile time or via rcu_assign_pointer() *and*
789 *
790 * a. You have not made *any* reader-visible changes to
791 * this structure since then *or*
792 * b. It is OK for readers accessing this structure from its
793 * new location to see the old state of the structure. (For
794 * example, the changes were to statistical counters or to
795 * other state where exact synchronization is not required.)
796 *
797 * Failure to follow these rules governing use of RCU_INIT_POINTER() will
798 * result in impossible-to-diagnose memory corruption. As in the structures
799 * will look OK in crash dumps, but any concurrent RCU readers might
800 * see pre-initialized values of the referenced data structure. So
801 * please be very careful how you use RCU_INIT_POINTER()!!!
802 *
803 * If you are creating an RCU-protected linked structure that is accessed
804 * by a single external-to-structure RCU-protected pointer, then you may
805 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
806 * pointers, but you must use rcu_assign_pointer() to initialize the
807 * external-to-structure pointer *after* you have completely initialized
808 * the reader-accessible portions of the linked structure.
809 *
810 * Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
811 * ordering guarantees for either the CPU or the compiler.
812 */
813#define RCU_INIT_POINTER(p, v) \
814 do { \
815 rcu_check_sparse(p, __rcu); \
816 WRITE_ONCE(p, RCU_INITIALIZER(v)); \
817 } while (0)
818
819/**
820 * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
821 * @p: The pointer to be initialized.
822 * @v: The value to initialized the pointer to.
823 *
824 * GCC-style initialization for an RCU-protected pointer in a structure field.
825 */
826#define RCU_POINTER_INITIALIZER(p, v) \
827 .p = RCU_INITIALIZER(v)
828
829/*
830 * Does the specified offset indicate that the corresponding rcu_head
831 * structure can be handled by kvfree_rcu()?
832 */
833#define __is_kvfree_rcu_offset(offset) ((offset) < 4096)
834
835/*
836 * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
837 */
838#define __kvfree_rcu(head, offset) \
839 do { \
840 BUILD_BUG_ON(!__is_kvfree_rcu_offset(offset)); \
841 kvfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \
842 } while (0)
843
844/**
845 * kfree_rcu() - kfree an object after a grace period.
846 * @ptr: pointer to kfree
847 * @rhf: the name of the struct rcu_head within the type of @ptr.
848 *
849 * Many rcu callbacks functions just call kfree() on the base structure.
850 * These functions are trivial, but their size adds up, and furthermore
851 * when they are used in a kernel module, that module must invoke the
852 * high-latency rcu_barrier() function at module-unload time.
853 *
854 * The kfree_rcu() function handles this issue. Rather than encoding a
855 * function address in the embedded rcu_head structure, kfree_rcu() instead
856 * encodes the offset of the rcu_head structure within the base structure.
857 * Because the functions are not allowed in the low-order 4096 bytes of
858 * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
859 * If the offset is larger than 4095 bytes, a compile-time error will
860 * be generated in __kvfree_rcu(). If this error is triggered, you can
861 * either fall back to use of call_rcu() or rearrange the structure to
862 * position the rcu_head structure into the first 4096 bytes.
863 *
864 * Note that the allowable offset might decrease in the future, for example,
865 * to allow something like kmem_cache_free_rcu().
866 *
867 * The BUILD_BUG_ON check must not involve any function calls, hence the
868 * checks are done in macros here.
869 */
870#define kfree_rcu(ptr, rhf) \
871do { \
872 typeof (ptr) ___p = (ptr); \
873 \
874 if (___p) \
875 __kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \
876} while (0)
877
878/**
879 * kvfree_rcu() - kvfree an object after a grace period.
880 *
881 * This macro consists of one or two arguments and it is
882 * based on whether an object is head-less or not. If it
883 * has a head then a semantic stays the same as it used
884 * to be before:
885 *
886 * kvfree_rcu(ptr, rhf);
887 *
888 * where @ptr is a pointer to kvfree(), @rhf is the name
889 * of the rcu_head structure within the type of @ptr.
890 *
891 * When it comes to head-less variant, only one argument
892 * is passed and that is just a pointer which has to be
893 * freed after a grace period. Therefore the semantic is
894 *
895 * kvfree_rcu(ptr);
896 *
897 * where @ptr is a pointer to kvfree().
898 *
899 * Please note, head-less way of freeing is permitted to
900 * use from a context that has to follow might_sleep()
901 * annotation. Otherwise, please switch and embed the
902 * rcu_head structure within the type of @ptr.
903 */
904#define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__, \
905 kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__)
906
907#define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME
908#define kvfree_rcu_arg_2(ptr, rhf) kfree_rcu(ptr, rhf)
909#define kvfree_rcu_arg_1(ptr) \
910do { \
911 typeof(ptr) ___p = (ptr); \
912 \
913 if (___p) \
914 kvfree_call_rcu(NULL, (rcu_callback_t) (___p)); \
915} while (0)
916
917/*
918 * Place this after a lock-acquisition primitive to guarantee that
919 * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies
920 * if the UNLOCK and LOCK are executed by the same CPU or if the
921 * UNLOCK and LOCK operate on the same lock variable.
922 */
923#ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
924#define smp_mb__after_unlock_lock() smp_mb() /* Full ordering for lock. */
925#else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
926#define smp_mb__after_unlock_lock() do { } while (0)
927#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
928
929
930/* Has the specified rcu_head structure been handed to call_rcu()? */
931
932/**
933 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
934 * @rhp: The rcu_head structure to initialize.
935 *
936 * If you intend to invoke rcu_head_after_call_rcu() to test whether a
937 * given rcu_head structure has already been passed to call_rcu(), then
938 * you must also invoke this rcu_head_init() function on it just after
939 * allocating that structure. Calls to this function must not race with
940 * calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
941 */
942static inline void rcu_head_init(struct rcu_head *rhp)
943{
944 rhp->func = (rcu_callback_t)~0L;
945}
946
947/**
948 * rcu_head_after_call_rcu - Has this rcu_head been passed to call_rcu()?
949 * @rhp: The rcu_head structure to test.
950 * @f: The function passed to call_rcu() along with @rhp.
951 *
952 * Returns @true if the @rhp has been passed to call_rcu() with @func,
953 * and @false otherwise. Emits a warning in any other case, including
954 * the case where @rhp has already been invoked after a grace period.
955 * Calls to this function must not race with callback invocation. One way
956 * to avoid such races is to enclose the call to rcu_head_after_call_rcu()
957 * in an RCU read-side critical section that includes a read-side fetch
958 * of the pointer to the structure containing @rhp.
959 */
960static inline bool
961rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
962{
963 rcu_callback_t func = READ_ONCE(rhp->func);
964
965 if (func == f)
966 return true;
967 WARN_ON_ONCE(func != (rcu_callback_t)~0L);
968 return false;
969}
970
971/* kernel/ksysfs.c definitions */
972extern int rcu_expedited;
973extern int rcu_normal;
974
975#endif /* __LINUX_RCUPDATE_H */
1/*
2 * Read-Copy Update mechanism for mutual exclusion
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2001
19 *
20 * Author: Dipankar Sarma <dipankar@in.ibm.com>
21 *
22 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
23 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
24 * Papers:
25 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
26 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
27 *
28 * For detailed explanation of Read-Copy Update mechanism see -
29 * http://lse.sourceforge.net/locking/rcupdate.html
30 *
31 */
32
33#ifndef __LINUX_RCUPDATE_H
34#define __LINUX_RCUPDATE_H
35
36#include <linux/types.h>
37#include <linux/cache.h>
38#include <linux/spinlock.h>
39#include <linux/threads.h>
40#include <linux/cpumask.h>
41#include <linux/seqlock.h>
42#include <linux/lockdep.h>
43#include <linux/completion.h>
44#include <linux/debugobjects.h>
45#include <linux/bug.h>
46#include <linux/compiler.h>
47
48#ifdef CONFIG_RCU_TORTURE_TEST
49extern int rcutorture_runnable; /* for sysctl */
50#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
51
52#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
53extern void rcutorture_record_test_transition(void);
54extern void rcutorture_record_progress(unsigned long vernum);
55extern void do_trace_rcu_torture_read(char *rcutorturename,
56 struct rcu_head *rhp);
57#else
58static inline void rcutorture_record_test_transition(void)
59{
60}
61static inline void rcutorture_record_progress(unsigned long vernum)
62{
63}
64#ifdef CONFIG_RCU_TRACE
65extern void do_trace_rcu_torture_read(char *rcutorturename,
66 struct rcu_head *rhp);
67#else
68#define do_trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
69#endif
70#endif
71
72#define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b))
73#define UINT_CMP_LT(a, b) (UINT_MAX / 2 < (a) - (b))
74#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
75#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
76
77/* Exported common interfaces */
78
79#ifdef CONFIG_PREEMPT_RCU
80
81/**
82 * call_rcu() - Queue an RCU callback for invocation after a grace period.
83 * @head: structure to be used for queueing the RCU updates.
84 * @func: actual callback function to be invoked after the grace period
85 *
86 * The callback function will be invoked some time after a full grace
87 * period elapses, in other words after all pre-existing RCU read-side
88 * critical sections have completed. However, the callback function
89 * might well execute concurrently with RCU read-side critical sections
90 * that started after call_rcu() was invoked. RCU read-side critical
91 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
92 * and may be nested.
93 */
94extern void call_rcu(struct rcu_head *head,
95 void (*func)(struct rcu_head *head));
96
97#else /* #ifdef CONFIG_PREEMPT_RCU */
98
99/* In classic RCU, call_rcu() is just call_rcu_sched(). */
100#define call_rcu call_rcu_sched
101
102#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
103
104/**
105 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
106 * @head: structure to be used for queueing the RCU updates.
107 * @func: actual callback function to be invoked after the grace period
108 *
109 * The callback function will be invoked some time after a full grace
110 * period elapses, in other words after all currently executing RCU
111 * read-side critical sections have completed. call_rcu_bh() assumes
112 * that the read-side critical sections end on completion of a softirq
113 * handler. This means that read-side critical sections in process
114 * context must not be interrupted by softirqs. This interface is to be
115 * used when most of the read-side critical sections are in softirq context.
116 * RCU read-side critical sections are delimited by :
117 * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
118 * OR
119 * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
120 * These may be nested.
121 */
122extern void call_rcu_bh(struct rcu_head *head,
123 void (*func)(struct rcu_head *head));
124
125/**
126 * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
127 * @head: structure to be used for queueing the RCU updates.
128 * @func: actual callback function to be invoked after the grace period
129 *
130 * The callback function will be invoked some time after a full grace
131 * period elapses, in other words after all currently executing RCU
132 * read-side critical sections have completed. call_rcu_sched() assumes
133 * that the read-side critical sections end on enabling of preemption
134 * or on voluntary preemption.
135 * RCU read-side critical sections are delimited by :
136 * - rcu_read_lock_sched() and rcu_read_unlock_sched(),
137 * OR
138 * anything that disables preemption.
139 * These may be nested.
140 */
141extern void call_rcu_sched(struct rcu_head *head,
142 void (*func)(struct rcu_head *rcu));
143
144extern void synchronize_sched(void);
145
146#ifdef CONFIG_PREEMPT_RCU
147
148extern void __rcu_read_lock(void);
149extern void __rcu_read_unlock(void);
150void synchronize_rcu(void);
151
152/*
153 * Defined as a macro as it is a very low level header included from
154 * areas that don't even know about current. This gives the rcu_read_lock()
155 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
156 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
157 */
158#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
159
160#else /* #ifdef CONFIG_PREEMPT_RCU */
161
162static inline void __rcu_read_lock(void)
163{
164 preempt_disable();
165}
166
167static inline void __rcu_read_unlock(void)
168{
169 preempt_enable();
170}
171
172static inline void synchronize_rcu(void)
173{
174 synchronize_sched();
175}
176
177static inline int rcu_preempt_depth(void)
178{
179 return 0;
180}
181
182#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
183
184/* Internal to kernel */
185extern void rcu_sched_qs(int cpu);
186extern void rcu_bh_qs(int cpu);
187extern void rcu_check_callbacks(int cpu, int user);
188struct notifier_block;
189extern void rcu_idle_enter(void);
190extern void rcu_idle_exit(void);
191extern void rcu_irq_enter(void);
192extern void rcu_irq_exit(void);
193extern void exit_rcu(void);
194
195/**
196 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
197 * @a: Code that RCU needs to pay attention to.
198 *
199 * RCU, RCU-bh, and RCU-sched read-side critical sections are forbidden
200 * in the inner idle loop, that is, between the rcu_idle_enter() and
201 * the rcu_idle_exit() -- RCU will happily ignore any such read-side
202 * critical sections. However, things like powertop need tracepoints
203 * in the inner idle loop.
204 *
205 * This macro provides the way out: RCU_NONIDLE(do_something_with_RCU())
206 * will tell RCU that it needs to pay attending, invoke its argument
207 * (in this example, a call to the do_something_with_RCU() function),
208 * and then tell RCU to go back to ignoring this CPU. It is permissible
209 * to nest RCU_NONIDLE() wrappers, but the nesting level is currently
210 * quite limited. If deeper nesting is required, it will be necessary
211 * to adjust DYNTICK_TASK_NESTING_VALUE accordingly.
212 *
213 * This macro may be used from process-level code only.
214 */
215#define RCU_NONIDLE(a) \
216 do { \
217 rcu_idle_exit(); \
218 do { a; } while (0); \
219 rcu_idle_enter(); \
220 } while (0)
221
222/*
223 * Infrastructure to implement the synchronize_() primitives in
224 * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
225 */
226
227typedef void call_rcu_func_t(struct rcu_head *head,
228 void (*func)(struct rcu_head *head));
229void wait_rcu_gp(call_rcu_func_t crf);
230
231#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
232#include <linux/rcutree.h>
233#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
234#include <linux/rcutiny.h>
235#else
236#error "Unknown RCU implementation specified to kernel configuration"
237#endif
238
239/*
240 * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
241 * initialization and destruction of rcu_head on the stack. rcu_head structures
242 * allocated dynamically in the heap or defined statically don't need any
243 * initialization.
244 */
245#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
246extern void init_rcu_head_on_stack(struct rcu_head *head);
247extern void destroy_rcu_head_on_stack(struct rcu_head *head);
248#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
249static inline void init_rcu_head_on_stack(struct rcu_head *head)
250{
251}
252
253static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
254{
255}
256#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
257
258#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
259bool rcu_lockdep_current_cpu_online(void);
260#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
261static inline bool rcu_lockdep_current_cpu_online(void)
262{
263 return 1;
264}
265#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
266
267#ifdef CONFIG_DEBUG_LOCK_ALLOC
268
269#ifdef CONFIG_PROVE_RCU
270extern int rcu_is_cpu_idle(void);
271#else /* !CONFIG_PROVE_RCU */
272static inline int rcu_is_cpu_idle(void)
273{
274 return 0;
275}
276#endif /* else !CONFIG_PROVE_RCU */
277
278static inline void rcu_lock_acquire(struct lockdep_map *map)
279{
280 lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);
281}
282
283static inline void rcu_lock_release(struct lockdep_map *map)
284{
285 lock_release(map, 1, _THIS_IP_);
286}
287
288extern struct lockdep_map rcu_lock_map;
289extern struct lockdep_map rcu_bh_lock_map;
290extern struct lockdep_map rcu_sched_lock_map;
291extern int debug_lockdep_rcu_enabled(void);
292
293/**
294 * rcu_read_lock_held() - might we be in RCU read-side critical section?
295 *
296 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
297 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
298 * this assumes we are in an RCU read-side critical section unless it can
299 * prove otherwise. This is useful for debug checks in functions that
300 * require that they be called within an RCU read-side critical section.
301 *
302 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
303 * and while lockdep is disabled.
304 *
305 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
306 * occur in the same context, for example, it is illegal to invoke
307 * rcu_read_unlock() in process context if the matching rcu_read_lock()
308 * was invoked from within an irq handler.
309 *
310 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
311 * offline from an RCU perspective, so check for those as well.
312 */
313static inline int rcu_read_lock_held(void)
314{
315 if (!debug_lockdep_rcu_enabled())
316 return 1;
317 if (rcu_is_cpu_idle())
318 return 0;
319 if (!rcu_lockdep_current_cpu_online())
320 return 0;
321 return lock_is_held(&rcu_lock_map);
322}
323
324/*
325 * rcu_read_lock_bh_held() is defined out of line to avoid #include-file
326 * hell.
327 */
328extern int rcu_read_lock_bh_held(void);
329
330/**
331 * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
332 *
333 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
334 * RCU-sched read-side critical section. In absence of
335 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
336 * critical section unless it can prove otherwise. Note that disabling
337 * of preemption (including disabling irqs) counts as an RCU-sched
338 * read-side critical section. This is useful for debug checks in functions
339 * that required that they be called within an RCU-sched read-side
340 * critical section.
341 *
342 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
343 * and while lockdep is disabled.
344 *
345 * Note that if the CPU is in the idle loop from an RCU point of
346 * view (ie: that we are in the section between rcu_idle_enter() and
347 * rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU
348 * did an rcu_read_lock(). The reason for this is that RCU ignores CPUs
349 * that are in such a section, considering these as in extended quiescent
350 * state, so such a CPU is effectively never in an RCU read-side critical
351 * section regardless of what RCU primitives it invokes. This state of
352 * affairs is required --- we need to keep an RCU-free window in idle
353 * where the CPU may possibly enter into low power mode. This way we can
354 * notice an extended quiescent state to other CPUs that started a grace
355 * period. Otherwise we would delay any grace period as long as we run in
356 * the idle task.
357 *
358 * Similarly, we avoid claiming an SRCU read lock held if the current
359 * CPU is offline.
360 */
361#ifdef CONFIG_PREEMPT_COUNT
362static inline int rcu_read_lock_sched_held(void)
363{
364 int lockdep_opinion = 0;
365
366 if (!debug_lockdep_rcu_enabled())
367 return 1;
368 if (rcu_is_cpu_idle())
369 return 0;
370 if (!rcu_lockdep_current_cpu_online())
371 return 0;
372 if (debug_locks)
373 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
374 return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
375}
376#else /* #ifdef CONFIG_PREEMPT_COUNT */
377static inline int rcu_read_lock_sched_held(void)
378{
379 return 1;
380}
381#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
382
383#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
384
385# define rcu_lock_acquire(a) do { } while (0)
386# define rcu_lock_release(a) do { } while (0)
387
388static inline int rcu_read_lock_held(void)
389{
390 return 1;
391}
392
393static inline int rcu_read_lock_bh_held(void)
394{
395 return 1;
396}
397
398#ifdef CONFIG_PREEMPT_COUNT
399static inline int rcu_read_lock_sched_held(void)
400{
401 return preempt_count() != 0 || irqs_disabled();
402}
403#else /* #ifdef CONFIG_PREEMPT_COUNT */
404static inline int rcu_read_lock_sched_held(void)
405{
406 return 1;
407}
408#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
409
410#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
411
412#ifdef CONFIG_PROVE_RCU
413
414extern int rcu_my_thread_group_empty(void);
415
416/**
417 * rcu_lockdep_assert - emit lockdep splat if specified condition not met
418 * @c: condition to check
419 * @s: informative message
420 */
421#define rcu_lockdep_assert(c, s) \
422 do { \
423 static bool __section(.data.unlikely) __warned; \
424 if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
425 __warned = true; \
426 lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
427 } \
428 } while (0)
429
430#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
431static inline void rcu_preempt_sleep_check(void)
432{
433 rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),
434 "Illegal context switch in RCU read-side "
435 "critical section");
436}
437#else /* #ifdef CONFIG_PROVE_RCU */
438static inline void rcu_preempt_sleep_check(void)
439{
440}
441#endif /* #else #ifdef CONFIG_PROVE_RCU */
442
443#define rcu_sleep_check() \
444 do { \
445 rcu_preempt_sleep_check(); \
446 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
447 "Illegal context switch in RCU-bh" \
448 " read-side critical section"); \
449 rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
450 "Illegal context switch in RCU-sched"\
451 " read-side critical section"); \
452 } while (0)
453
454#else /* #ifdef CONFIG_PROVE_RCU */
455
456#define rcu_lockdep_assert(c, s) do { } while (0)
457#define rcu_sleep_check() do { } while (0)
458
459#endif /* #else #ifdef CONFIG_PROVE_RCU */
460
461/*
462 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
463 * and rcu_assign_pointer(). Some of these could be folded into their
464 * callers, but they are left separate in order to ease introduction of
465 * multiple flavors of pointers to match the multiple flavors of RCU
466 * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
467 * the future.
468 */
469
470#ifdef __CHECKER__
471#define rcu_dereference_sparse(p, space) \
472 ((void)(((typeof(*p) space *)p) == p))
473#else /* #ifdef __CHECKER__ */
474#define rcu_dereference_sparse(p, space)
475#endif /* #else #ifdef __CHECKER__ */
476
477#define __rcu_access_pointer(p, space) \
478 ({ \
479 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
480 rcu_dereference_sparse(p, space); \
481 ((typeof(*p) __force __kernel *)(_________p1)); \
482 })
483#define __rcu_dereference_check(p, c, space) \
484 ({ \
485 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
486 rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
487 " usage"); \
488 rcu_dereference_sparse(p, space); \
489 smp_read_barrier_depends(); \
490 ((typeof(*p) __force __kernel *)(_________p1)); \
491 })
492#define __rcu_dereference_protected(p, c, space) \
493 ({ \
494 rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
495 " usage"); \
496 rcu_dereference_sparse(p, space); \
497 ((typeof(*p) __force __kernel *)(p)); \
498 })
499
500#define __rcu_access_index(p, space) \
501 ({ \
502 typeof(p) _________p1 = ACCESS_ONCE(p); \
503 rcu_dereference_sparse(p, space); \
504 (_________p1); \
505 })
506#define __rcu_dereference_index_check(p, c) \
507 ({ \
508 typeof(p) _________p1 = ACCESS_ONCE(p); \
509 rcu_lockdep_assert(c, \
510 "suspicious rcu_dereference_index_check()" \
511 " usage"); \
512 smp_read_barrier_depends(); \
513 (_________p1); \
514 })
515#define __rcu_assign_pointer(p, v, space) \
516 ({ \
517 smp_wmb(); \
518 (p) = (typeof(*v) __force space *)(v); \
519 })
520
521
522/**
523 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
524 * @p: The pointer to read
525 *
526 * Return the value of the specified RCU-protected pointer, but omit the
527 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
528 * when the value of this pointer is accessed, but the pointer is not
529 * dereferenced, for example, when testing an RCU-protected pointer against
530 * NULL. Although rcu_access_pointer() may also be used in cases where
531 * update-side locks prevent the value of the pointer from changing, you
532 * should instead use rcu_dereference_protected() for this use case.
533 *
534 * It is also permissible to use rcu_access_pointer() when read-side
535 * access to the pointer was removed at least one grace period ago, as
536 * is the case in the context of the RCU callback that is freeing up
537 * the data, or after a synchronize_rcu() returns. This can be useful
538 * when tearing down multi-linked structures after a grace period
539 * has elapsed.
540 */
541#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
542
543/**
544 * rcu_dereference_check() - rcu_dereference with debug checking
545 * @p: The pointer to read, prior to dereferencing
546 * @c: The conditions under which the dereference will take place
547 *
548 * Do an rcu_dereference(), but check that the conditions under which the
549 * dereference will take place are correct. Typically the conditions
550 * indicate the various locking conditions that should be held at that
551 * point. The check should return true if the conditions are satisfied.
552 * An implicit check for being in an RCU read-side critical section
553 * (rcu_read_lock()) is included.
554 *
555 * For example:
556 *
557 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
558 *
559 * could be used to indicate to lockdep that foo->bar may only be dereferenced
560 * if either rcu_read_lock() is held, or that the lock required to replace
561 * the bar struct at foo->bar is held.
562 *
563 * Note that the list of conditions may also include indications of when a lock
564 * need not be held, for example during initialisation or destruction of the
565 * target struct:
566 *
567 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
568 * atomic_read(&foo->usage) == 0);
569 *
570 * Inserts memory barriers on architectures that require them
571 * (currently only the Alpha), prevents the compiler from refetching
572 * (and from merging fetches), and, more importantly, documents exactly
573 * which pointers are protected by RCU and checks that the pointer is
574 * annotated as __rcu.
575 */
576#define rcu_dereference_check(p, c) \
577 __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
578
579/**
580 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
581 * @p: The pointer to read, prior to dereferencing
582 * @c: The conditions under which the dereference will take place
583 *
584 * This is the RCU-bh counterpart to rcu_dereference_check().
585 */
586#define rcu_dereference_bh_check(p, c) \
587 __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
588
589/**
590 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
591 * @p: The pointer to read, prior to dereferencing
592 * @c: The conditions under which the dereference will take place
593 *
594 * This is the RCU-sched counterpart to rcu_dereference_check().
595 */
596#define rcu_dereference_sched_check(p, c) \
597 __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
598 __rcu)
599
600#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
601
602/**
603 * rcu_access_index() - fetch RCU index with no dereferencing
604 * @p: The index to read
605 *
606 * Return the value of the specified RCU-protected index, but omit the
607 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
608 * when the value of this index is accessed, but the index is not
609 * dereferenced, for example, when testing an RCU-protected index against
610 * -1. Although rcu_access_index() may also be used in cases where
611 * update-side locks prevent the value of the index from changing, you
612 * should instead use rcu_dereference_index_protected() for this use case.
613 */
614#define rcu_access_index(p) __rcu_access_index((p), __rcu)
615
616/**
617 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
618 * @p: The pointer to read, prior to dereferencing
619 * @c: The conditions under which the dereference will take place
620 *
621 * Similar to rcu_dereference_check(), but omits the sparse checking.
622 * This allows rcu_dereference_index_check() to be used on integers,
623 * which can then be used as array indices. Attempting to use
624 * rcu_dereference_check() on an integer will give compiler warnings
625 * because the sparse address-space mechanism relies on dereferencing
626 * the RCU-protected pointer. Dereferencing integers is not something
627 * that even gcc will put up with.
628 *
629 * Note that this function does not implicitly check for RCU read-side
630 * critical sections. If this function gains lots of uses, it might
631 * make sense to provide versions for each flavor of RCU, but it does
632 * not make sense as of early 2010.
633 */
634#define rcu_dereference_index_check(p, c) \
635 __rcu_dereference_index_check((p), (c))
636
637/**
638 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
639 * @p: The pointer to read, prior to dereferencing
640 * @c: The conditions under which the dereference will take place
641 *
642 * Return the value of the specified RCU-protected pointer, but omit
643 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
644 * is useful in cases where update-side locks prevent the value of the
645 * pointer from changing. Please note that this primitive does -not-
646 * prevent the compiler from repeating this reference or combining it
647 * with other references, so it should not be used without protection
648 * of appropriate locks.
649 *
650 * This function is only for update-side use. Using this function
651 * when protected only by rcu_read_lock() will result in infrequent
652 * but very ugly failures.
653 */
654#define rcu_dereference_protected(p, c) \
655 __rcu_dereference_protected((p), (c), __rcu)
656
657
658/**
659 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
660 * @p: The pointer to read, prior to dereferencing
661 *
662 * This is a simple wrapper around rcu_dereference_check().
663 */
664#define rcu_dereference(p) rcu_dereference_check(p, 0)
665
666/**
667 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
668 * @p: The pointer to read, prior to dereferencing
669 *
670 * Makes rcu_dereference_check() do the dirty work.
671 */
672#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
673
674/**
675 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
676 * @p: The pointer to read, prior to dereferencing
677 *
678 * Makes rcu_dereference_check() do the dirty work.
679 */
680#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
681
682/**
683 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
684 *
685 * When synchronize_rcu() is invoked on one CPU while other CPUs
686 * are within RCU read-side critical sections, then the
687 * synchronize_rcu() is guaranteed to block until after all the other
688 * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
689 * on one CPU while other CPUs are within RCU read-side critical
690 * sections, invocation of the corresponding RCU callback is deferred
691 * until after the all the other CPUs exit their critical sections.
692 *
693 * Note, however, that RCU callbacks are permitted to run concurrently
694 * with new RCU read-side critical sections. One way that this can happen
695 * is via the following sequence of events: (1) CPU 0 enters an RCU
696 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
697 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
698 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
699 * callback is invoked. This is legal, because the RCU read-side critical
700 * section that was running concurrently with the call_rcu() (and which
701 * therefore might be referencing something that the corresponding RCU
702 * callback would free up) has completed before the corresponding
703 * RCU callback is invoked.
704 *
705 * RCU read-side critical sections may be nested. Any deferred actions
706 * will be deferred until the outermost RCU read-side critical section
707 * completes.
708 *
709 * You can avoid reading and understanding the next paragraph by
710 * following this rule: don't put anything in an rcu_read_lock() RCU
711 * read-side critical section that would block in a !PREEMPT kernel.
712 * But if you want the full story, read on!
713 *
714 * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
715 * is illegal to block while in an RCU read-side critical section. In
716 * preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
717 * in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
718 * be preempted, but explicit blocking is illegal. Finally, in preemptible
719 * RCU implementations in real-time (CONFIG_PREEMPT_RT) kernel builds,
720 * RCU read-side critical sections may be preempted and they may also
721 * block, but only when acquiring spinlocks that are subject to priority
722 * inheritance.
723 */
724static inline void rcu_read_lock(void)
725{
726 __rcu_read_lock();
727 __acquire(RCU);
728 rcu_lock_acquire(&rcu_lock_map);
729 rcu_lockdep_assert(!rcu_is_cpu_idle(),
730 "rcu_read_lock() used illegally while idle");
731}
732
733/*
734 * So where is rcu_write_lock()? It does not exist, as there is no
735 * way for writers to lock out RCU readers. This is a feature, not
736 * a bug -- this property is what provides RCU's performance benefits.
737 * Of course, writers must coordinate with each other. The normal
738 * spinlock primitives work well for this, but any other technique may be
739 * used as well. RCU does not care how the writers keep out of each
740 * others' way, as long as they do so.
741 */
742
743/**
744 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
745 *
746 * See rcu_read_lock() for more information.
747 */
748static inline void rcu_read_unlock(void)
749{
750 rcu_lockdep_assert(!rcu_is_cpu_idle(),
751 "rcu_read_unlock() used illegally while idle");
752 rcu_lock_release(&rcu_lock_map);
753 __release(RCU);
754 __rcu_read_unlock();
755}
756
757/**
758 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
759 *
760 * This is equivalent of rcu_read_lock(), but to be used when updates
761 * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
762 * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
763 * softirq handler to be a quiescent state, a process in RCU read-side
764 * critical section must be protected by disabling softirqs. Read-side
765 * critical sections in interrupt context can use just rcu_read_lock(),
766 * though this should at least be commented to avoid confusing people
767 * reading the code.
768 *
769 * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
770 * must occur in the same context, for example, it is illegal to invoke
771 * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
772 * was invoked from some other task.
773 */
774static inline void rcu_read_lock_bh(void)
775{
776 local_bh_disable();
777 __acquire(RCU_BH);
778 rcu_lock_acquire(&rcu_bh_lock_map);
779 rcu_lockdep_assert(!rcu_is_cpu_idle(),
780 "rcu_read_lock_bh() used illegally while idle");
781}
782
783/*
784 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
785 *
786 * See rcu_read_lock_bh() for more information.
787 */
788static inline void rcu_read_unlock_bh(void)
789{
790 rcu_lockdep_assert(!rcu_is_cpu_idle(),
791 "rcu_read_unlock_bh() used illegally while idle");
792 rcu_lock_release(&rcu_bh_lock_map);
793 __release(RCU_BH);
794 local_bh_enable();
795}
796
797/**
798 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
799 *
800 * This is equivalent of rcu_read_lock(), but to be used when updates
801 * are being done using call_rcu_sched() or synchronize_rcu_sched().
802 * Read-side critical sections can also be introduced by anything that
803 * disables preemption, including local_irq_disable() and friends.
804 *
805 * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
806 * must occur in the same context, for example, it is illegal to invoke
807 * rcu_read_unlock_sched() from process context if the matching
808 * rcu_read_lock_sched() was invoked from an NMI handler.
809 */
810static inline void rcu_read_lock_sched(void)
811{
812 preempt_disable();
813 __acquire(RCU_SCHED);
814 rcu_lock_acquire(&rcu_sched_lock_map);
815 rcu_lockdep_assert(!rcu_is_cpu_idle(),
816 "rcu_read_lock_sched() used illegally while idle");
817}
818
819/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
820static inline notrace void rcu_read_lock_sched_notrace(void)
821{
822 preempt_disable_notrace();
823 __acquire(RCU_SCHED);
824}
825
826/*
827 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
828 *
829 * See rcu_read_lock_sched for more information.
830 */
831static inline void rcu_read_unlock_sched(void)
832{
833 rcu_lockdep_assert(!rcu_is_cpu_idle(),
834 "rcu_read_unlock_sched() used illegally while idle");
835 rcu_lock_release(&rcu_sched_lock_map);
836 __release(RCU_SCHED);
837 preempt_enable();
838}
839
840/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
841static inline notrace void rcu_read_unlock_sched_notrace(void)
842{
843 __release(RCU_SCHED);
844 preempt_enable_notrace();
845}
846
847/**
848 * rcu_assign_pointer() - assign to RCU-protected pointer
849 * @p: pointer to assign to
850 * @v: value to assign (publish)
851 *
852 * Assigns the specified value to the specified RCU-protected
853 * pointer, ensuring that any concurrent RCU readers will see
854 * any prior initialization. Returns the value assigned.
855 *
856 * Inserts memory barriers on architectures that require them
857 * (which is most of them), and also prevents the compiler from
858 * reordering the code that initializes the structure after the pointer
859 * assignment. More importantly, this call documents which pointers
860 * will be dereferenced by RCU read-side code.
861 *
862 * In some special cases, you may use RCU_INIT_POINTER() instead
863 * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
864 * to the fact that it does not constrain either the CPU or the compiler.
865 * That said, using RCU_INIT_POINTER() when you should have used
866 * rcu_assign_pointer() is a very bad thing that results in
867 * impossible-to-diagnose memory corruption. So please be careful.
868 * See the RCU_INIT_POINTER() comment header for details.
869 */
870#define rcu_assign_pointer(p, v) \
871 __rcu_assign_pointer((p), (v), __rcu)
872
873/**
874 * RCU_INIT_POINTER() - initialize an RCU protected pointer
875 *
876 * Initialize an RCU-protected pointer in special cases where readers
877 * do not need ordering constraints on the CPU or the compiler. These
878 * special cases are:
879 *
880 * 1. This use of RCU_INIT_POINTER() is NULLing out the pointer -or-
881 * 2. The caller has taken whatever steps are required to prevent
882 * RCU readers from concurrently accessing this pointer -or-
883 * 3. The referenced data structure has already been exposed to
884 * readers either at compile time or via rcu_assign_pointer() -and-
885 * a. You have not made -any- reader-visible changes to
886 * this structure since then -or-
887 * b. It is OK for readers accessing this structure from its
888 * new location to see the old state of the structure. (For
889 * example, the changes were to statistical counters or to
890 * other state where exact synchronization is not required.)
891 *
892 * Failure to follow these rules governing use of RCU_INIT_POINTER() will
893 * result in impossible-to-diagnose memory corruption. As in the structures
894 * will look OK in crash dumps, but any concurrent RCU readers might
895 * see pre-initialized values of the referenced data structure. So
896 * please be very careful how you use RCU_INIT_POINTER()!!!
897 *
898 * If you are creating an RCU-protected linked structure that is accessed
899 * by a single external-to-structure RCU-protected pointer, then you may
900 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
901 * pointers, but you must use rcu_assign_pointer() to initialize the
902 * external-to-structure pointer -after- you have completely initialized
903 * the reader-accessible portions of the linked structure.
904 */
905#define RCU_INIT_POINTER(p, v) \
906 p = (typeof(*v) __force __rcu *)(v)
907
908static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)
909{
910 return offset < 4096;
911}
912
913static __always_inline
914void __kfree_rcu(struct rcu_head *head, unsigned long offset)
915{
916 typedef void (*rcu_callback)(struct rcu_head *);
917
918 BUILD_BUG_ON(!__builtin_constant_p(offset));
919
920 /* See the kfree_rcu() header comment. */
921 BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));
922
923 kfree_call_rcu(head, (rcu_callback)offset);
924}
925
926/*
927 * Does the specified offset indicate that the corresponding rcu_head
928 * structure can be handled by kfree_rcu()?
929 */
930#define __is_kfree_rcu_offset(offset) ((offset) < 4096)
931
932/*
933 * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
934 */
935#define __kfree_rcu(head, offset) \
936 do { \
937 BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
938 call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
939 } while (0)
940
941/**
942 * kfree_rcu() - kfree an object after a grace period.
943 * @ptr: pointer to kfree
944 * @rcu_head: the name of the struct rcu_head within the type of @ptr.
945 *
946 * Many rcu callbacks functions just call kfree() on the base structure.
947 * These functions are trivial, but their size adds up, and furthermore
948 * when they are used in a kernel module, that module must invoke the
949 * high-latency rcu_barrier() function at module-unload time.
950 *
951 * The kfree_rcu() function handles this issue. Rather than encoding a
952 * function address in the embedded rcu_head structure, kfree_rcu() instead
953 * encodes the offset of the rcu_head structure within the base structure.
954 * Because the functions are not allowed in the low-order 4096 bytes of
955 * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
956 * If the offset is larger than 4095 bytes, a compile-time error will
957 * be generated in __kfree_rcu(). If this error is triggered, you can
958 * either fall back to use of call_rcu() or rearrange the structure to
959 * position the rcu_head structure into the first 4096 bytes.
960 *
961 * Note that the allowable offset might decrease in the future, for example,
962 * to allow something like kmem_cache_free_rcu().
963 *
964 * The BUILD_BUG_ON check must not involve any function calls, hence the
965 * checks are done in macros here.
966 */
967#define kfree_rcu(ptr, rcu_head) \
968 __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
969
970#endif /* __LINUX_RCUPDATE_H */