1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * Read-Copy Update mechanism for mutual exclusion
  4 *
  5 * Copyright IBM Corporation, 2001
  6 *
  7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
  8 *	    Manfred Spraul <manfred@colorfullife.com>
  9 *
 10 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
 11 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
 12 * Papers:
 13 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
 14 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
 15 *
 16 * For detailed explanation of Read-Copy Update mechanism see -
 17 *		http://lse.sourceforge.net/locking/rcupdate.html
 18 *
 19 */
 20#include <linux/types.h>
 21#include <linux/kernel.h>
 22#include <linux/init.h>
 23#include <linux/spinlock.h>
 24#include <linux/smp.h>
 25#include <linux/interrupt.h>
 26#include <linux/sched/signal.h>
 27#include <linux/sched/debug.h>
 28#include <linux/atomic.h>
 29#include <linux/bitops.h>
 30#include <linux/percpu.h>
 31#include <linux/notifier.h>
 32#include <linux/cpu.h>
 33#include <linux/mutex.h>
 34#include <linux/export.h>
 35#include <linux/hardirq.h>
 36#include <linux/delay.h>
 37#include <linux/moduleparam.h>
 38#include <linux/kthread.h>
 39#include <linux/tick.h>
 40#include <linux/rcupdate_wait.h>
 41#include <linux/sched/isolation.h>
 42#include <linux/kprobes.h>
 43#include <linux/slab.h>
 44#include <linux/irq_work.h>
 45#include <linux/rcupdate_trace.h>
 46
 47#define CREATE_TRACE_POINTS
 48
 49#include "rcu.h"
 50
 51#ifdef MODULE_PARAM_PREFIX
 52#undef MODULE_PARAM_PREFIX
 53#endif
 54#define MODULE_PARAM_PREFIX "rcupdate."
 55
 56#ifndef data_race
 57#define data_race(expr)							\
 58	({								\
 59		expr;							\
 60	})
 61#endif
 62#ifndef ASSERT_EXCLUSIVE_WRITER
 63#define ASSERT_EXCLUSIVE_WRITER(var) do { } while (0)
 64#endif
 65#ifndef ASSERT_EXCLUSIVE_ACCESS
 66#define ASSERT_EXCLUSIVE_ACCESS(var) do { } while (0)
 67#endif
 68
 69#ifndef CONFIG_TINY_RCU
 70module_param(rcu_expedited, int, 0);
 71module_param(rcu_normal, int, 0);
 72static int rcu_normal_after_boot;
 73module_param(rcu_normal_after_boot, int, 0);
 74#endif /* #ifndef CONFIG_TINY_RCU */
 75
 76#ifdef CONFIG_DEBUG_LOCK_ALLOC
 77/**
 78 * rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section?
 79 * @ret:	Best guess answer if lockdep cannot be relied on
 80 *
 81 * Returns true if lockdep must be ignored, in which case ``*ret`` contains
 82 * the best guess described below.  Otherwise returns false, in which
 83 * case ``*ret`` tells the caller nothing and the caller should instead
 84 * consult lockdep.
 85 *
 86 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set ``*ret`` to nonzero iff in an
 87 * RCU-sched read-side critical section.  In absence of
 88 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
 89 * critical section unless it can prove otherwise.  Note that disabling
 90 * of preemption (including disabling irqs) counts as an RCU-sched
 91 * read-side critical section.  This is useful for debug checks in functions
 92 * that required that they be called within an RCU-sched read-side
 93 * critical section.
 94 *
 95 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
 96 * and while lockdep is disabled.
 97 *
 98 * Note that if the CPU is in the idle loop from an RCU point of view (ie:
 99 * that we are in the section between rcu_idle_enter() and rcu_idle_exit())
100 * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an
101 * rcu_read_lock().  The reason for this is that RCU ignores CPUs that are
102 * in such a section, considering these as in extended quiescent state,
103 * so such a CPU is effectively never in an RCU read-side critical section
104 * regardless of what RCU primitives it invokes.  This state of affairs is
105 * required --- we need to keep an RCU-free window in idle where the CPU may
106 * possibly enter into low power mode. This way we can notice an extended
107 * quiescent state to other CPUs that started a grace period. Otherwise
108 * we would delay any grace period as long as we run in the idle task.
109 *
110 * Similarly, we avoid claiming an RCU read lock held if the current
111 * CPU is offline.
112 */
113static bool rcu_read_lock_held_common(bool *ret)
114{
115	if (!debug_lockdep_rcu_enabled()) {
116		*ret = true;
117		return true;
118	}
119	if (!rcu_is_watching()) {
120		*ret = false;
121		return true;
122	}
123	if (!rcu_lockdep_current_cpu_online()) {
124		*ret = false;
125		return true;
126	}
127	return false;
128}
129
130int rcu_read_lock_sched_held(void)
131{
132	bool ret;
133
134	if (rcu_read_lock_held_common(&ret))
135		return ret;
136	return lock_is_held(&rcu_sched_lock_map) || !preemptible();
137}
138EXPORT_SYMBOL(rcu_read_lock_sched_held);
139#endif
140
141#ifndef CONFIG_TINY_RCU
142
143/*
144 * Should expedited grace-period primitives always fall back to their
145 * non-expedited counterparts?  Intended for use within RCU.  Note
146 * that if the user specifies both rcu_expedited and rcu_normal, then
147 * rcu_normal wins.  (Except during the time period during boot from
148 * when the first task is spawned until the rcu_set_runtime_mode()
149 * core_initcall() is invoked, at which point everything is expedited.)
150 */
151bool rcu_gp_is_normal(void)
152{
153	return READ_ONCE(rcu_normal) &&
154	       rcu_scheduler_active != RCU_SCHEDULER_INIT;
155}
156EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
157
158static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
159
160/*
161 * Should normal grace-period primitives be expedited?  Intended for
162 * use within RCU.  Note that this function takes the rcu_expedited
163 * sysfs/boot variable and rcu_scheduler_active into account as well
164 * as the rcu_expedite_gp() nesting.  So looping on rcu_unexpedite_gp()
165 * until rcu_gp_is_expedited() returns false is a -really- bad idea.
166 */
167bool rcu_gp_is_expedited(void)
168{
169	return rcu_expedited || atomic_read(&rcu_expedited_nesting);
170}
171EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
172
173/**
174 * rcu_expedite_gp - Expedite future RCU grace periods
175 *
176 * After a call to this function, future calls to synchronize_rcu() and
177 * friends act as the corresponding synchronize_rcu_expedited() function
178 * had instead been called.
179 */
180void rcu_expedite_gp(void)
181{
182	atomic_inc(&rcu_expedited_nesting);
183}
184EXPORT_SYMBOL_GPL(rcu_expedite_gp);
185
186/**
187 * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
188 *
189 * Undo a prior call to rcu_expedite_gp().  If all prior calls to
190 * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
191 * and if the rcu_expedited sysfs/boot parameter is not set, then all
192 * subsequent calls to synchronize_rcu() and friends will return to
193 * their normal non-expedited behavior.
194 */
195void rcu_unexpedite_gp(void)
196{
197	atomic_dec(&rcu_expedited_nesting);
198}
199EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
200
201static bool rcu_boot_ended __read_mostly;
202
203/*
204 * Inform RCU of the end of the in-kernel boot sequence.
205 */
206void rcu_end_inkernel_boot(void)
207{
208	rcu_unexpedite_gp();
209	if (rcu_normal_after_boot)
210		WRITE_ONCE(rcu_normal, 1);
211	rcu_boot_ended = true;
212}
213
214/*
215 * Let rcutorture know when it is OK to turn it up to eleven.
216 */
217bool rcu_inkernel_boot_has_ended(void)
218{
219	return rcu_boot_ended;
220}
221EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
222
223#endif /* #ifndef CONFIG_TINY_RCU */
224
225/*
226 * Test each non-SRCU synchronous grace-period wait API.  This is
227 * useful just after a change in mode for these primitives, and
228 * during early boot.
229 */
230void rcu_test_sync_prims(void)
231{
232	if (!IS_ENABLED(CONFIG_PROVE_RCU))
233		return;
234	synchronize_rcu();
235	synchronize_rcu_expedited();
236}
237
238#if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU)
239
240/*
241 * Switch to run-time mode once RCU has fully initialized.
242 */
243static int __init rcu_set_runtime_mode(void)
244{
245	rcu_test_sync_prims();
246	rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
247	kfree_rcu_scheduler_running();
248	rcu_test_sync_prims();
249	return 0;
250}
251core_initcall(rcu_set_runtime_mode);
252
253#endif /* #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU) */
254
255#ifdef CONFIG_DEBUG_LOCK_ALLOC
256static struct lock_class_key rcu_lock_key;
257struct lockdep_map rcu_lock_map = {
258	.name = "rcu_read_lock",
259	.key = &rcu_lock_key,
260	.wait_type_outer = LD_WAIT_FREE,
261	.wait_type_inner = LD_WAIT_CONFIG, /* XXX PREEMPT_RCU ? */
262};
263EXPORT_SYMBOL_GPL(rcu_lock_map);
264
265static struct lock_class_key rcu_bh_lock_key;
266struct lockdep_map rcu_bh_lock_map = {
267	.name = "rcu_read_lock_bh",
268	.key = &rcu_bh_lock_key,
269	.wait_type_outer = LD_WAIT_FREE,
270	.wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_LOCK also makes BH preemptible */
271};
272EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
273
274static struct lock_class_key rcu_sched_lock_key;
275struct lockdep_map rcu_sched_lock_map = {
276	.name = "rcu_read_lock_sched",
277	.key = &rcu_sched_lock_key,
278	.wait_type_outer = LD_WAIT_FREE,
279	.wait_type_inner = LD_WAIT_SPIN,
280};
281EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
282
283// Tell lockdep when RCU callbacks are being invoked.
284static struct lock_class_key rcu_callback_key;
285struct lockdep_map rcu_callback_map =
286	STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
287EXPORT_SYMBOL_GPL(rcu_callback_map);
288
289noinstr int notrace debug_lockdep_rcu_enabled(void)
290{
291	return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks &&
292	       current->lockdep_recursion == 0;
293}
294EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
295
296/**
297 * rcu_read_lock_held() - might we be in RCU read-side critical section?
298 *
299 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
300 * read-side critical section.  In absence of CONFIG_DEBUG_LOCK_ALLOC,
301 * this assumes we are in an RCU read-side critical section unless it can
302 * prove otherwise.  This is useful for debug checks in functions that
303 * require that they be called within an RCU read-side critical section.
304 *
305 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
306 * and while lockdep is disabled.
307 *
308 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
309 * occur in the same context, for example, it is illegal to invoke
310 * rcu_read_unlock() in process context if the matching rcu_read_lock()
311 * was invoked from within an irq handler.
312 *
313 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
314 * offline from an RCU perspective, so check for those as well.
315 */
316int rcu_read_lock_held(void)
317{
318	bool ret;
319
320	if (rcu_read_lock_held_common(&ret))
321		return ret;
322	return lock_is_held(&rcu_lock_map);
323}
324EXPORT_SYMBOL_GPL(rcu_read_lock_held);
325
326/**
327 * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
328 *
329 * Check for bottom half being disabled, which covers both the
330 * CONFIG_PROVE_RCU and not cases.  Note that if someone uses
331 * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
332 * will show the situation.  This is useful for debug checks in functions
333 * that require that they be called within an RCU read-side critical
334 * section.
335 *
336 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
337 *
338 * Note that rcu_read_lock_bh() is disallowed if the CPU is either idle or
339 * offline from an RCU perspective, so check for those as well.
340 */
341int rcu_read_lock_bh_held(void)
342{
343	bool ret;
344
345	if (rcu_read_lock_held_common(&ret))
346		return ret;
347	return in_softirq() || irqs_disabled();
348}
349EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
350
351int rcu_read_lock_any_held(void)
352{
353	bool ret;
354
355	if (rcu_read_lock_held_common(&ret))
356		return ret;
357	if (lock_is_held(&rcu_lock_map) ||
358	    lock_is_held(&rcu_bh_lock_map) ||
359	    lock_is_held(&rcu_sched_lock_map))
360		return 1;
361	return !preemptible();
362}
363EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
364
365#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
366
367/**
368 * wakeme_after_rcu() - Callback function to awaken a task after grace period
369 * @head: Pointer to rcu_head member within rcu_synchronize structure
370 *
371 * Awaken the corresponding task now that a grace period has elapsed.
372 */
373void wakeme_after_rcu(struct rcu_head *head)
374{
375	struct rcu_synchronize *rcu;
376
377	rcu = container_of(head, struct rcu_synchronize, head);
378	complete(&rcu->completion);
379}
380EXPORT_SYMBOL_GPL(wakeme_after_rcu);
381
382void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array,
383		   struct rcu_synchronize *rs_array)
384{
385	int i;
386	int j;
387
388	/* Initialize and register callbacks for each crcu_array element. */
389	for (i = 0; i < n; i++) {
390		if (checktiny &&
391		    (crcu_array[i] == call_rcu)) {
392			might_sleep();
393			continue;
394		}
395		for (j = 0; j < i; j++)
396			if (crcu_array[j] == crcu_array[i])
397				break;
398		if (j == i) {
399			init_rcu_head_on_stack(&rs_array[i].head);
400			init_completion(&rs_array[i].completion);
401			(crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
402		}
403	}
404
405	/* Wait for all callbacks to be invoked. */
406	for (i = 0; i < n; i++) {
407		if (checktiny &&
408		    (crcu_array[i] == call_rcu))
409			continue;
410		for (j = 0; j < i; j++)
411			if (crcu_array[j] == crcu_array[i])
412				break;
413		if (j == i) {
414			wait_for_completion(&rs_array[i].completion);
415			destroy_rcu_head_on_stack(&rs_array[i].head);
416		}
417	}
418}
419EXPORT_SYMBOL_GPL(__wait_rcu_gp);
420
421#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
422void init_rcu_head(struct rcu_head *head)
423{
424	debug_object_init(head, &rcuhead_debug_descr);
425}
426EXPORT_SYMBOL_GPL(init_rcu_head);
427
428void destroy_rcu_head(struct rcu_head *head)
429{
430	debug_object_free(head, &rcuhead_debug_descr);
431}
432EXPORT_SYMBOL_GPL(destroy_rcu_head);
433
434static bool rcuhead_is_static_object(void *addr)
435{
436	return true;
437}
438
439/**
440 * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
441 * @head: pointer to rcu_head structure to be initialized
442 *
443 * This function informs debugobjects of a new rcu_head structure that
444 * has been allocated as an auto variable on the stack.  This function
445 * is not required for rcu_head structures that are statically defined or
446 * that are dynamically allocated on the heap.  This function has no
447 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
448 */
449void init_rcu_head_on_stack(struct rcu_head *head)
450{
451	debug_object_init_on_stack(head, &rcuhead_debug_descr);
452}
453EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
454
455/**
456 * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
457 * @head: pointer to rcu_head structure to be initialized
458 *
459 * This function informs debugobjects that an on-stack rcu_head structure
460 * is about to go out of scope.  As with init_rcu_head_on_stack(), this
461 * function is not required for rcu_head structures that are statically
462 * defined or that are dynamically allocated on the heap.  Also as with
463 * init_rcu_head_on_stack(), this function has no effect for
464 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
465 */
466void destroy_rcu_head_on_stack(struct rcu_head *head)
467{
468	debug_object_free(head, &rcuhead_debug_descr);
469}
470EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
471
472struct debug_obj_descr rcuhead_debug_descr = {
473	.name = "rcu_head",
474	.is_static_object = rcuhead_is_static_object,
475};
476EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
477#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
478
479#if defined(CONFIG_TREE_RCU) || defined(CONFIG_RCU_TRACE)
480void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
481			       unsigned long secs,
482			       unsigned long c_old, unsigned long c)
483{
484	trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
485}
486EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
487#else
488#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
489	do { } while (0)
490#endif
491
492#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST)
493/* Get rcutorture access to sched_setaffinity(). */
494long rcutorture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
495{
496	int ret;
497
498	ret = sched_setaffinity(pid, in_mask);
499	WARN_ONCE(ret, "%s: sched_setaffinity() returned %d\n", __func__, ret);
500	return ret;
501}
502EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity);
503#endif
504
505#ifdef CONFIG_RCU_STALL_COMMON
506int rcu_cpu_stall_ftrace_dump __read_mostly;
507module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
508int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
509EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
510module_param(rcu_cpu_stall_suppress, int, 0644);
511int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
512module_param(rcu_cpu_stall_timeout, int, 0644);
513#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
514
515// Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
516// warnings.  Also used by rcutorture even if stall warnings are excluded.
517int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
518EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
519module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
520
521#ifdef CONFIG_PROVE_RCU
522
523/*
524 * Early boot self test parameters.
525 */
526static bool rcu_self_test;
527module_param(rcu_self_test, bool, 0444);
528
529static int rcu_self_test_counter;
530
531static void test_callback(struct rcu_head *r)
532{
533	rcu_self_test_counter++;
534	pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
535}
536
537DEFINE_STATIC_SRCU(early_srcu);
538
539struct early_boot_kfree_rcu {
540	struct rcu_head rh;
541};
542
543static void early_boot_test_call_rcu(void)
544{
545	static struct rcu_head head;
546	static struct rcu_head shead;
547	struct early_boot_kfree_rcu *rhp;
548
549	call_rcu(&head, test_callback);
550	if (IS_ENABLED(CONFIG_SRCU))
551		call_srcu(&early_srcu, &shead, test_callback);
552	rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
553	if (!WARN_ON_ONCE(!rhp))
554		kfree_rcu(rhp, rh);
555}
556
557void rcu_early_boot_tests(void)
558{
559	pr_info("Running RCU self tests\n");
560
561	if (rcu_self_test)
562		early_boot_test_call_rcu();
563	rcu_test_sync_prims();
564}
565
566static int rcu_verify_early_boot_tests(void)
567{
568	int ret = 0;
569	int early_boot_test_counter = 0;
570
571	if (rcu_self_test) {
572		early_boot_test_counter++;
573		rcu_barrier();
574		if (IS_ENABLED(CONFIG_SRCU)) {
575			early_boot_test_counter++;
576			srcu_barrier(&early_srcu);
577		}
578	}
579	if (rcu_self_test_counter != early_boot_test_counter) {
580		WARN_ON(1);
581		ret = -1;
582	}
583
584	return ret;
585}
586late_initcall(rcu_verify_early_boot_tests);
587#else
588void rcu_early_boot_tests(void) {}
589#endif /* CONFIG_PROVE_RCU */
590
591#include "tasks.h"
592
593#ifndef CONFIG_TINY_RCU
594
595/*
596 * Print any significant non-default boot-time settings.
597 */
598void __init rcupdate_announce_bootup_oddness(void)
599{
600	if (rcu_normal)
601		pr_info("\tNo expedited grace period (rcu_normal).\n");
602	else if (rcu_normal_after_boot)
603		pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
604	else if (rcu_expedited)
605		pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
606	if (rcu_cpu_stall_suppress)
607		pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
608	if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
609		pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
610	rcu_tasks_bootup_oddness();
611}
612
613#endif /* #ifndef CONFIG_TINY_RCU */