1/*
  2 * Sleepable Read-Copy Update mechanism for mutual exclusion,
  3 *	tiny version for non-preemptible single-CPU use.
  4 *
  5 * This program is free software; you can redistribute it and/or modify
  6 * it under the terms of the GNU General Public License as published by
  7 * the Free Software Foundation; either version 2 of the License, or
  8 * (at your option) any later version.
  9 *
 10 * This program is distributed in the hope that it will be useful,
 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 * GNU General Public License for more details.
 14 *
 15 * You should have received a copy of the GNU General Public License
 16 * along with this program; if not, you can access it online at
 17 * http://www.gnu.org/licenses/gpl-2.0.html.
 18 *
 19 * Copyright (C) IBM Corporation, 2017
 20 *
 21 * Author: Paul McKenney <paulmck@us.ibm.com>
 22 */
 23
 24#include <linux/export.h>
 25#include <linux/mutex.h>
 26#include <linux/preempt.h>
 27#include <linux/rcupdate_wait.h>
 28#include <linux/sched.h>
 29#include <linux/delay.h>
 30#include <linux/srcu.h>
 31
 32#include <linux/rcu_node_tree.h>
 33#include "rcu_segcblist.h"
 34#include "rcu.h"
 35
 36int rcu_scheduler_active __read_mostly;
 37
 38static int init_srcu_struct_fields(struct srcu_struct *sp)
 39{
 40	sp->srcu_lock_nesting[0] = 0;
 41	sp->srcu_lock_nesting[1] = 0;
 42	init_swait_queue_head(&sp->srcu_wq);
 43	sp->srcu_cb_head = NULL;
 44	sp->srcu_cb_tail = &sp->srcu_cb_head;
 45	sp->srcu_gp_running = false;
 46	sp->srcu_gp_waiting = false;
 47	sp->srcu_idx = 0;
 48	INIT_WORK(&sp->srcu_work, srcu_drive_gp);
 49	return 0;
 50}
 51
 52#ifdef CONFIG_DEBUG_LOCK_ALLOC
 53
 54int __init_srcu_struct(struct srcu_struct *sp, const char *name,
 55		       struct lock_class_key *key)
 56{
 57	/* Don't re-initialize a lock while it is held. */
 58	debug_check_no_locks_freed((void *)sp, sizeof(*sp));
 59	lockdep_init_map(&sp->dep_map, name, key, 0);
 60	return init_srcu_struct_fields(sp);
 61}
 62EXPORT_SYMBOL_GPL(__init_srcu_struct);
 63
 64#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 65
 66/*
 67 * init_srcu_struct - initialize a sleep-RCU structure
 68 * @sp: structure to initialize.
 69 *
 70 * Must invoke this on a given srcu_struct before passing that srcu_struct
 71 * to any other function.  Each srcu_struct represents a separate domain
 72 * of SRCU protection.
 73 */
 74int init_srcu_struct(struct srcu_struct *sp)
 75{
 76	return init_srcu_struct_fields(sp);
 77}
 78EXPORT_SYMBOL_GPL(init_srcu_struct);
 79
 80#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 81
 82/*
 83 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
 84 * @sp: structure to clean up.
 85 *
 86 * Must invoke this after you are finished using a given srcu_struct that
 87 * was initialized via init_srcu_struct(), else you leak memory.
 88 */
 89void cleanup_srcu_struct(struct srcu_struct *sp)
 90{
 91	WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);
 92	flush_work(&sp->srcu_work);
 93	WARN_ON(sp->srcu_gp_running);
 94	WARN_ON(sp->srcu_gp_waiting);
 95	WARN_ON(sp->srcu_cb_head);
 96	WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);
 97}
 98EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
 99
100/*
101 * Removes the count for the old reader from the appropriate element of
102 * the srcu_struct.
103 */
104void __srcu_read_unlock(struct srcu_struct *sp, int idx)
105{
106	int newval = sp->srcu_lock_nesting[idx] - 1;
107
108	WRITE_ONCE(sp->srcu_lock_nesting[idx], newval);
109	if (!newval && READ_ONCE(sp->srcu_gp_waiting))
110		swake_up(&sp->srcu_wq);
111}
112EXPORT_SYMBOL_GPL(__srcu_read_unlock);
113
114/*
115 * Workqueue handler to drive one grace period and invoke any callbacks
116 * that become ready as a result.  Single-CPU and !PREEMPT operation
117 * means that we get away with murder on synchronization.  ;-)
118 */
119void srcu_drive_gp(struct work_struct *wp)
120{
121	int idx;
122	struct rcu_head *lh;
123	struct rcu_head *rhp;
124	struct srcu_struct *sp;
125
126	sp = container_of(wp, struct srcu_struct, srcu_work);
127	if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))
128		return; /* Already running or nothing to do. */
129
130	/* Remove recently arrived callbacks and wait for readers. */
131	WRITE_ONCE(sp->srcu_gp_running, true);
132	local_irq_disable();
133	lh = sp->srcu_cb_head;
134	sp->srcu_cb_head = NULL;
135	sp->srcu_cb_tail = &sp->srcu_cb_head;
136	local_irq_enable();
137	idx = sp->srcu_idx;
138	WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);
139	WRITE_ONCE(sp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
140	swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));
141	WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
142
143	/* Invoke the callbacks we removed above. */
144	while (lh) {
145		rhp = lh;
146		lh = lh->next;
147		local_bh_disable();
148		rhp->func(rhp);
149		local_bh_enable();
150	}
151
152	/*
153	 * Enable rescheduling, and if there are more callbacks,
154	 * reschedule ourselves.  This can race with a call_srcu()
155	 * at interrupt level, but the ->srcu_gp_running checks will
156	 * straighten that out.
157	 */
158	WRITE_ONCE(sp->srcu_gp_running, false);
159	if (READ_ONCE(sp->srcu_cb_head))
160		schedule_work(&sp->srcu_work);
161}
162EXPORT_SYMBOL_GPL(srcu_drive_gp);
163
164/*
165 * Enqueue an SRCU callback on the specified srcu_struct structure,
166 * initiating grace-period processing if it is not already running.
167 */
168void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
169	       rcu_callback_t func)
170{
171	unsigned long flags;
172
173	rhp->func = func;
174	rhp->next = NULL;
175	local_irq_save(flags);
176	*sp->srcu_cb_tail = rhp;
177	sp->srcu_cb_tail = &rhp->next;
178	local_irq_restore(flags);
179	if (!READ_ONCE(sp->srcu_gp_running))
180		schedule_work(&sp->srcu_work);
181}
182EXPORT_SYMBOL_GPL(call_srcu);
183
184/*
185 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
186 */
187void synchronize_srcu(struct srcu_struct *sp)
188{
189	struct rcu_synchronize rs;
190
191	init_rcu_head_on_stack(&rs.head);
192	init_completion(&rs.completion);
193	call_srcu(sp, &rs.head, wakeme_after_rcu);
194	wait_for_completion(&rs.completion);
195	destroy_rcu_head_on_stack(&rs.head);
196}
197EXPORT_SYMBOL_GPL(synchronize_srcu);
198
199/* Lockdep diagnostics.  */
200void __init rcu_scheduler_starting(void)
201{
202	rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
203}