1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * Read-Copy Update module-based scalability-test facility
  4 *
  5 * Copyright (C) IBM Corporation, 2015
  6 *
  7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8 */
  9
 10#define pr_fmt(fmt) fmt
 11
 12#include <linux/types.h>
 13#include <linux/kernel.h>
 14#include <linux/init.h>
 15#include <linux/mm.h>
 16#include <linux/module.h>
 17#include <linux/kthread.h>
 18#include <linux/err.h>
 19#include <linux/spinlock.h>
 20#include <linux/smp.h>
 21#include <linux/rcupdate.h>
 22#include <linux/interrupt.h>
 23#include <linux/sched.h>
 24#include <uapi/linux/sched/types.h>
 25#include <linux/atomic.h>
 26#include <linux/bitops.h>
 27#include <linux/completion.h>
 28#include <linux/moduleparam.h>
 29#include <linux/percpu.h>
 30#include <linux/notifier.h>
 31#include <linux/reboot.h>
 32#include <linux/freezer.h>
 33#include <linux/cpu.h>
 34#include <linux/delay.h>
 35#include <linux/stat.h>
 36#include <linux/srcu.h>
 37#include <linux/slab.h>
 38#include <asm/byteorder.h>
 39#include <linux/torture.h>
 40#include <linux/vmalloc.h>
 41#include <linux/rcupdate_trace.h>
 
 42
 43#include "rcu.h"
 44
 
 45MODULE_LICENSE("GPL");
 46MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
 47
 48#define SCALE_FLAG "-scale:"
 49#define SCALEOUT_STRING(s) \
 50	pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
 51#define VERBOSE_SCALEOUT_STRING(s) \
 52	do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
 53#define VERBOSE_SCALEOUT_ERRSTRING(s) \
 54	do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s); } while (0)
 55
 56/*
 57 * The intended use cases for the nreaders and nwriters module parameters
 58 * are as follows:
 59 *
 60 * 1.	Specify only the nr_cpus kernel boot parameter.  This will
 61 *	set both nreaders and nwriters to the value specified by
 62 *	nr_cpus for a mixed reader/writer test.
 63 *
 64 * 2.	Specify the nr_cpus kernel boot parameter, but set
 65 *	rcuscale.nreaders to zero.  This will set nwriters to the
 66 *	value specified by nr_cpus for an update-only test.
 67 *
 68 * 3.	Specify the nr_cpus kernel boot parameter, but set
 69 *	rcuscale.nwriters to zero.  This will set nreaders to the
 70 *	value specified by nr_cpus for a read-only test.
 71 *
 72 * Various other use cases may of course be specified.
 73 *
 74 * Note that this test's readers are intended only as a test load for
 75 * the writers.  The reader scalability statistics will be overly
 76 * pessimistic due to the per-critical-section interrupt disabling,
 77 * test-end checks, and the pair of calls through pointers.
 78 */
 79
 80#ifdef MODULE
 81# define RCUSCALE_SHUTDOWN 0
 82#else
 83# define RCUSCALE_SHUTDOWN 1
 84#endif
 85
 86torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives");
 87torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader");
 88torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 89torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
 
 90torture_param(int, nreaders, -1, "Number of RCU reader threads");
 91torture_param(int, nwriters, -1, "Number of RCU updater threads");
 92torture_param(bool, shutdown, RCUSCALE_SHUTDOWN,
 93	      "Shutdown at end of scalability tests.");
 94torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
 95torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
 
 96torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?");
 97torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate.");
 
 98
 99static char *scale_type = "rcu";
100module_param(scale_type, charp, 0444);
101MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)");
102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103static int nrealreaders;
104static int nrealwriters;
105static struct task_struct **writer_tasks;
106static struct task_struct **reader_tasks;
107static struct task_struct *shutdown_task;
108
109static u64 **writer_durations;
 
 
110static int *writer_n_durations;
111static atomic_t n_rcu_scale_reader_started;
112static atomic_t n_rcu_scale_writer_started;
113static atomic_t n_rcu_scale_writer_finished;
114static wait_queue_head_t shutdown_wq;
115static u64 t_rcu_scale_writer_started;
116static u64 t_rcu_scale_writer_finished;
117static unsigned long b_rcu_gp_test_started;
118static unsigned long b_rcu_gp_test_finished;
119static DEFINE_PER_CPU(atomic_t, n_async_inflight);
120
121#define MAX_MEAS 10000
122#define MIN_MEAS 100
123
124/*
125 * Operations vector for selecting different types of tests.
126 */
127
128struct rcu_scale_ops {
129	int ptype;
130	void (*init)(void);
131	void (*cleanup)(void);
132	int (*readlock)(void);
133	void (*readunlock)(int idx);
134	unsigned long (*get_gp_seq)(void);
135	unsigned long (*gp_diff)(unsigned long new, unsigned long old);
136	unsigned long (*exp_completed)(void);
137	void (*async)(struct rcu_head *head, rcu_callback_t func);
138	void (*gp_barrier)(void);
139	void (*sync)(void);
140	void (*exp_sync)(void);
 
 
141	const char *name;
142};
143
144static struct rcu_scale_ops *cur_ops;
145
146/*
147 * Definitions for rcu scalability testing.
148 */
149
150static int rcu_scale_read_lock(void) __acquires(RCU)
151{
152	rcu_read_lock();
153	return 0;
154}
155
156static void rcu_scale_read_unlock(int idx) __releases(RCU)
157{
158	rcu_read_unlock();
159}
160
161static unsigned long __maybe_unused rcu_no_completed(void)
162{
163	return 0;
164}
165
166static void rcu_sync_scale_init(void)
167{
168}
169
170static struct rcu_scale_ops rcu_ops = {
171	.ptype		= RCU_FLAVOR,
172	.init		= rcu_sync_scale_init,
173	.readlock	= rcu_scale_read_lock,
174	.readunlock	= rcu_scale_read_unlock,
175	.get_gp_seq	= rcu_get_gp_seq,
176	.gp_diff	= rcu_seq_diff,
177	.exp_completed	= rcu_exp_batches_completed,
178	.async		= call_rcu,
179	.gp_barrier	= rcu_barrier,
180	.sync		= synchronize_rcu,
181	.exp_sync	= synchronize_rcu_expedited,
182	.name		= "rcu"
183};
184
185/*
186 * Definitions for srcu scalability testing.
187 */
188
189DEFINE_STATIC_SRCU(srcu_ctl_scale);
190static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale;
191
192static int srcu_scale_read_lock(void) __acquires(srcu_ctlp)
193{
194	return srcu_read_lock(srcu_ctlp);
195}
196
197static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp)
198{
199	srcu_read_unlock(srcu_ctlp, idx);
200}
201
202static unsigned long srcu_scale_completed(void)
203{
204	return srcu_batches_completed(srcu_ctlp);
205}
206
207static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func)
208{
209	call_srcu(srcu_ctlp, head, func);
210}
211
212static void srcu_rcu_barrier(void)
213{
214	srcu_barrier(srcu_ctlp);
215}
216
217static void srcu_scale_synchronize(void)
218{
219	synchronize_srcu(srcu_ctlp);
220}
221
 
 
 
 
 
222static void srcu_scale_synchronize_expedited(void)
223{
224	synchronize_srcu_expedited(srcu_ctlp);
225}
226
227static struct rcu_scale_ops srcu_ops = {
228	.ptype		= SRCU_FLAVOR,
229	.init		= rcu_sync_scale_init,
230	.readlock	= srcu_scale_read_lock,
231	.readunlock	= srcu_scale_read_unlock,
232	.get_gp_seq	= srcu_scale_completed,
233	.gp_diff	= rcu_seq_diff,
234	.exp_completed	= srcu_scale_completed,
235	.async		= srcu_call_rcu,
236	.gp_barrier	= srcu_rcu_barrier,
237	.sync		= srcu_scale_synchronize,
238	.exp_sync	= srcu_scale_synchronize_expedited,
 
239	.name		= "srcu"
240};
241
242static struct srcu_struct srcud;
243
244static void srcu_sync_scale_init(void)
245{
246	srcu_ctlp = &srcud;
247	init_srcu_struct(srcu_ctlp);
248}
249
250static void srcu_sync_scale_cleanup(void)
251{
252	cleanup_srcu_struct(srcu_ctlp);
253}
254
255static struct rcu_scale_ops srcud_ops = {
256	.ptype		= SRCU_FLAVOR,
257	.init		= srcu_sync_scale_init,
258	.cleanup	= srcu_sync_scale_cleanup,
259	.readlock	= srcu_scale_read_lock,
260	.readunlock	= srcu_scale_read_unlock,
261	.get_gp_seq	= srcu_scale_completed,
262	.gp_diff	= rcu_seq_diff,
263	.exp_completed	= srcu_scale_completed,
264	.async		= srcu_call_rcu,
265	.gp_barrier	= srcu_rcu_barrier,
266	.sync		= srcu_scale_synchronize,
267	.exp_sync	= srcu_scale_synchronize_expedited,
 
268	.name		= "srcud"
269};
270
 
 
271/*
272 * Definitions for RCU-tasks scalability testing.
273 */
274
275static int tasks_scale_read_lock(void)
276{
277	return 0;
278}
279
280static void tasks_scale_read_unlock(int idx)
281{
282}
283
 
 
 
 
 
284static struct rcu_scale_ops tasks_ops = {
285	.ptype		= RCU_TASKS_FLAVOR,
286	.init		= rcu_sync_scale_init,
287	.readlock	= tasks_scale_read_lock,
288	.readunlock	= tasks_scale_read_unlock,
289	.get_gp_seq	= rcu_no_completed,
290	.gp_diff	= rcu_seq_diff,
291	.async		= call_rcu_tasks,
292	.gp_barrier	= rcu_barrier_tasks,
293	.sync		= synchronize_rcu_tasks,
294	.exp_sync	= synchronize_rcu_tasks,
 
 
295	.name		= "tasks"
296};
297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
298/*
299 * Definitions for RCU-tasks-trace scalability testing.
300 */
301
302static int tasks_trace_scale_read_lock(void)
303{
304	rcu_read_lock_trace();
305	return 0;
306}
307
308static void tasks_trace_scale_read_unlock(int idx)
309{
310	rcu_read_unlock_trace();
311}
312
 
 
 
 
 
313static struct rcu_scale_ops tasks_tracing_ops = {
314	.ptype		= RCU_TASKS_FLAVOR,
315	.init		= rcu_sync_scale_init,
316	.readlock	= tasks_trace_scale_read_lock,
317	.readunlock	= tasks_trace_scale_read_unlock,
318	.get_gp_seq	= rcu_no_completed,
319	.gp_diff	= rcu_seq_diff,
320	.async		= call_rcu_tasks_trace,
321	.gp_barrier	= rcu_barrier_tasks_trace,
322	.sync		= synchronize_rcu_tasks_trace,
323	.exp_sync	= synchronize_rcu_tasks_trace,
 
 
324	.name		= "tasks-tracing"
325};
326
 
 
 
 
 
 
 
 
327static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old)
328{
329	if (!cur_ops->gp_diff)
330		return new - old;
331	return cur_ops->gp_diff(new, old);
332}
333
334/*
335 * If scalability tests complete, wait for shutdown to commence.
336 */
337static void rcu_scale_wait_shutdown(void)
338{
339	cond_resched_tasks_rcu_qs();
340	if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters)
341		return;
342	while (!torture_must_stop())
343		schedule_timeout_uninterruptible(1);
344}
345
346/*
347 * RCU scalability reader kthread.  Repeatedly does empty RCU read-side
348 * critical section, minimizing update-side interference.  However, the
349 * point of this test is not to evaluate reader scalability, but instead
350 * to serve as a test load for update-side scalability testing.
351 */
352static int
353rcu_scale_reader(void *arg)
354{
355	unsigned long flags;
356	int idx;
357	long me = (long)arg;
358
359	VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started");
360	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
361	set_user_nice(current, MAX_NICE);
362	atomic_inc(&n_rcu_scale_reader_started);
363
364	do {
365		local_irq_save(flags);
366		idx = cur_ops->readlock();
367		cur_ops->readunlock(idx);
368		local_irq_restore(flags);
369		rcu_scale_wait_shutdown();
370	} while (!torture_must_stop());
371	torture_kthread_stopping("rcu_scale_reader");
372	return 0;
373}
374
375/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
376 * Callback function for asynchronous grace periods from rcu_scale_writer().
377 */
378static void rcu_scale_async_cb(struct rcu_head *rhp)
379{
380	atomic_dec(this_cpu_ptr(&n_async_inflight));
381	kfree(rhp);
 
 
 
382}
383
384/*
385 * RCU scale writer kthread.  Repeatedly does a grace period.
386 */
387static int
388rcu_scale_writer(void *arg)
389{
390	int i = 0;
391	int i_max;
 
392	long me = (long)arg;
393	struct rcu_head *rhp = NULL;
394	bool started = false, done = false, alldone = false;
395	u64 t;
 
396	u64 *wdp;
397	u64 *wdpp = writer_durations[me];
 
 
398
399	VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started");
400	WARN_ON(!wdpp);
401	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
 
402	sched_set_fifo_low(current);
403
404	if (holdoff)
405		schedule_timeout_uninterruptible(holdoff * HZ);
406
407	/*
408	 * Wait until rcu_end_inkernel_boot() is called for normal GP tests
409	 * so that RCU is not always expedited for normal GP tests.
410	 * The system_state test is approximate, but works well in practice.
411	 */
412	while (!gp_exp && system_state != SYSTEM_RUNNING)
413		schedule_timeout_uninterruptible(1);
414
415	t = ktime_get_mono_fast_ns();
416	if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) {
417		t_rcu_scale_writer_started = t;
418		if (gp_exp) {
419			b_rcu_gp_test_started =
420				cur_ops->exp_completed() / 2;
421		} else {
422			b_rcu_gp_test_started = cur_ops->get_gp_seq();
423		}
424	}
425
 
426	do {
 
 
427		if (writer_holdoff)
428			udelay(writer_holdoff);
 
 
429		wdp = &wdpp[i];
430		*wdp = ktime_get_mono_fast_ns();
431		if (gp_async) {
432retry:
433			if (!rhp)
434				rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
435			if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) {
436				atomic_inc(this_cpu_ptr(&n_async_inflight));
437				cur_ops->async(rhp, rcu_scale_async_cb);
438				rhp = NULL;
439			} else if (!kthread_should_stop()) {
440				cur_ops->gp_barrier();
441				goto retry;
442			} else {
443				kfree(rhp); /* Because we are stopping. */
 
444			}
445		} else if (gp_exp) {
446			cur_ops->exp_sync();
 
447		} else {
448			cur_ops->sync();
 
449		}
450		t = ktime_get_mono_fast_ns();
451		*wdp = t - *wdp;
452		i_max = i;
453		if (!started &&
454		    atomic_read(&n_rcu_scale_writer_started) >= nrealwriters)
455			started = true;
456		if (!done && i >= MIN_MEAS) {
457			done = true;
 
458			sched_set_normal(current, 0);
459			pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n",
460				 scale_type, SCALE_FLAG, me, MIN_MEAS);
461			if (atomic_inc_return(&n_rcu_scale_writer_finished) >=
462			    nrealwriters) {
463				schedule_timeout_interruptible(10);
464				rcu_ftrace_dump(DUMP_ALL);
465				SCALEOUT_STRING("Test complete");
466				t_rcu_scale_writer_finished = t;
467				if (gp_exp) {
468					b_rcu_gp_test_finished =
469						cur_ops->exp_completed() / 2;
470				} else {
471					b_rcu_gp_test_finished =
472						cur_ops->get_gp_seq();
473				}
474				if (shutdown) {
475					smp_mb(); /* Assign before wake. */
476					wake_up(&shutdown_wq);
477				}
478			}
479		}
480		if (done && !alldone &&
481		    atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters)
482			alldone = true;
483		if (started && !alldone && i < MAX_MEAS - 1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
484			i++;
485		rcu_scale_wait_shutdown();
486	} while (!torture_must_stop());
487	if (gp_async) {
 
488		cur_ops->gp_barrier();
489	}
490	writer_n_durations[me] = i_max;
491	torture_kthread_stopping("rcu_scale_writer");
492	return 0;
493}
494
495static void
496rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag)
497{
498	pr_alert("%s" SCALE_FLAG
499		 "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n",
500		 scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown);
501}
502
503static void
504rcu_scale_cleanup(void)
505{
506	int i;
507	int j;
508	int ngps = 0;
509	u64 *wdp;
510	u64 *wdpp;
511
512	/*
513	 * Would like warning at start, but everything is expedited
514	 * during the mid-boot phase, so have to wait till the end.
515	 */
516	if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
517		VERBOSE_SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
518	if (rcu_gp_is_normal() && gp_exp)
519		VERBOSE_SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
520	if (gp_exp && gp_async)
521		VERBOSE_SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
522
523	if (torture_cleanup_begin())
524		return;
525	if (!cur_ops) {
526		torture_cleanup_end();
527		return;
528	}
529
530	if (reader_tasks) {
531		for (i = 0; i < nrealreaders; i++)
532			torture_stop_kthread(rcu_scale_reader,
533					     reader_tasks[i]);
534		kfree(reader_tasks);
535	}
536
537	if (writer_tasks) {
538		for (i = 0; i < nrealwriters; i++) {
539			torture_stop_kthread(rcu_scale_writer,
540					     writer_tasks[i]);
541			if (!writer_n_durations)
542				continue;
543			j = writer_n_durations[i];
544			pr_alert("%s%s writer %d gps: %d\n",
545				 scale_type, SCALE_FLAG, i, j);
546			ngps += j;
547		}
548		pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n",
549			 scale_type, SCALE_FLAG,
550			 t_rcu_scale_writer_started, t_rcu_scale_writer_finished,
551			 t_rcu_scale_writer_finished -
552			 t_rcu_scale_writer_started,
553			 ngps,
554			 rcuscale_seq_diff(b_rcu_gp_test_finished,
555					   b_rcu_gp_test_started));
556		for (i = 0; i < nrealwriters; i++) {
557			if (!writer_durations)
558				break;
559			if (!writer_n_durations)
560				continue;
561			wdpp = writer_durations[i];
562			if (!wdpp)
563				continue;
564			for (j = 0; j <= writer_n_durations[i]; j++) {
565				wdp = &wdpp[j];
566				pr_alert("%s%s %4d writer-duration: %5d %llu\n",
567					scale_type, SCALE_FLAG,
568					i, j, *wdp);
569				if (j % 100 == 0)
570					schedule_timeout_uninterruptible(1);
571			}
572			kfree(writer_durations[i]);
573		}
574		kfree(writer_tasks);
575		kfree(writer_durations);
576		kfree(writer_n_durations);
577	}
578
579	/* Do torture-type-specific cleanup operations.  */
580	if (cur_ops->cleanup != NULL)
581		cur_ops->cleanup();
582
583	torture_cleanup_end();
584}
585
586/*
587 * Return the number if non-negative.  If -1, the number of CPUs.
588 * If less than -1, that much less than the number of CPUs, but
589 * at least one.
590 */
591static int compute_real(int n)
592{
593	int nr;
594
595	if (n >= 0) {
596		nr = n;
597	} else {
598		nr = num_online_cpus() + 1 + n;
599		if (nr <= 0)
600			nr = 1;
601	}
602	return nr;
603}
604
605/*
606 * RCU scalability shutdown kthread.  Just waits to be awakened, then shuts
607 * down system.
608 */
609static int
610rcu_scale_shutdown(void *arg)
611{
612	wait_event(shutdown_wq,
613		   atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters);
614	smp_mb(); /* Wake before output. */
615	rcu_scale_cleanup();
616	kernel_power_off();
617	return -EINVAL;
618}
619
620/*
621 * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number
622 * of iterations and measure total time and number of GP for all iterations to complete.
623 */
624
625torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu().");
626torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration.");
627torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees.");
628torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?");
629torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?");
630
631static struct task_struct **kfree_reader_tasks;
632static int kfree_nrealthreads;
633static atomic_t n_kfree_scale_thread_started;
634static atomic_t n_kfree_scale_thread_ended;
 
 
635
636struct kfree_obj {
637	char kfree_obj[8];
638	struct rcu_head rh;
639};
640
 
 
 
 
 
 
 
 
641static int
642kfree_scale_thread(void *arg)
643{
644	int i, loop = 0;
645	long me = (long)arg;
646	struct kfree_obj *alloc_ptr;
647	u64 start_time, end_time;
648	long long mem_begin, mem_during = 0;
649	bool kfree_rcu_test_both;
650	DEFINE_TORTURE_RANDOM(tr);
651
652	VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started");
653	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
654	set_user_nice(current, MAX_NICE);
655	kfree_rcu_test_both = (kfree_rcu_test_single == kfree_rcu_test_double);
656
657	start_time = ktime_get_mono_fast_ns();
658
659	if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) {
660		if (gp_exp)
661			b_rcu_gp_test_started = cur_ops->exp_completed() / 2;
662		else
663			b_rcu_gp_test_started = cur_ops->get_gp_seq();
664	}
665
666	do {
667		if (!mem_during) {
668			mem_during = mem_begin = si_mem_available();
669		} else if (loop % (kfree_loops / 4) == 0) {
670			mem_during = (mem_during + si_mem_available()) / 2;
671		}
672
673		for (i = 0; i < kfree_alloc_num; i++) {
674			alloc_ptr = kmalloc(kfree_mult * sizeof(struct kfree_obj), GFP_KERNEL);
675			if (!alloc_ptr)
676				return -ENOMEM;
677
 
 
 
 
 
678			// By default kfree_rcu_test_single and kfree_rcu_test_double are
679			// initialized to false. If both have the same value (false or true)
680			// both are randomly tested, otherwise only the one with value true
681			// is tested.
682			if ((kfree_rcu_test_single && !kfree_rcu_test_double) ||
683					(kfree_rcu_test_both && torture_random(&tr) & 0x800))
684				kfree_rcu(alloc_ptr);
685			else
686				kfree_rcu(alloc_ptr, rh);
687		}
688
689		cond_resched();
690	} while (!torture_must_stop() && ++loop < kfree_loops);
691
692	if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) {
693		end_time = ktime_get_mono_fast_ns();
694
695		if (gp_exp)
696			b_rcu_gp_test_finished = cur_ops->exp_completed() / 2;
697		else
698			b_rcu_gp_test_finished = cur_ops->get_gp_seq();
699
700		pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
701		       (unsigned long long)(end_time - start_time), kfree_loops,
702		       rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
703		       (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
704
705		if (shutdown) {
706			smp_mb(); /* Assign before wake. */
707			wake_up(&shutdown_wq);
708		}
709	}
710
711	torture_kthread_stopping("kfree_scale_thread");
712	return 0;
713}
714
715static void
716kfree_scale_cleanup(void)
717{
718	int i;
719
720	if (torture_cleanup_begin())
721		return;
722
723	if (kfree_reader_tasks) {
724		for (i = 0; i < kfree_nrealthreads; i++)
725			torture_stop_kthread(kfree_scale_thread,
726					     kfree_reader_tasks[i]);
727		kfree(kfree_reader_tasks);
 
728	}
729
730	torture_cleanup_end();
731}
732
733/*
734 * shutdown kthread.  Just waits to be awakened, then shuts down system.
735 */
736static int
737kfree_scale_shutdown(void *arg)
738{
739	wait_event(shutdown_wq,
740		   atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads);
741
742	smp_mb(); /* Wake before output. */
743
744	kfree_scale_cleanup();
745	kernel_power_off();
746	return -EINVAL;
747}
748
 
 
 
 
 
 
 
 
 
 
749static int __init
750kfree_scale_init(void)
751{
752	long i;
753	int firsterr = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
754
755	kfree_nrealthreads = compute_real(kfree_nthreads);
756	/* Start up the kthreads. */
757	if (shutdown) {
758		init_waitqueue_head(&shutdown_wq);
759		firsterr = torture_create_kthread(kfree_scale_shutdown, NULL,
760						  shutdown_task);
761		if (firsterr)
762			goto unwind;
763		schedule_timeout_uninterruptible(1);
764	}
765
766	pr_alert("kfree object size=%zu\n", kfree_mult * sizeof(struct kfree_obj));
 
 
767
768	kfree_reader_tasks = kcalloc(kfree_nrealthreads, sizeof(kfree_reader_tasks[0]),
769			       GFP_KERNEL);
770	if (kfree_reader_tasks == NULL) {
771		firsterr = -ENOMEM;
772		goto unwind;
773	}
774
775	for (i = 0; i < kfree_nrealthreads; i++) {
776		firsterr = torture_create_kthread(kfree_scale_thread, (void *)i,
777						  kfree_reader_tasks[i]);
778		if (firsterr)
779			goto unwind;
780	}
781
782	while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads)
783		schedule_timeout_uninterruptible(1);
784
785	torture_init_end();
786	return 0;
787
788unwind:
789	torture_init_end();
790	kfree_scale_cleanup();
791	return firsterr;
792}
793
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
794static int __init
795rcu_scale_init(void)
796{
797	long i;
798	int firsterr = 0;
 
 
799	static struct rcu_scale_ops *scale_ops[] = {
800		&rcu_ops, &srcu_ops, &srcud_ops, &tasks_ops, &tasks_tracing_ops
801	};
802
803	if (!torture_init_begin(scale_type, verbose))
804		return -EBUSY;
805
806	/* Process args and announce that the scalability'er is on the job. */
807	for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
808		cur_ops = scale_ops[i];
809		if (strcmp(scale_type, cur_ops->name) == 0)
810			break;
811	}
812	if (i == ARRAY_SIZE(scale_ops)) {
813		pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
814		pr_alert("rcu-scale types:");
815		for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
816			pr_cont(" %s", scale_ops[i]->name);
817		pr_cont("\n");
818		firsterr = -EINVAL;
819		cur_ops = NULL;
820		goto unwind;
821	}
822	if (cur_ops->init)
823		cur_ops->init();
824
 
 
 
 
 
825	if (kfree_rcu_test)
826		return kfree_scale_init();
827
828	nrealwriters = compute_real(nwriters);
829	nrealreaders = compute_real(nreaders);
830	atomic_set(&n_rcu_scale_reader_started, 0);
831	atomic_set(&n_rcu_scale_writer_started, 0);
832	atomic_set(&n_rcu_scale_writer_finished, 0);
833	rcu_scale_print_module_parms(cur_ops, "Start of test");
834
835	/* Start up the kthreads. */
836
837	if (shutdown) {
838		init_waitqueue_head(&shutdown_wq);
839		firsterr = torture_create_kthread(rcu_scale_shutdown, NULL,
840						  shutdown_task);
841		if (firsterr)
842			goto unwind;
843		schedule_timeout_uninterruptible(1);
844	}
845	reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
846			       GFP_KERNEL);
847	if (reader_tasks == NULL) {
848		VERBOSE_SCALEOUT_ERRSTRING("out of memory");
849		firsterr = -ENOMEM;
850		goto unwind;
851	}
852	for (i = 0; i < nrealreaders; i++) {
853		firsterr = torture_create_kthread(rcu_scale_reader, (void *)i,
854						  reader_tasks[i]);
855		if (firsterr)
856			goto unwind;
857	}
858	while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders)
859		schedule_timeout_uninterruptible(1);
860	writer_tasks = kcalloc(nrealwriters, sizeof(reader_tasks[0]),
861			       GFP_KERNEL);
862	writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations),
863				   GFP_KERNEL);
864	writer_n_durations =
865		kcalloc(nrealwriters, sizeof(*writer_n_durations),
866			GFP_KERNEL);
867	if (!writer_tasks || !writer_durations || !writer_n_durations) {
868		VERBOSE_SCALEOUT_ERRSTRING("out of memory");
 
 
 
 
 
 
 
 
869		firsterr = -ENOMEM;
870		goto unwind;
871	}
872	for (i = 0; i < nrealwriters; i++) {
873		writer_durations[i] =
874			kcalloc(MAX_MEAS, sizeof(*writer_durations[i]),
875				GFP_KERNEL);
876		if (!writer_durations[i]) {
877			firsterr = -ENOMEM;
878			goto unwind;
879		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
880		firsterr = torture_create_kthread(rcu_scale_writer, (void *)i,
881						  writer_tasks[i]);
882		if (firsterr)
883			goto unwind;
884	}
885	torture_init_end();
886	return 0;
887
888unwind:
889	torture_init_end();
890	rcu_scale_cleanup();
891	if (shutdown) {
892		WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST));
893		kernel_power_off();
894	}
895	return firsterr;
896}
897
898module_init(rcu_scale_init);
899module_exit(rcu_scale_cleanup);