1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2018 Davidlohr Bueso.
  4 *
  5 * Benchmark the various operations allowed for epoll_ctl(2).
  6 * The idea is to concurrently stress a single epoll instance
  7 */
  8#ifdef HAVE_EVENTFD
  9/* For the CLR_() macros */
 10#include <string.h>
 11#include <pthread.h>
 12
 13#include <errno.h>
 14#include <inttypes.h>
 15#include <signal.h>
 16#include <stdlib.h>
 17#include <unistd.h>
 18#include <linux/compiler.h>
 19#include <linux/kernel.h>
 20#include <sys/time.h>
 21#include <sys/resource.h>
 22#include <sys/epoll.h>
 23#include <sys/eventfd.h>
 24#include <internal/cpumap.h>
 25#include <perf/cpumap.h>
 26
 27#include "../util/stat.h"
 28#include <subcmd/parse-options.h>
 29#include "bench.h"
 30
 31#include <err.h>
 32
 33#define printinfo(fmt, arg...) \
 34	do { if (__verbose) printf(fmt, ## arg); } while (0)
 35
 36static unsigned int nthreads = 0;
 37static unsigned int nsecs    = 8;
 38struct timeval start, end, runtime;
 39static bool done, __verbose, randomize;
 40
 41/*
 42 * epoll related shared variables.
 43 */
 44
 45/* Maximum number of nesting allowed inside epoll sets */
 46#define EPOLL_MAXNESTS 4
 47
 48enum {
 49	OP_EPOLL_ADD,
 50	OP_EPOLL_MOD,
 51	OP_EPOLL_DEL,
 52	EPOLL_NR_OPS,
 53};
 54
 55static int epollfd;
 56static int *epollfdp;
 57static bool noaffinity;
 58static unsigned int nested = 0;
 59
 60/* amount of fds to monitor, per thread */
 61static unsigned int nfds = 64;
 62
 63static pthread_mutex_t thread_lock;
 64static unsigned int threads_starting;
 65static struct stats all_stats[EPOLL_NR_OPS];
 66static pthread_cond_t thread_parent, thread_worker;
 67
 68struct worker {
 69	int tid;
 70	pthread_t thread;
 71	unsigned long ops[EPOLL_NR_OPS];
 72	int *fdmap;
 73};
 74
 75static const struct option options[] = {
 76	OPT_UINTEGER('t', "threads", &nthreads, "Specify amount of threads"),
 77	OPT_UINTEGER('r', "runtime", &nsecs,    "Specify runtime (in seconds)"),
 78	OPT_UINTEGER('f', "nfds", &nfds, "Specify amount of file descriptors to monitor for each thread"),
 79	OPT_BOOLEAN( 'n', "noaffinity",  &noaffinity,   "Disables CPU affinity"),
 80	OPT_UINTEGER( 'N', "nested",  &nested,   "Nesting level epoll hierarchy (default is 0, no nesting)"),
 81	OPT_BOOLEAN( 'R', "randomize", &randomize,   "Perform random operations on random fds"),
 82	OPT_BOOLEAN( 'v', "verbose",  &__verbose,   "Verbose mode"),
 83	OPT_END()
 84};
 85
 86static const char * const bench_epoll_ctl_usage[] = {
 87	"perf bench epoll ctl <options>",
 88	NULL
 89};
 90
 91static void toggle_done(int sig __maybe_unused,
 92			siginfo_t *info __maybe_unused,
 93			void *uc __maybe_unused)
 94{
 95	/* inform all threads that we're done for the day */
 96	done = true;
 97	gettimeofday(&end, NULL);
 98	timersub(&end, &start, &runtime);
 99}
100
101static void nest_epollfd(void)
102{
103	unsigned int i;
104	struct epoll_event ev;
105
106	if (nested > EPOLL_MAXNESTS)
107		nested = EPOLL_MAXNESTS;
108	printinfo("Nesting level(s): %d\n", nested);
109
110	epollfdp = calloc(nested, sizeof(int));
111	if (!epollfd)
112		err(EXIT_FAILURE, "calloc");
113
114	for (i = 0; i < nested; i++) {
115		epollfdp[i] = epoll_create(1);
116		if (epollfd < 0)
117			err(EXIT_FAILURE, "epoll_create");
118	}
119
120	ev.events = EPOLLHUP; /* anything */
121	ev.data.u64 = i; /* any number */
122
123	for (i = nested - 1; i; i--) {
124		if (epoll_ctl(epollfdp[i - 1], EPOLL_CTL_ADD,
125			      epollfdp[i], &ev) < 0)
126			err(EXIT_FAILURE, "epoll_ctl");
127	}
128
129	if (epoll_ctl(epollfd, EPOLL_CTL_ADD, *epollfdp, &ev) < 0)
130		err(EXIT_FAILURE, "epoll_ctl");
131}
132
133static inline void do_epoll_op(struct worker *w, int op, int fd)
134{
135	int error;
136	struct epoll_event ev;
137
138	ev.events = EPOLLIN;
139	ev.data.u64 = fd;
140
141	switch (op) {
142	case OP_EPOLL_ADD:
143		error = epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev);
144		break;
145	case OP_EPOLL_MOD:
146		ev.events = EPOLLOUT;
147		error = epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev);
148		break;
149	case OP_EPOLL_DEL:
150		error = epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, NULL);
151		break;
152	default:
153		error = 1;
154		break;
155	}
156
157	if (!error)
158		w->ops[op]++;
159}
160
161static inline void do_random_epoll_op(struct worker *w)
162{
163	unsigned long rnd1 = random(), rnd2 = random();
164	int op, fd;
165
166	fd = w->fdmap[rnd1 % nfds];
167	op = rnd2 % EPOLL_NR_OPS;
168
169	do_epoll_op(w, op, fd);
170}
171
172static void *workerfn(void *arg)
173{
174	unsigned int i;
175	struct worker *w = (struct worker *) arg;
176	struct timespec ts = { .tv_sec = 0,
177			       .tv_nsec = 250 };
178
179	pthread_mutex_lock(&thread_lock);
180	threads_starting--;
181	if (!threads_starting)
182		pthread_cond_signal(&thread_parent);
183	pthread_cond_wait(&thread_worker, &thread_lock);
184	pthread_mutex_unlock(&thread_lock);
185
186	/* Let 'em loose */
187	do {
188		/* random */
189		if (randomize) {
190			do_random_epoll_op(w);
191		} else {
192			for (i = 0; i < nfds; i++) {
193				do_epoll_op(w, OP_EPOLL_ADD, w->fdmap[i]);
194				do_epoll_op(w, OP_EPOLL_MOD, w->fdmap[i]);
195				do_epoll_op(w, OP_EPOLL_DEL, w->fdmap[i]);
196			}
197		}
198
199		nanosleep(&ts, NULL);
200	}  while (!done);
201
202	return NULL;
203}
204
205static void init_fdmaps(struct worker *w, int pct)
206{
207	unsigned int i;
208	int inc;
209	struct epoll_event ev;
210
211	if (!pct)
212		return;
213
214	inc = 100/pct;
215	for (i = 0; i < nfds; i+=inc) {
216		ev.data.fd = w->fdmap[i];
217		ev.events = EPOLLIN;
218
219		if (epoll_ctl(epollfd, EPOLL_CTL_ADD, w->fdmap[i], &ev) < 0)
220			err(EXIT_FAILURE, "epoll_ct");
221	}
222}
223
224static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
225{
226	pthread_attr_t thread_attr, *attrp = NULL;
227	cpu_set_t cpuset;
228	unsigned int i, j;
229	int ret = 0;
230
231	if (!noaffinity)
232		pthread_attr_init(&thread_attr);
233
234	for (i = 0; i < nthreads; i++) {
235		struct worker *w = &worker[i];
236
237		w->tid = i;
238		w->fdmap = calloc(nfds, sizeof(int));
239		if (!w->fdmap)
240			return 1;
241
242		for (j = 0; j < nfds; j++) {
243			w->fdmap[j] = eventfd(0, EFD_NONBLOCK);
244			if (w->fdmap[j] < 0)
245				err(EXIT_FAILURE, "eventfd");
246		}
247
248		/*
249		 * Lets add 50% of the fdmap to the epoll instance, and
250		 * do it before any threads are started; otherwise there is
251		 * an initial bias of the call failing  (mod and del ops).
252		 */
253		if (randomize)
254			init_fdmaps(w, 50);
255
256		if (!noaffinity) {
257			CPU_ZERO(&cpuset);
258			CPU_SET(cpu->map[i % cpu->nr], &cpuset);
259
260			ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpuset);
261			if (ret)
262				err(EXIT_FAILURE, "pthread_attr_setaffinity_np");
263
264			attrp = &thread_attr;
265		}
266
267		ret = pthread_create(&w->thread, attrp, workerfn,
268				     (void *)(struct worker *) w);
269		if (ret)
270			err(EXIT_FAILURE, "pthread_create");
271	}
272
273	if (!noaffinity)
274		pthread_attr_destroy(&thread_attr);
275
276	return ret;
277}
278
279static void print_summary(void)
280{
281	int i;
282	unsigned long avg[EPOLL_NR_OPS];
283	double stddev[EPOLL_NR_OPS];
284
285	for (i = 0; i < EPOLL_NR_OPS; i++) {
286		avg[i] = avg_stats(&all_stats[i]);
287		stddev[i] = stddev_stats(&all_stats[i]);
288	}
289
290	printf("\nAveraged %ld ADD operations (+- %.2f%%)\n",
291	       avg[OP_EPOLL_ADD], rel_stddev_stats(stddev[OP_EPOLL_ADD],
292						   avg[OP_EPOLL_ADD]));
293	printf("Averaged %ld MOD operations (+- %.2f%%)\n",
294	       avg[OP_EPOLL_MOD], rel_stddev_stats(stddev[OP_EPOLL_MOD],
295						   avg[OP_EPOLL_MOD]));
296	printf("Averaged %ld DEL operations (+- %.2f%%)\n",
297	       avg[OP_EPOLL_DEL], rel_stddev_stats(stddev[OP_EPOLL_DEL],
298						   avg[OP_EPOLL_DEL]));
299}
300
301int bench_epoll_ctl(int argc, const char **argv)
302{
303	int j, ret = 0;
304	struct sigaction act;
305	struct worker *worker = NULL;
306	struct perf_cpu_map *cpu;
307	struct rlimit rl, prevrl;
308	unsigned int i;
309
310	argc = parse_options(argc, argv, options, bench_epoll_ctl_usage, 0);
311	if (argc) {
312		usage_with_options(bench_epoll_ctl_usage, options);
313		exit(EXIT_FAILURE);
314	}
315
316	sigfillset(&act.sa_mask);
317	act.sa_sigaction = toggle_done;
318	sigaction(SIGINT, &act, NULL);
319
320	cpu = perf_cpu_map__new(NULL);
321	if (!cpu)
322		goto errmem;
323
324	/* a single, main epoll instance */
325	epollfd = epoll_create(1);
326	if (epollfd < 0)
327		err(EXIT_FAILURE, "epoll_create");
328
329	/*
330	 * Deal with nested epolls, if any.
331	 */
332	if (nested)
333		nest_epollfd();
334
335	/* default to the number of CPUs */
336	if (!nthreads)
337		nthreads = cpu->nr;
338
339	worker = calloc(nthreads, sizeof(*worker));
340	if (!worker)
341		goto errmem;
342
343	if (getrlimit(RLIMIT_NOFILE, &prevrl))
344	    err(EXIT_FAILURE, "getrlimit");
345	rl.rlim_cur = rl.rlim_max = nfds * nthreads * 2 + 50;
346	printinfo("Setting RLIMIT_NOFILE rlimit from %" PRIu64 " to: %" PRIu64 "\n",
347		  (uint64_t)prevrl.rlim_max, (uint64_t)rl.rlim_max);
348	if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
349		err(EXIT_FAILURE, "setrlimit");
350
351	printf("Run summary [PID %d]: %d threads doing epoll_ctl ops "
352	       "%d file-descriptors for %d secs.\n\n",
353	       getpid(), nthreads, nfds, nsecs);
354
355	for (i = 0; i < EPOLL_NR_OPS; i++)
356		init_stats(&all_stats[i]);
357
358	pthread_mutex_init(&thread_lock, NULL);
359	pthread_cond_init(&thread_parent, NULL);
360	pthread_cond_init(&thread_worker, NULL);
361
362	threads_starting = nthreads;
363
364	gettimeofday(&start, NULL);
365
366	do_threads(worker, cpu);
367
368	pthread_mutex_lock(&thread_lock);
369	while (threads_starting)
370		pthread_cond_wait(&thread_parent, &thread_lock);
371	pthread_cond_broadcast(&thread_worker);
372	pthread_mutex_unlock(&thread_lock);
373
374	sleep(nsecs);
375	toggle_done(0, NULL, NULL);
376	printinfo("main thread: toggling done\n");
377
378	for (i = 0; i < nthreads; i++) {
379		ret = pthread_join(worker[i].thread, NULL);
380		if (ret)
381			err(EXIT_FAILURE, "pthread_join");
382	}
383
384	/* cleanup & report results */
385	pthread_cond_destroy(&thread_parent);
386	pthread_cond_destroy(&thread_worker);
387	pthread_mutex_destroy(&thread_lock);
388
389	for (i = 0; i < nthreads; i++) {
390		unsigned long t[EPOLL_NR_OPS];
391
392		for (j = 0; j < EPOLL_NR_OPS; j++) {
393			t[j] = worker[i].ops[j];
394			update_stats(&all_stats[j], t[j]);
395		}
396
397		if (nfds == 1)
398			printf("[thread %2d] fdmap: %p [ add: %04ld; mod: %04ld; del: %04lds ops ]\n",
399			       worker[i].tid, &worker[i].fdmap[0],
400			       t[OP_EPOLL_ADD], t[OP_EPOLL_MOD], t[OP_EPOLL_DEL]);
401		else
402			printf("[thread %2d] fdmap: %p ... %p [ add: %04ld ops; mod: %04ld ops; del: %04ld ops ]\n",
403			       worker[i].tid, &worker[i].fdmap[0],
404			       &worker[i].fdmap[nfds-1],
405			       t[OP_EPOLL_ADD], t[OP_EPOLL_MOD], t[OP_EPOLL_DEL]);
406	}
407
408	print_summary();
409
410	close(epollfd);
411	return ret;
412errmem:
413	err(EXIT_FAILURE, "calloc");
414}
415#endif // HAVE_EVENTFD