1// SPDX-License-Identifier: GPL-2.0
  2#include "bcachefs.h"
  3#include "clock.h"
  4
  5#include <linux/freezer.h>
  6#include <linux/kthread.h>
  7#include <linux/preempt.h>
  8
  9static inline long io_timer_cmp(io_timer_heap *h,
 10				struct io_timer *l,
 11				struct io_timer *r)
 12{
 13	return l->expire - r->expire;
 14}
 15
 16void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
 17{
 18	size_t i;
 19
 20	spin_lock(&clock->timer_lock);
 21
 22	if (time_after_eq((unsigned long) atomic64_read(&clock->now),
 23			  timer->expire)) {
 24		spin_unlock(&clock->timer_lock);
 25		timer->fn(timer);
 26		return;
 27	}
 28
 29	for (i = 0; i < clock->timers.used; i++)
 30		if (clock->timers.data[i] == timer)
 31			goto out;
 32
 33	BUG_ON(!heap_add(&clock->timers, timer, io_timer_cmp, NULL));
 34out:
 35	spin_unlock(&clock->timer_lock);
 36}
 37
 38void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer)
 39{
 40	size_t i;
 41
 42	spin_lock(&clock->timer_lock);
 43
 44	for (i = 0; i < clock->timers.used; i++)
 45		if (clock->timers.data[i] == timer) {
 46			heap_del(&clock->timers, i, io_timer_cmp, NULL);
 47			break;
 48		}
 49
 50	spin_unlock(&clock->timer_lock);
 51}
 52
 53struct io_clock_wait {
 54	struct io_timer		io_timer;
 55	struct timer_list	cpu_timer;
 56	struct task_struct	*task;
 57	int			expired;
 58};
 59
 60static void io_clock_wait_fn(struct io_timer *timer)
 61{
 62	struct io_clock_wait *wait = container_of(timer,
 63				struct io_clock_wait, io_timer);
 64
 65	wait->expired = 1;
 66	wake_up_process(wait->task);
 67}
 68
 69static void io_clock_cpu_timeout(struct timer_list *timer)
 70{
 71	struct io_clock_wait *wait = container_of(timer,
 72				struct io_clock_wait, cpu_timer);
 73
 74	wait->expired = 1;
 75	wake_up_process(wait->task);
 76}
 77
 78void bch2_io_clock_schedule_timeout(struct io_clock *clock, unsigned long until)
 79{
 80	struct io_clock_wait wait;
 81
 82	/* XXX: calculate sleep time rigorously */
 83	wait.io_timer.expire	= until;
 84	wait.io_timer.fn	= io_clock_wait_fn;
 85	wait.task		= current;
 86	wait.expired		= 0;
 87	bch2_io_timer_add(clock, &wait.io_timer);
 88
 89	schedule();
 90
 91	bch2_io_timer_del(clock, &wait.io_timer);
 92}
 93
 94void bch2_kthread_io_clock_wait(struct io_clock *clock,
 95				unsigned long io_until,
 96				unsigned long cpu_timeout)
 97{
 98	bool kthread = (current->flags & PF_KTHREAD) != 0;
 99	struct io_clock_wait wait;
100
101	wait.io_timer.expire	= io_until;
102	wait.io_timer.fn	= io_clock_wait_fn;
103	wait.task		= current;
104	wait.expired		= 0;
105	bch2_io_timer_add(clock, &wait.io_timer);
106
107	timer_setup_on_stack(&wait.cpu_timer, io_clock_cpu_timeout, 0);
108
109	if (cpu_timeout != MAX_SCHEDULE_TIMEOUT)
110		mod_timer(&wait.cpu_timer, cpu_timeout + jiffies);
111
112	do {
113		set_current_state(TASK_INTERRUPTIBLE);
114		if (kthread && kthread_should_stop())
115			break;
116
117		if (wait.expired)
118			break;
119
120		schedule();
121		try_to_freeze();
122	} while (0);
123
124	__set_current_state(TASK_RUNNING);
125	del_timer_sync(&wait.cpu_timer);
126	destroy_timer_on_stack(&wait.cpu_timer);
127	bch2_io_timer_del(clock, &wait.io_timer);
128}
129
130static struct io_timer *get_expired_timer(struct io_clock *clock,
131					  unsigned long now)
132{
133	struct io_timer *ret = NULL;
134
135	spin_lock(&clock->timer_lock);
136
137	if (clock->timers.used &&
138	    time_after_eq(now, clock->timers.data[0]->expire))
139		heap_pop(&clock->timers, ret, io_timer_cmp, NULL);
140
141	spin_unlock(&clock->timer_lock);
142
143	return ret;
144}
145
146void __bch2_increment_clock(struct io_clock *clock, unsigned sectors)
147{
148	struct io_timer *timer;
149	unsigned long now = atomic64_add_return(sectors, &clock->now);
150
151	while ((timer = get_expired_timer(clock, now)))
152		timer->fn(timer);
153}
154
155void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock)
156{
157	unsigned long now;
158	unsigned i;
159
160	out->atomic++;
161	spin_lock(&clock->timer_lock);
162	now = atomic64_read(&clock->now);
163
164	for (i = 0; i < clock->timers.used; i++)
165		prt_printf(out, "%ps:\t%li\n",
166		       clock->timers.data[i]->fn,
167		       clock->timers.data[i]->expire - now);
168	spin_unlock(&clock->timer_lock);
169	--out->atomic;
170}
171
172void bch2_io_clock_exit(struct io_clock *clock)
173{
174	free_heap(&clock->timers);
175	free_percpu(clock->pcpu_buf);
176}
177
178int bch2_io_clock_init(struct io_clock *clock)
179{
180	atomic64_set(&clock->now, 0);
181	spin_lock_init(&clock->timer_lock);
182
183	clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
184
185	clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
186	if (!clock->pcpu_buf)
187		return -BCH_ERR_ENOMEM_io_clock_init;
188
189	if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL))
190		return -BCH_ERR_ENOMEM_io_clock_init;
191
192	return 0;
193}