1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Block stat tracking code
  4 *
  5 * Copyright (C) 2016 Jens Axboe
  6 */
  7#include <linux/kernel.h>
  8#include <linux/rculist.h>
  9#include <linux/blk-mq.h>
 10
 11#include "blk-stat.h"
 12#include "blk-mq.h"
 13#include "blk.h"
 14
 15struct blk_queue_stats {
 16	struct list_head callbacks;
 17	spinlock_t lock;
 18	bool enable_accounting;
 19};
 20
 21void blk_rq_stat_init(struct blk_rq_stat *stat)
 22{
 23	stat->min = -1ULL;
 24	stat->max = stat->nr_samples = stat->mean = 0;
 25	stat->batch = 0;
 26}
 27
 28/* src is a per-cpu stat, mean isn't initialized */
 29void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
 30{
 31	if (!src->nr_samples)
 32		return;
 33
 34	dst->min = min(dst->min, src->min);
 35	dst->max = max(dst->max, src->max);
 36
 37	dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
 38				dst->nr_samples + src->nr_samples);
 39
 40	dst->nr_samples += src->nr_samples;
 41}
 42
 43void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
 44{
 45	stat->min = min(stat->min, value);
 46	stat->max = max(stat->max, value);
 47	stat->batch += value;
 48	stat->nr_samples++;
 49}
 50
 51void blk_stat_add(struct request *rq, u64 now)
 52{
 53	struct request_queue *q = rq->q;
 54	struct blk_stat_callback *cb;
 55	struct blk_rq_stat *stat;
 56	int bucket;
 57	u64 value;
 58
 59	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
 60
 61	blk_throtl_stat_add(rq, value);
 62
 63	rcu_read_lock();
 64	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
 65		if (!blk_stat_is_active(cb))
 66			continue;
 67
 68		bucket = cb->bucket_fn(rq);
 69		if (bucket < 0)
 70			continue;
 71
 72		stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
 73		blk_rq_stat_add(stat, value);
 74		put_cpu_ptr(cb->cpu_stat);
 75	}
 76	rcu_read_unlock();
 77}
 78
 79static void blk_stat_timer_fn(struct timer_list *t)
 80{
 81	struct blk_stat_callback *cb = from_timer(cb, t, timer);
 82	unsigned int bucket;
 83	int cpu;
 84
 85	for (bucket = 0; bucket < cb->buckets; bucket++)
 86		blk_rq_stat_init(&cb->stat[bucket]);
 87
 88	for_each_online_cpu(cpu) {
 89		struct blk_rq_stat *cpu_stat;
 90
 91		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
 92		for (bucket = 0; bucket < cb->buckets; bucket++) {
 93			blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
 94			blk_rq_stat_init(&cpu_stat[bucket]);
 95		}
 96	}
 97
 98	cb->timer_fn(cb);
 99}
100
101struct blk_stat_callback *
102blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
103			int (*bucket_fn)(const struct request *),
104			unsigned int buckets, void *data)
105{
106	struct blk_stat_callback *cb;
107
108	cb = kmalloc(sizeof(*cb), GFP_KERNEL);
109	if (!cb)
110		return NULL;
111
112	cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
113				 GFP_KERNEL);
114	if (!cb->stat) {
115		kfree(cb);
116		return NULL;
117	}
118	cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
119				      __alignof__(struct blk_rq_stat));
120	if (!cb->cpu_stat) {
121		kfree(cb->stat);
122		kfree(cb);
123		return NULL;
124	}
125
126	cb->timer_fn = timer_fn;
127	cb->bucket_fn = bucket_fn;
128	cb->data = data;
129	cb->buckets = buckets;
130	timer_setup(&cb->timer, blk_stat_timer_fn, 0);
131
132	return cb;
133}
134
135void blk_stat_add_callback(struct request_queue *q,
136			   struct blk_stat_callback *cb)
137{
138	unsigned int bucket;
139	int cpu;
140
141	for_each_possible_cpu(cpu) {
142		struct blk_rq_stat *cpu_stat;
143
144		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
145		for (bucket = 0; bucket < cb->buckets; bucket++)
146			blk_rq_stat_init(&cpu_stat[bucket]);
147	}
148
149	spin_lock(&q->stats->lock);
150	list_add_tail_rcu(&cb->list, &q->stats->callbacks);
151	blk_queue_flag_set(QUEUE_FLAG_STATS, q);
152	spin_unlock(&q->stats->lock);
153}
154
155void blk_stat_remove_callback(struct request_queue *q,
156			      struct blk_stat_callback *cb)
157{
158	spin_lock(&q->stats->lock);
159	list_del_rcu(&cb->list);
160	if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
161		blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
162	spin_unlock(&q->stats->lock);
163
164	del_timer_sync(&cb->timer);
165}
166
167static void blk_stat_free_callback_rcu(struct rcu_head *head)
168{
169	struct blk_stat_callback *cb;
170
171	cb = container_of(head, struct blk_stat_callback, rcu);
172	free_percpu(cb->cpu_stat);
173	kfree(cb->stat);
174	kfree(cb);
175}
176
177void blk_stat_free_callback(struct blk_stat_callback *cb)
178{
179	if (cb)
180		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
181}
182
183void blk_stat_enable_accounting(struct request_queue *q)
184{
185	spin_lock(&q->stats->lock);
186	q->stats->enable_accounting = true;
187	blk_queue_flag_set(QUEUE_FLAG_STATS, q);
188	spin_unlock(&q->stats->lock);
189}
190EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
191
192struct blk_queue_stats *blk_alloc_queue_stats(void)
193{
194	struct blk_queue_stats *stats;
195
196	stats = kmalloc(sizeof(*stats), GFP_KERNEL);
197	if (!stats)
198		return NULL;
199
200	INIT_LIST_HEAD(&stats->callbacks);
201	spin_lock_init(&stats->lock);
202	stats->enable_accounting = false;
203
204	return stats;
205}
206
207void blk_free_queue_stats(struct blk_queue_stats *stats)
208{
209	if (!stats)
210		return;
211
212	WARN_ON(!list_empty(&stats->callbacks));
213
214	kfree(stats);
215}