Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef BLK_STAT_H
3#define BLK_STAT_H
4
5#include <linux/kernel.h>
6#include <linux/blkdev.h>
7#include <linux/ktime.h>
8#include <linux/rcupdate.h>
9#include <linux/timer.h>
10
11/**
12 * struct blk_stat_callback - Block statistics callback.
13 *
14 * A &struct blk_stat_callback is associated with a &struct request_queue. While
15 * @timer is active, that queue's request completion latencies are sorted into
16 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the
17 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked.
18 */
19struct blk_stat_callback {
20 /*
21 * @list: RCU list of callbacks for a &struct request_queue.
22 */
23 struct list_head list;
24
25 /**
26 * @timer: Timer for the next callback invocation.
27 */
28 struct timer_list timer;
29
30 /**
31 * @cpu_stat: Per-cpu statistics buckets.
32 */
33 struct blk_rq_stat __percpu *cpu_stat;
34
35 /**
36 * @bucket_fn: Given a request, returns which statistics bucket it
37 * should be accounted under. Return -1 for no bucket for this
38 * request.
39 */
40 int (*bucket_fn)(const struct request *);
41
42 /**
43 * @buckets: Number of statistics buckets.
44 */
45 unsigned int buckets;
46
47 /**
48 * @stat: Array of statistics buckets.
49 */
50 struct blk_rq_stat *stat;
51
52 /**
53 * @fn: Callback function.
54 */
55 void (*timer_fn)(struct blk_stat_callback *);
56
57 /**
58 * @data: Private pointer for the user.
59 */
60 void *data;
61
62 struct rcu_head rcu;
63};
64
65struct blk_queue_stats *blk_alloc_queue_stats(void);
66void blk_free_queue_stats(struct blk_queue_stats *);
67bool blk_stats_alloc_enable(struct request_queue *q);
68
69void blk_stat_add(struct request *rq, u64 now);
70
71/* record time/size info in request but not add a callback */
72void blk_stat_enable_accounting(struct request_queue *q);
73void blk_stat_disable_accounting(struct request_queue *q);
74
75/**
76 * blk_stat_alloc_callback() - Allocate a block statistics callback.
77 * @timer_fn: Timer callback function.
78 * @bucket_fn: Bucket callback function.
79 * @buckets: Number of statistics buckets.
80 * @data: Value for the @data field of the &struct blk_stat_callback.
81 *
82 * See &struct blk_stat_callback for details on the callback functions.
83 *
84 * Return: &struct blk_stat_callback on success or NULL on ENOMEM.
85 */
86struct blk_stat_callback *
87blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
88 int (*bucket_fn)(const struct request *),
89 unsigned int buckets, void *data);
90
91/**
92 * blk_stat_add_callback() - Add a block statistics callback to be run on a
93 * request queue.
94 * @q: The request queue.
95 * @cb: The callback.
96 *
97 * Note that a single &struct blk_stat_callback can only be added to a single
98 * &struct request_queue.
99 */
100void blk_stat_add_callback(struct request_queue *q,
101 struct blk_stat_callback *cb);
102
103/**
104 * blk_stat_remove_callback() - Remove a block statistics callback from a
105 * request queue.
106 * @q: The request queue.
107 * @cb: The callback.
108 *
109 * When this returns, the callback is not running on any CPUs and will not be
110 * called again unless readded.
111 */
112void blk_stat_remove_callback(struct request_queue *q,
113 struct blk_stat_callback *cb);
114
115/**
116 * blk_stat_free_callback() - Free a block statistics callback.
117 * @cb: The callback.
118 *
119 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must
120 * not be associated with a request queue. I.e., if it was previously added with
121 * blk_stat_add_callback(), it must also have been removed since then with
122 * blk_stat_remove_callback().
123 */
124void blk_stat_free_callback(struct blk_stat_callback *cb);
125
126/**
127 * blk_stat_is_active() - Check if a block statistics callback is currently
128 * gathering statistics.
129 * @cb: The callback.
130 */
131static inline bool blk_stat_is_active(struct blk_stat_callback *cb)
132{
133 return timer_pending(&cb->timer);
134}
135
136/**
137 * blk_stat_activate_nsecs() - Gather block statistics during a time window in
138 * nanoseconds.
139 * @cb: The callback.
140 * @nsecs: Number of nanoseconds to gather statistics for.
141 *
142 * The timer callback will be called when the window expires.
143 */
144static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb,
145 u64 nsecs)
146{
147 mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs));
148}
149
150static inline void blk_stat_deactivate(struct blk_stat_callback *cb)
151{
152 del_timer_sync(&cb->timer);
153}
154
155/**
156 * blk_stat_activate_msecs() - Gather block statistics during a time window in
157 * milliseconds.
158 * @cb: The callback.
159 * @msecs: Number of milliseconds to gather statistics for.
160 *
161 * The timer callback will be called when the window expires.
162 */
163static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
164 unsigned int msecs)
165{
166 mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs));
167}
168
169void blk_rq_stat_add(struct blk_rq_stat *, u64);
170void blk_rq_stat_sum(struct blk_rq_stat *, struct blk_rq_stat *);
171void blk_rq_stat_init(struct blk_rq_stat *);
172
173#endif
1#ifndef BLK_STAT_H
2#define BLK_STAT_H
3
4/*
5 * ~0.13s window as a power-of-2 (2^27 nsecs)
6 */
7#define BLK_STAT_NSEC 134217728ULL
8#define BLK_STAT_NSEC_MASK ~(BLK_STAT_NSEC - 1)
9
10/*
11 * Upper 3 bits can be used elsewhere
12 */
13#define BLK_STAT_RES_BITS 3
14#define BLK_STAT_SHIFT (64 - BLK_STAT_RES_BITS)
15#define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SHIFT) - 1)
16#define BLK_STAT_MASK ~BLK_STAT_TIME_MASK
17
18enum {
19 BLK_STAT_READ = 0,
20 BLK_STAT_WRITE,
21};
22
23void blk_stat_add(struct blk_rq_stat *, struct request *);
24void blk_hctx_stat_get(struct blk_mq_hw_ctx *, struct blk_rq_stat *);
25void blk_queue_stat_get(struct request_queue *, struct blk_rq_stat *);
26void blk_stat_clear(struct request_queue *);
27void blk_stat_init(struct blk_rq_stat *);
28bool blk_stat_is_current(struct blk_rq_stat *);
29void blk_stat_set_issue_time(struct blk_issue_stat *);
30bool blk_stat_enable(struct request_queue *);
31
32static inline u64 __blk_stat_time(u64 time)
33{
34 return time & BLK_STAT_TIME_MASK;
35}
36
37static inline u64 blk_stat_time(struct blk_issue_stat *stat)
38{
39 return __blk_stat_time(stat->time);
40}
41
42#endif