1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef BLK_STAT_H
  3#define BLK_STAT_H
  4
  5#include <linux/kernel.h>
  6#include <linux/blkdev.h>
  7#include <linux/ktime.h>
  8#include <linux/rcupdate.h>
  9#include <linux/timer.h>
 10
 11/**
 12 * struct blk_stat_callback - Block statistics callback.
 13 *
 14 * A &struct blk_stat_callback is associated with a &struct request_queue. While
 15 * @timer is active, that queue's request completion latencies are sorted into
 16 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the
 17 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked.
 18 */
 19struct blk_stat_callback {
 20	/*
 21	 * @list: RCU list of callbacks for a &struct request_queue.
 22	 */
 23	struct list_head list;
 24
 25	/**
 26	 * @timer: Timer for the next callback invocation.
 27	 */
 28	struct timer_list timer;
 29
 30	/**
 31	 * @cpu_stat: Per-cpu statistics buckets.
 32	 */
 33	struct blk_rq_stat __percpu *cpu_stat;
 34
 35	/**
 36	 * @bucket_fn: Given a request, returns which statistics bucket it
 37	 * should be accounted under. Return -1 for no bucket for this
 38	 * request.
 39	 */
 40	int (*bucket_fn)(const struct request *);
 41
 42	/**
 43	 * @buckets: Number of statistics buckets.
 44	 */
 45	unsigned int buckets;
 46
 47	/**
 48	 * @stat: Array of statistics buckets.
 49	 */
 50	struct blk_rq_stat *stat;
 51
 52	/**
 53	 * @fn: Callback function.
 54	 */
 55	void (*timer_fn)(struct blk_stat_callback *);
 56
 57	/**
 58	 * @data: Private pointer for the user.
 59	 */
 60	void *data;
 61
 62	struct rcu_head rcu;
 63};
 64
 65struct blk_queue_stats *blk_alloc_queue_stats(void);
 66void blk_free_queue_stats(struct blk_queue_stats *);
 67
 68void blk_stat_add(struct request *rq, u64 now);
 69
 70/* record time/size info in request but not add a callback */
 71void blk_stat_enable_accounting(struct request_queue *q);
 72void blk_stat_disable_accounting(struct request_queue *q);
 73
 74/**
 75 * blk_stat_alloc_callback() - Allocate a block statistics callback.
 76 * @timer_fn: Timer callback function.
 77 * @bucket_fn: Bucket callback function.
 78 * @buckets: Number of statistics buckets.
 79 * @data: Value for the @data field of the &struct blk_stat_callback.
 80 *
 81 * See &struct blk_stat_callback for details on the callback functions.
 82 *
 83 * Return: &struct blk_stat_callback on success or NULL on ENOMEM.
 84 */
 85struct blk_stat_callback *
 86blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
 87			int (*bucket_fn)(const struct request *),
 88			unsigned int buckets, void *data);
 89
 90/**
 91 * blk_stat_add_callback() - Add a block statistics callback to be run on a
 92 * request queue.
 93 * @q: The request queue.
 94 * @cb: The callback.
 95 *
 96 * Note that a single &struct blk_stat_callback can only be added to a single
 97 * &struct request_queue.
 98 */
 99void blk_stat_add_callback(struct request_queue *q,
100			   struct blk_stat_callback *cb);
101
102/**
103 * blk_stat_remove_callback() - Remove a block statistics callback from a
104 * request queue.
105 * @q: The request queue.
106 * @cb: The callback.
107 *
108 * When this returns, the callback is not running on any CPUs and will not be
109 * called again unless readded.
110 */
111void blk_stat_remove_callback(struct request_queue *q,
112			      struct blk_stat_callback *cb);
113
114/**
115 * blk_stat_free_callback() - Free a block statistics callback.
116 * @cb: The callback.
117 *
118 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must
119 * not be associated with a request queue. I.e., if it was previously added with
120 * blk_stat_add_callback(), it must also have been removed since then with
121 * blk_stat_remove_callback().
122 */
123void blk_stat_free_callback(struct blk_stat_callback *cb);
124
125/**
126 * blk_stat_is_active() - Check if a block statistics callback is currently
127 * gathering statistics.
128 * @cb: The callback.
129 */
130static inline bool blk_stat_is_active(struct blk_stat_callback *cb)
131{
132	return timer_pending(&cb->timer);
133}
134
135/**
136 * blk_stat_activate_nsecs() - Gather block statistics during a time window in
137 * nanoseconds.
138 * @cb: The callback.
139 * @nsecs: Number of nanoseconds to gather statistics for.
140 *
141 * The timer callback will be called when the window expires.
142 */
143static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb,
144					   u64 nsecs)
145{
146	mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs));
147}
148
149static inline void blk_stat_deactivate(struct blk_stat_callback *cb)
150{
151	del_timer_sync(&cb->timer);
152}
153
154/**
155 * blk_stat_activate_msecs() - Gather block statistics during a time window in
156 * milliseconds.
157 * @cb: The callback.
158 * @msecs: Number of milliseconds to gather statistics for.
159 *
160 * The timer callback will be called when the window expires.
161 */
162static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
163					   unsigned int msecs)
164{
165	mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs));
166}
167
168void blk_rq_stat_add(struct blk_rq_stat *, u64);
169void blk_rq_stat_sum(struct blk_rq_stat *, struct blk_rq_stat *);
170void blk_rq_stat_init(struct blk_rq_stat *);
171
172#endif