1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef WB_THROTTLE_H
  3#define WB_THROTTLE_H
  4
  5#include <linux/kernel.h>
  6#include <linux/atomic.h>
  7#include <linux/wait.h>
  8#include <linux/timer.h>
  9#include <linux/ktime.h>
 10
 11#include "blk-stat.h"
 
 12
 13enum wbt_flags {
 14	WBT_TRACKED		= 1,	/* write, tracked for throttling */
 15	WBT_READ		= 2,	/* read */
 16	WBT_KSWAPD		= 4,	/* write, from kswapd */
 
 17
 18	WBT_NR_BITS		= 3,	/* number of bits */
 19};
 20
 21enum {
 22	WBT_NUM_RWQ		= 2,
 
 
 
 23};
 24
 25/*
 26 * Enable states. Either off, or on by default (done at init time),
 27 * or on through manual setup in sysfs.
 28 */
 29enum {
 30	WBT_STATE_ON_DEFAULT	= 1,
 31	WBT_STATE_ON_MANUAL	= 2,
 32};
 33
 34static inline void wbt_clear_state(struct blk_issue_stat *stat)
 35{
 36	stat->stat &= ~BLK_STAT_RES_MASK;
 37}
 38
 39static inline enum wbt_flags wbt_stat_to_mask(struct blk_issue_stat *stat)
 40{
 41	return (stat->stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT;
 42}
 43
 44static inline void wbt_track(struct blk_issue_stat *stat, enum wbt_flags wb_acct)
 45{
 46	stat->stat |= ((u64) wb_acct) << BLK_STAT_RES_SHIFT;
 47}
 48
 49static inline bool wbt_is_tracked(struct blk_issue_stat *stat)
 50{
 51	return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED;
 52}
 53
 54static inline bool wbt_is_read(struct blk_issue_stat *stat)
 55{
 56	return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_READ;
 57}
 58
 59struct rq_wait {
 60	wait_queue_head_t wait;
 61	atomic_t inflight;
 62};
 63
 64struct rq_wb {
 65	/*
 66	 * Settings that govern how we throttle
 67	 */
 68	unsigned int wb_background;		/* background writeback */
 69	unsigned int wb_normal;			/* normal writeback */
 70	unsigned int wb_max;			/* max throughput writeback */
 71	int scale_step;
 72	bool scaled_max;
 73
 74	short enable_state;			/* WBT_STATE_* */
 75
 76	/*
 77	 * Number of consecutive periods where we don't have enough
 78	 * information to make a firm scale up/down decision.
 79	 */
 80	unsigned int unknown_cnt;
 81
 82	u64 win_nsec;				/* default window size */
 83	u64 cur_win_nsec;			/* current window size */
 84
 85	struct blk_stat_callback *cb;
 86
 87	s64 sync_issue;
 88	void *sync_cookie;
 89
 90	unsigned int wc;
 91	unsigned int queue_depth;
 92
 93	unsigned long last_issue;		/* last non-throttled issue */
 94	unsigned long last_comp;		/* last non-throttled comp */
 95	unsigned long min_lat_nsec;
 96	struct request_queue *queue;
 97	struct rq_wait rq_wait[WBT_NUM_RWQ];
 
 98};
 99
 
 
 
 
 
100static inline unsigned int wbt_inflight(struct rq_wb *rwb)
101{
102	unsigned int i, ret = 0;
103
104	for (i = 0; i < WBT_NUM_RWQ; i++)
105		ret += atomic_read(&rwb->rq_wait[i].inflight);
106
107	return ret;
108}
109
 
110#ifdef CONFIG_BLK_WBT
111
112void __wbt_done(struct rq_wb *, enum wbt_flags);
113void wbt_done(struct rq_wb *, struct blk_issue_stat *);
114enum wbt_flags wbt_wait(struct rq_wb *, struct bio *, spinlock_t *);
115int wbt_init(struct request_queue *);
116void wbt_exit(struct request_queue *);
117void wbt_update_limits(struct rq_wb *);
118void wbt_requeue(struct rq_wb *, struct blk_issue_stat *);
119void wbt_issue(struct rq_wb *, struct blk_issue_stat *);
120void wbt_disable_default(struct request_queue *);
121void wbt_enable_default(struct request_queue *);
122
123void wbt_set_queue_depth(struct rq_wb *, unsigned int);
124void wbt_set_write_cache(struct rq_wb *, bool);
 
 
125
126u64 wbt_default_latency_nsec(struct request_queue *);
127
128#else
129
130static inline void __wbt_done(struct rq_wb *rwb, enum wbt_flags flags)
131{
132}
133static inline void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
134{
135}
136static inline enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio,
137				      spinlock_t *lock)
138{
139	return 0;
140}
141static inline int wbt_init(struct request_queue *q)
142{
143	return -EINVAL;
144}
145static inline void wbt_exit(struct request_queue *q)
146{
147}
148static inline void wbt_update_limits(struct rq_wb *rwb)
149{
150}
151static inline void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
152{
153}
154static inline void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
155{
156}
157static inline void wbt_disable_default(struct request_queue *q)
158{
159}
160static inline void wbt_enable_default(struct request_queue *q)
161{
162}
163static inline void wbt_set_queue_depth(struct rq_wb *rwb, unsigned int depth)
 
 
 
164{
 
165}
166static inline void wbt_set_write_cache(struct rq_wb *rwb, bool wc)
167{
168}
169static inline u64 wbt_default_latency_nsec(struct request_queue *q)
170{
171	return 0;
172}
173
174#endif /* CONFIG_BLK_WBT */
175
176#endif

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef WB_THROTTLE_H
  3#define WB_THROTTLE_H
  4
  5#include <linux/kernel.h>
  6#include <linux/atomic.h>
  7#include <linux/wait.h>
  8#include <linux/timer.h>
  9#include <linux/ktime.h>
 10
 11#include "blk-stat.h"
 12#include "blk-rq-qos.h"
 13
 14enum wbt_flags {
 15	WBT_TRACKED		= 1,	/* write, tracked for throttling */
 16	WBT_READ		= 2,	/* read */
 17	WBT_KSWAPD		= 4,	/* write, from kswapd */
 18	WBT_DISCARD		= 8,	/* discard */
 19
 20	WBT_NR_BITS		= 4,	/* number of bits */
 21};
 22
 23enum {
 24	WBT_RWQ_BG		= 0,
 25	WBT_RWQ_KSWAPD,
 26	WBT_RWQ_DISCARD,
 27	WBT_NUM_RWQ,
 28};
 29
 30/*
 31 * Enable states. Either off, or on by default (done at init time),
 32 * or on through manual setup in sysfs.
 33 */
 34enum {
 35	WBT_STATE_ON_DEFAULT	= 1,
 36	WBT_STATE_ON_MANUAL	= 2,
 37	WBT_STATE_OFF_DEFAULT
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 38};
 39
 40struct rq_wb {
 41	/*
 42	 * Settings that govern how we throttle
 43	 */
 44	unsigned int wb_background;		/* background writeback */
 45	unsigned int wb_normal;			/* normal writeback */
 
 
 
 46
 47	short enable_state;			/* WBT_STATE_* */
 48
 49	/*
 50	 * Number of consecutive periods where we don't have enough
 51	 * information to make a firm scale up/down decision.
 52	 */
 53	unsigned int unknown_cnt;
 54
 55	u64 win_nsec;				/* default window size */
 56	u64 cur_win_nsec;			/* current window size */
 57
 58	struct blk_stat_callback *cb;
 59
 60	u64 sync_issue;
 61	void *sync_cookie;
 62
 63	unsigned int wc;
 
 64
 65	unsigned long last_issue;		/* last non-throttled issue */
 66	unsigned long last_comp;		/* last non-throttled comp */
 67	unsigned long min_lat_nsec;
 68	struct rq_qos rqos;
 69	struct rq_wait rq_wait[WBT_NUM_RWQ];
 70	struct rq_depth rq_depth;
 71};
 72
 73static inline struct rq_wb *RQWB(struct rq_qos *rqos)
 74{
 75	return container_of(rqos, struct rq_wb, rqos);
 76}
 77
 78static inline unsigned int wbt_inflight(struct rq_wb *rwb)
 79{
 80	unsigned int i, ret = 0;
 81
 82	for (i = 0; i < WBT_NUM_RWQ; i++)
 83		ret += atomic_read(&rwb->rq_wait[i].inflight);
 84
 85	return ret;
 86}
 87
 88
 89#ifdef CONFIG_BLK_WBT
 90
 
 
 
 91int wbt_init(struct request_queue *);
 
 
 
 
 92void wbt_disable_default(struct request_queue *);
 93void wbt_enable_default(struct request_queue *);
 94
 95u64 wbt_get_min_lat(struct request_queue *q);
 96void wbt_set_min_lat(struct request_queue *q, u64 val);
 97
 98void wbt_set_write_cache(struct request_queue *, bool);
 99
100u64 wbt_default_latency_nsec(struct request_queue *);
101
102#else
103
104static inline void wbt_track(struct request *rq, enum wbt_flags flags)
 
 
 
105{
106}
 
 
 
 
 
107static inline int wbt_init(struct request_queue *q)
108{
109	return -EINVAL;
110}
 
 
 
 
 
 
 
 
 
 
 
 
111static inline void wbt_disable_default(struct request_queue *q)
112{
113}
114static inline void wbt_enable_default(struct request_queue *q)
115{
116}
117static inline void wbt_set_write_cache(struct request_queue *q, bool wc)
118{
119}
120static inline u64 wbt_get_min_lat(struct request_queue *q)
121{
122	return 0;
123}
124static inline void wbt_set_min_lat(struct request_queue *q, u64 val)
125{
126}
127static inline u64 wbt_default_latency_nsec(struct request_queue *q)
128{
129	return 0;
130}
131
132#endif /* CONFIG_BLK_WBT */
133
134#endif