1#ifndef BLK_THROTTLE_H
  2#define BLK_THROTTLE_H
  3
  4#include "blk-cgroup-rwstat.h"
  5
  6/*
  7 * To implement hierarchical throttling, throtl_grps form a tree and bios
  8 * are dispatched upwards level by level until they reach the top and get
  9 * issued.  When dispatching bios from the children and local group at each
 10 * level, if the bios are dispatched into a single bio_list, there's a risk
 11 * of a local or child group which can queue many bios at once filling up
 12 * the list starving others.
 13 *
 14 * To avoid such starvation, dispatched bios are queued separately
 15 * according to where they came from.  When they are again dispatched to
 16 * the parent, they're popped in round-robin order so that no single source
 17 * hogs the dispatch window.
 18 *
 19 * throtl_qnode is used to keep the queued bios separated by their sources.
 20 * Bios are queued to throtl_qnode which in turn is queued to
 21 * throtl_service_queue and then dispatched in round-robin order.
 22 *
 23 * It's also used to track the reference counts on blkg's.  A qnode always
 24 * belongs to a throtl_grp and gets queued on itself or the parent, so
 25 * incrementing the reference of the associated throtl_grp when a qnode is
 26 * queued and decrementing when dequeued is enough to keep the whole blkg
 27 * tree pinned while bios are in flight.
 28 */
 29struct throtl_qnode {
 30	struct list_head	node;		/* service_queue->queued[] */
 31	struct bio_list		bios;		/* queued bios */
 32	struct throtl_grp	*tg;		/* tg this qnode belongs to */
 33};
 34
 35struct throtl_service_queue {
 36	struct throtl_service_queue *parent_sq;	/* the parent service_queue */
 37
 38	/*
 39	 * Bios queued directly to this service_queue or dispatched from
 40	 * children throtl_grp's.
 41	 */
 42	struct list_head	queued[2];	/* throtl_qnode [READ/WRITE] */
 43	unsigned int		nr_queued[2];	/* number of queued bios */
 44
 45	/*
 46	 * RB tree of active children throtl_grp's, which are sorted by
 47	 * their ->disptime.
 48	 */
 49	struct rb_root_cached	pending_tree;	/* RB tree of active tgs */
 50	unsigned int		nr_pending;	/* # queued in the tree */
 51	unsigned long		first_pending_disptime;	/* disptime of the first tg */
 52	struct timer_list	pending_timer;	/* fires on first_pending_disptime */
 53};
 54
 55enum tg_state_flags {
 56	THROTL_TG_PENDING	= 1 << 0,	/* on parent's pending tree */
 57	THROTL_TG_WAS_EMPTY	= 1 << 1,	/* bio_lists[] became non-empty */
 58	THROTL_TG_CANCELING	= 1 << 2,	/* starts to cancel bio */
 59};
 60
 61struct throtl_grp {
 62	/* must be the first member */
 63	struct blkg_policy_data pd;
 64
 65	/* active throtl group service_queue member */
 66	struct rb_node rb_node;
 67
 68	/* throtl_data this group belongs to */
 69	struct throtl_data *td;
 70
 71	/* this group's service queue */
 72	struct throtl_service_queue service_queue;
 73
 74	/*
 75	 * qnode_on_self is used when bios are directly queued to this
 76	 * throtl_grp so that local bios compete fairly with bios
 77	 * dispatched from children.  qnode_on_parent is used when bios are
 78	 * dispatched from this throtl_grp into its parent and will compete
 79	 * with the sibling qnode_on_parents and the parent's
 80	 * qnode_on_self.
 81	 */
 82	struct throtl_qnode qnode_on_self[2];
 83	struct throtl_qnode qnode_on_parent[2];
 84
 85	/*
 86	 * Dispatch time in jiffies. This is the estimated time when group
 87	 * will unthrottle and is ready to dispatch more bio. It is used as
 88	 * key to sort active groups in service tree.
 89	 */
 90	unsigned long disptime;
 91
 92	unsigned int flags;
 93
 94	/* are there any throtl rules between this group and td? */
 95	bool has_rules_bps[2];
 96	bool has_rules_iops[2];
 97
 98	/* bytes per second rate limits */
 99	uint64_t bps[2];
100
101	/* IOPS limits */
102	unsigned int iops[2];
103
104	/* Number of bytes dispatched in current slice */
105	uint64_t bytes_disp[2];
106	/* Number of bio's dispatched in current slice */
107	unsigned int io_disp[2];
108
109	uint64_t last_bytes_disp[2];
110	unsigned int last_io_disp[2];
111
112	/*
113	 * The following two fields are updated when new configuration is
114	 * submitted while some bios are still throttled, they record how many
115	 * bytes/ios are waited already in previous configuration, and they will
116	 * be used to calculate wait time under new configuration.
117	 */
118	long long carryover_bytes[2];
119	int carryover_ios[2];
120
121	unsigned long last_check_time;
122
123	/* When did we start a new slice */
124	unsigned long slice_start[2];
125	unsigned long slice_end[2];
126
127	struct blkg_rwstat stat_bytes;
128	struct blkg_rwstat stat_ios;
129};
130
131extern struct blkcg_policy blkcg_policy_throtl;
132
133static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
134{
135	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
136}
137
138static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
139{
140	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
141}
142
143/*
144 * Internal throttling interface
145 */
146#ifndef CONFIG_BLK_DEV_THROTTLING
147static inline void blk_throtl_exit(struct gendisk *disk) { }
148static inline bool blk_throtl_bio(struct bio *bio) { return false; }
149static inline void blk_throtl_cancel_bios(struct gendisk *disk) { }
150#else /* CONFIG_BLK_DEV_THROTTLING */
151void blk_throtl_exit(struct gendisk *disk);
152bool __blk_throtl_bio(struct bio *bio);
153void blk_throtl_cancel_bios(struct gendisk *disk);
154
155static inline bool blk_throtl_activated(struct request_queue *q)
156{
157	return q->td != NULL;
158}
159
160static inline bool blk_should_throtl(struct bio *bio)
161{
162	struct throtl_grp *tg;
163	int rw = bio_data_dir(bio);
164
165	/*
166	 * This is called under bio_queue_enter(), and it's synchronized with
167	 * the activation of blk-throtl, which is protected by
168	 * blk_mq_freeze_queue().
169	 */
170	if (!blk_throtl_activated(bio->bi_bdev->bd_queue))
171		return false;
172
173	tg = blkg_to_tg(bio->bi_blkg);
174	if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
175		if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
176			bio_set_flag(bio, BIO_CGROUP_ACCT);
177			blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
178					bio->bi_iter.bi_size);
179		}
180		blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
181	}
182
183	/* iops limit is always counted */
184	if (tg->has_rules_iops[rw])
185		return true;
186
187	if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED))
188		return true;
189
190	return false;
191}
192
193static inline bool blk_throtl_bio(struct bio *bio)
194{
195
196	if (!blk_should_throtl(bio))
197		return false;
198
199	return __blk_throtl_bio(bio);
200}
201#endif /* CONFIG_BLK_DEV_THROTTLING */
202
203#endif