Loading...
1/*
2 * Common Block IO controller cgroup interface
3 *
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6 *
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
9 *
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
12 *
13 * For policy-specific per-blkcg data:
14 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
15 * Arianna Avanzini <avanzini.arianna@gmail.com>
16 */
17#include <linux/ioprio.h>
18#include <linux/kdev_t.h>
19#include <linux/module.h>
20#include <linux/sched/signal.h>
21#include <linux/err.h>
22#include <linux/blkdev.h>
23#include <linux/backing-dev.h>
24#include <linux/slab.h>
25#include <linux/genhd.h>
26#include <linux/delay.h>
27#include <linux/atomic.h>
28#include <linux/ctype.h>
29#include <linux/blk-cgroup.h>
30#include "blk.h"
31
32#define MAX_KEY_LEN 100
33
34/*
35 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
36 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
37 * policy [un]register operations including cgroup file additions /
38 * removals. Putting cgroup file registration outside blkcg_pol_mutex
39 * allows grabbing it from cgroup callbacks.
40 */
41static DEFINE_MUTEX(blkcg_pol_register_mutex);
42static DEFINE_MUTEX(blkcg_pol_mutex);
43
44struct blkcg blkcg_root;
45EXPORT_SYMBOL_GPL(blkcg_root);
46
47struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
48
49static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
50
51static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
52
53static bool blkcg_policy_enabled(struct request_queue *q,
54 const struct blkcg_policy *pol)
55{
56 return pol && test_bit(pol->plid, q->blkcg_pols);
57}
58
59/**
60 * blkg_free - free a blkg
61 * @blkg: blkg to free
62 *
63 * Free @blkg which may be partially allocated.
64 */
65static void blkg_free(struct blkcg_gq *blkg)
66{
67 int i;
68
69 if (!blkg)
70 return;
71
72 for (i = 0; i < BLKCG_MAX_POLS; i++)
73 if (blkg->pd[i])
74 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
75
76 if (blkg->blkcg != &blkcg_root)
77 blk_exit_rl(blkg->q, &blkg->rl);
78
79 blkg_rwstat_exit(&blkg->stat_ios);
80 blkg_rwstat_exit(&blkg->stat_bytes);
81 kfree(blkg);
82}
83
84/**
85 * blkg_alloc - allocate a blkg
86 * @blkcg: block cgroup the new blkg is associated with
87 * @q: request_queue the new blkg is associated with
88 * @gfp_mask: allocation mask to use
89 *
90 * Allocate a new blkg assocating @blkcg and @q.
91 */
92static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
93 gfp_t gfp_mask)
94{
95 struct blkcg_gq *blkg;
96 int i;
97
98 /* alloc and init base part */
99 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
100 if (!blkg)
101 return NULL;
102
103 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
104 blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
105 goto err_free;
106
107 blkg->q = q;
108 INIT_LIST_HEAD(&blkg->q_node);
109 blkg->blkcg = blkcg;
110 atomic_set(&blkg->refcnt, 1);
111
112 /* root blkg uses @q->root_rl, init rl only for !root blkgs */
113 if (blkcg != &blkcg_root) {
114 if (blk_init_rl(&blkg->rl, q, gfp_mask))
115 goto err_free;
116 blkg->rl.blkg = blkg;
117 }
118
119 for (i = 0; i < BLKCG_MAX_POLS; i++) {
120 struct blkcg_policy *pol = blkcg_policy[i];
121 struct blkg_policy_data *pd;
122
123 if (!blkcg_policy_enabled(q, pol))
124 continue;
125
126 /* alloc per-policy data and attach it to blkg */
127 pd = pol->pd_alloc_fn(gfp_mask, q->node);
128 if (!pd)
129 goto err_free;
130
131 blkg->pd[i] = pd;
132 pd->blkg = blkg;
133 pd->plid = i;
134 }
135
136 return blkg;
137
138err_free:
139 blkg_free(blkg);
140 return NULL;
141}
142
143struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
144 struct request_queue *q, bool update_hint)
145{
146 struct blkcg_gq *blkg;
147
148 /*
149 * Hint didn't match. Look up from the radix tree. Note that the
150 * hint can only be updated under queue_lock as otherwise @blkg
151 * could have already been removed from blkg_tree. The caller is
152 * responsible for grabbing queue_lock if @update_hint.
153 */
154 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
155 if (blkg && blkg->q == q) {
156 if (update_hint) {
157 lockdep_assert_held(q->queue_lock);
158 rcu_assign_pointer(blkcg->blkg_hint, blkg);
159 }
160 return blkg;
161 }
162
163 return NULL;
164}
165EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
166
167/*
168 * If @new_blkg is %NULL, this function tries to allocate a new one as
169 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
170 */
171static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
172 struct request_queue *q,
173 struct blkcg_gq *new_blkg)
174{
175 struct blkcg_gq *blkg;
176 struct bdi_writeback_congested *wb_congested;
177 int i, ret;
178
179 WARN_ON_ONCE(!rcu_read_lock_held());
180 lockdep_assert_held(q->queue_lock);
181
182 /* blkg holds a reference to blkcg */
183 if (!css_tryget_online(&blkcg->css)) {
184 ret = -ENODEV;
185 goto err_free_blkg;
186 }
187
188 wb_congested = wb_congested_get_create(q->backing_dev_info,
189 blkcg->css.id,
190 GFP_NOWAIT | __GFP_NOWARN);
191 if (!wb_congested) {
192 ret = -ENOMEM;
193 goto err_put_css;
194 }
195
196 /* allocate */
197 if (!new_blkg) {
198 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
199 if (unlikely(!new_blkg)) {
200 ret = -ENOMEM;
201 goto err_put_congested;
202 }
203 }
204 blkg = new_blkg;
205 blkg->wb_congested = wb_congested;
206
207 /* link parent */
208 if (blkcg_parent(blkcg)) {
209 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
210 if (WARN_ON_ONCE(!blkg->parent)) {
211 ret = -ENODEV;
212 goto err_put_congested;
213 }
214 blkg_get(blkg->parent);
215 }
216
217 /* invoke per-policy init */
218 for (i = 0; i < BLKCG_MAX_POLS; i++) {
219 struct blkcg_policy *pol = blkcg_policy[i];
220
221 if (blkg->pd[i] && pol->pd_init_fn)
222 pol->pd_init_fn(blkg->pd[i]);
223 }
224
225 /* insert */
226 spin_lock(&blkcg->lock);
227 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
228 if (likely(!ret)) {
229 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
230 list_add(&blkg->q_node, &q->blkg_list);
231
232 for (i = 0; i < BLKCG_MAX_POLS; i++) {
233 struct blkcg_policy *pol = blkcg_policy[i];
234
235 if (blkg->pd[i] && pol->pd_online_fn)
236 pol->pd_online_fn(blkg->pd[i]);
237 }
238 }
239 blkg->online = true;
240 spin_unlock(&blkcg->lock);
241
242 if (!ret)
243 return blkg;
244
245 /* @blkg failed fully initialized, use the usual release path */
246 blkg_put(blkg);
247 return ERR_PTR(ret);
248
249err_put_congested:
250 wb_congested_put(wb_congested);
251err_put_css:
252 css_put(&blkcg->css);
253err_free_blkg:
254 blkg_free(new_blkg);
255 return ERR_PTR(ret);
256}
257
258/**
259 * blkg_lookup_create - lookup blkg, try to create one if not there
260 * @blkcg: blkcg of interest
261 * @q: request_queue of interest
262 *
263 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
264 * create one. blkg creation is performed recursively from blkcg_root such
265 * that all non-root blkg's have access to the parent blkg. This function
266 * should be called under RCU read lock and @q->queue_lock.
267 *
268 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
269 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not
270 * dead and bypassing, returns ERR_PTR(-EBUSY).
271 */
272struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
273 struct request_queue *q)
274{
275 struct blkcg_gq *blkg;
276
277 WARN_ON_ONCE(!rcu_read_lock_held());
278 lockdep_assert_held(q->queue_lock);
279
280 /*
281 * This could be the first entry point of blkcg implementation and
282 * we shouldn't allow anything to go through for a bypassing queue.
283 */
284 if (unlikely(blk_queue_bypass(q)))
285 return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
286
287 blkg = __blkg_lookup(blkcg, q, true);
288 if (blkg)
289 return blkg;
290
291 /*
292 * Create blkgs walking down from blkcg_root to @blkcg, so that all
293 * non-root blkgs have access to their parents.
294 */
295 while (true) {
296 struct blkcg *pos = blkcg;
297 struct blkcg *parent = blkcg_parent(blkcg);
298
299 while (parent && !__blkg_lookup(parent, q, false)) {
300 pos = parent;
301 parent = blkcg_parent(parent);
302 }
303
304 blkg = blkg_create(pos, q, NULL);
305 if (pos == blkcg || IS_ERR(blkg))
306 return blkg;
307 }
308}
309
310static void blkg_pd_offline(struct blkcg_gq *blkg)
311{
312 int i;
313
314 lockdep_assert_held(blkg->q->queue_lock);
315 lockdep_assert_held(&blkg->blkcg->lock);
316
317 for (i = 0; i < BLKCG_MAX_POLS; i++) {
318 struct blkcg_policy *pol = blkcg_policy[i];
319
320 if (blkg->pd[i] && !blkg->pd[i]->offline &&
321 pol->pd_offline_fn) {
322 pol->pd_offline_fn(blkg->pd[i]);
323 blkg->pd[i]->offline = true;
324 }
325 }
326}
327
328static void blkg_destroy(struct blkcg_gq *blkg)
329{
330 struct blkcg *blkcg = blkg->blkcg;
331 struct blkcg_gq *parent = blkg->parent;
332
333 lockdep_assert_held(blkg->q->queue_lock);
334 lockdep_assert_held(&blkcg->lock);
335
336 /* Something wrong if we are trying to remove same group twice */
337 WARN_ON_ONCE(list_empty(&blkg->q_node));
338 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
339
340 if (parent) {
341 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
342 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
343 }
344
345 blkg->online = false;
346
347 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
348 list_del_init(&blkg->q_node);
349 hlist_del_init_rcu(&blkg->blkcg_node);
350
351 /*
352 * Both setting lookup hint to and clearing it from @blkg are done
353 * under queue_lock. If it's not pointing to @blkg now, it never
354 * will. Hint assignment itself can race safely.
355 */
356 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
357 rcu_assign_pointer(blkcg->blkg_hint, NULL);
358
359 /*
360 * Put the reference taken at the time of creation so that when all
361 * queues are gone, group can be destroyed.
362 */
363 blkg_put(blkg);
364}
365
366/**
367 * blkg_destroy_all - destroy all blkgs associated with a request_queue
368 * @q: request_queue of interest
369 *
370 * Destroy all blkgs associated with @q.
371 */
372static void blkg_destroy_all(struct request_queue *q)
373{
374 struct blkcg_gq *blkg, *n;
375
376 lockdep_assert_held(q->queue_lock);
377
378 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
379 struct blkcg *blkcg = blkg->blkcg;
380
381 spin_lock(&blkcg->lock);
382 blkg_pd_offline(blkg);
383 blkg_destroy(blkg);
384 spin_unlock(&blkcg->lock);
385 }
386
387 q->root_blkg = NULL;
388 q->root_rl.blkg = NULL;
389}
390
391/*
392 * A group is RCU protected, but having an rcu lock does not mean that one
393 * can access all the fields of blkg and assume these are valid. For
394 * example, don't try to follow throtl_data and request queue links.
395 *
396 * Having a reference to blkg under an rcu allows accesses to only values
397 * local to groups like group stats and group rate limits.
398 */
399void __blkg_release_rcu(struct rcu_head *rcu_head)
400{
401 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
402
403 /* release the blkcg and parent blkg refs this blkg has been holding */
404 css_put(&blkg->blkcg->css);
405 if (blkg->parent)
406 blkg_put(blkg->parent);
407
408 wb_congested_put(blkg->wb_congested);
409
410 blkg_free(blkg);
411}
412EXPORT_SYMBOL_GPL(__blkg_release_rcu);
413
414/*
415 * The next function used by blk_queue_for_each_rl(). It's a bit tricky
416 * because the root blkg uses @q->root_rl instead of its own rl.
417 */
418struct request_list *__blk_queue_next_rl(struct request_list *rl,
419 struct request_queue *q)
420{
421 struct list_head *ent;
422 struct blkcg_gq *blkg;
423
424 /*
425 * Determine the current blkg list_head. The first entry is
426 * root_rl which is off @q->blkg_list and mapped to the head.
427 */
428 if (rl == &q->root_rl) {
429 ent = &q->blkg_list;
430 /* There are no more block groups, hence no request lists */
431 if (list_empty(ent))
432 return NULL;
433 } else {
434 blkg = container_of(rl, struct blkcg_gq, rl);
435 ent = &blkg->q_node;
436 }
437
438 /* walk to the next list_head, skip root blkcg */
439 ent = ent->next;
440 if (ent == &q->root_blkg->q_node)
441 ent = ent->next;
442 if (ent == &q->blkg_list)
443 return NULL;
444
445 blkg = container_of(ent, struct blkcg_gq, q_node);
446 return &blkg->rl;
447}
448
449static int blkcg_reset_stats(struct cgroup_subsys_state *css,
450 struct cftype *cftype, u64 val)
451{
452 struct blkcg *blkcg = css_to_blkcg(css);
453 struct blkcg_gq *blkg;
454 int i;
455
456 mutex_lock(&blkcg_pol_mutex);
457 spin_lock_irq(&blkcg->lock);
458
459 /*
460 * Note that stat reset is racy - it doesn't synchronize against
461 * stat updates. This is a debug feature which shouldn't exist
462 * anyway. If you get hit by a race, retry.
463 */
464 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
465 blkg_rwstat_reset(&blkg->stat_bytes);
466 blkg_rwstat_reset(&blkg->stat_ios);
467
468 for (i = 0; i < BLKCG_MAX_POLS; i++) {
469 struct blkcg_policy *pol = blkcg_policy[i];
470
471 if (blkg->pd[i] && pol->pd_reset_stats_fn)
472 pol->pd_reset_stats_fn(blkg->pd[i]);
473 }
474 }
475
476 spin_unlock_irq(&blkcg->lock);
477 mutex_unlock(&blkcg_pol_mutex);
478 return 0;
479}
480
481const char *blkg_dev_name(struct blkcg_gq *blkg)
482{
483 /* some drivers (floppy) instantiate a queue w/o disk registered */
484 if (blkg->q->backing_dev_info->dev)
485 return dev_name(blkg->q->backing_dev_info->dev);
486 return NULL;
487}
488EXPORT_SYMBOL_GPL(blkg_dev_name);
489
490/**
491 * blkcg_print_blkgs - helper for printing per-blkg data
492 * @sf: seq_file to print to
493 * @blkcg: blkcg of interest
494 * @prfill: fill function to print out a blkg
495 * @pol: policy in question
496 * @data: data to be passed to @prfill
497 * @show_total: to print out sum of prfill return values or not
498 *
499 * This function invokes @prfill on each blkg of @blkcg if pd for the
500 * policy specified by @pol exists. @prfill is invoked with @sf, the
501 * policy data and @data and the matching queue lock held. If @show_total
502 * is %true, the sum of the return values from @prfill is printed with
503 * "Total" label at the end.
504 *
505 * This is to be used to construct print functions for
506 * cftype->read_seq_string method.
507 */
508void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
509 u64 (*prfill)(struct seq_file *,
510 struct blkg_policy_data *, int),
511 const struct blkcg_policy *pol, int data,
512 bool show_total)
513{
514 struct blkcg_gq *blkg;
515 u64 total = 0;
516
517 rcu_read_lock();
518 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
519 spin_lock_irq(blkg->q->queue_lock);
520 if (blkcg_policy_enabled(blkg->q, pol))
521 total += prfill(sf, blkg->pd[pol->plid], data);
522 spin_unlock_irq(blkg->q->queue_lock);
523 }
524 rcu_read_unlock();
525
526 if (show_total)
527 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
528}
529EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
530
531/**
532 * __blkg_prfill_u64 - prfill helper for a single u64 value
533 * @sf: seq_file to print to
534 * @pd: policy private data of interest
535 * @v: value to print
536 *
537 * Print @v to @sf for the device assocaited with @pd.
538 */
539u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
540{
541 const char *dname = blkg_dev_name(pd->blkg);
542
543 if (!dname)
544 return 0;
545
546 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
547 return v;
548}
549EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
550
551/**
552 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
553 * @sf: seq_file to print to
554 * @pd: policy private data of interest
555 * @rwstat: rwstat to print
556 *
557 * Print @rwstat to @sf for the device assocaited with @pd.
558 */
559u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
560 const struct blkg_rwstat *rwstat)
561{
562 static const char *rwstr[] = {
563 [BLKG_RWSTAT_READ] = "Read",
564 [BLKG_RWSTAT_WRITE] = "Write",
565 [BLKG_RWSTAT_SYNC] = "Sync",
566 [BLKG_RWSTAT_ASYNC] = "Async",
567 };
568 const char *dname = blkg_dev_name(pd->blkg);
569 u64 v;
570 int i;
571
572 if (!dname)
573 return 0;
574
575 for (i = 0; i < BLKG_RWSTAT_NR; i++)
576 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
577 (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
578
579 v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
580 atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
581 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
582 return v;
583}
584EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
585
586/**
587 * blkg_prfill_stat - prfill callback for blkg_stat
588 * @sf: seq_file to print to
589 * @pd: policy private data of interest
590 * @off: offset to the blkg_stat in @pd
591 *
592 * prfill callback for printing a blkg_stat.
593 */
594u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
595{
596 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
597}
598EXPORT_SYMBOL_GPL(blkg_prfill_stat);
599
600/**
601 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
602 * @sf: seq_file to print to
603 * @pd: policy private data of interest
604 * @off: offset to the blkg_rwstat in @pd
605 *
606 * prfill callback for printing a blkg_rwstat.
607 */
608u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
609 int off)
610{
611 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
612
613 return __blkg_prfill_rwstat(sf, pd, &rwstat);
614}
615EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
616
617static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
618 struct blkg_policy_data *pd, int off)
619{
620 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
621
622 return __blkg_prfill_rwstat(sf, pd, &rwstat);
623}
624
625/**
626 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
627 * @sf: seq_file to print to
628 * @v: unused
629 *
630 * To be used as cftype->seq_show to print blkg->stat_bytes.
631 * cftype->private must be set to the blkcg_policy.
632 */
633int blkg_print_stat_bytes(struct seq_file *sf, void *v)
634{
635 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
636 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
637 offsetof(struct blkcg_gq, stat_bytes), true);
638 return 0;
639}
640EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
641
642/**
643 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
644 * @sf: seq_file to print to
645 * @v: unused
646 *
647 * To be used as cftype->seq_show to print blkg->stat_ios. cftype->private
648 * must be set to the blkcg_policy.
649 */
650int blkg_print_stat_ios(struct seq_file *sf, void *v)
651{
652 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
653 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
654 offsetof(struct blkcg_gq, stat_ios), true);
655 return 0;
656}
657EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
658
659static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
660 struct blkg_policy_data *pd,
661 int off)
662{
663 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
664 NULL, off);
665 return __blkg_prfill_rwstat(sf, pd, &rwstat);
666}
667
668/**
669 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
670 * @sf: seq_file to print to
671 * @v: unused
672 */
673int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
674{
675 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
676 blkg_prfill_rwstat_field_recursive,
677 (void *)seq_cft(sf)->private,
678 offsetof(struct blkcg_gq, stat_bytes), true);
679 return 0;
680}
681EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
682
683/**
684 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
685 * @sf: seq_file to print to
686 * @v: unused
687 */
688int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
689{
690 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
691 blkg_prfill_rwstat_field_recursive,
692 (void *)seq_cft(sf)->private,
693 offsetof(struct blkcg_gq, stat_ios), true);
694 return 0;
695}
696EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
697
698/**
699 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
700 * @blkg: blkg of interest
701 * @pol: blkcg_policy which contains the blkg_stat
702 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
703 *
704 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
705 * online descendants and their aux counts. The caller must be holding the
706 * queue lock for online tests.
707 *
708 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
709 * at @off bytes into @blkg's blkg_policy_data of the policy.
710 */
711u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
712 struct blkcg_policy *pol, int off)
713{
714 struct blkcg_gq *pos_blkg;
715 struct cgroup_subsys_state *pos_css;
716 u64 sum = 0;
717
718 lockdep_assert_held(blkg->q->queue_lock);
719
720 rcu_read_lock();
721 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
722 struct blkg_stat *stat;
723
724 if (!pos_blkg->online)
725 continue;
726
727 if (pol)
728 stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
729 else
730 stat = (void *)blkg + off;
731
732 sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
733 }
734 rcu_read_unlock();
735
736 return sum;
737}
738EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
739
740/**
741 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
742 * @blkg: blkg of interest
743 * @pol: blkcg_policy which contains the blkg_rwstat
744 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
745 *
746 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
747 * online descendants and their aux counts. The caller must be holding the
748 * queue lock for online tests.
749 *
750 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
751 * is at @off bytes into @blkg's blkg_policy_data of the policy.
752 */
753struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
754 struct blkcg_policy *pol, int off)
755{
756 struct blkcg_gq *pos_blkg;
757 struct cgroup_subsys_state *pos_css;
758 struct blkg_rwstat sum = { };
759 int i;
760
761 lockdep_assert_held(blkg->q->queue_lock);
762
763 rcu_read_lock();
764 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
765 struct blkg_rwstat *rwstat;
766
767 if (!pos_blkg->online)
768 continue;
769
770 if (pol)
771 rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
772 else
773 rwstat = (void *)pos_blkg + off;
774
775 for (i = 0; i < BLKG_RWSTAT_NR; i++)
776 atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
777 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
778 &sum.aux_cnt[i]);
779 }
780 rcu_read_unlock();
781
782 return sum;
783}
784EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
785
786/* Performs queue bypass and policy enabled checks then looks up blkg. */
787static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
788 const struct blkcg_policy *pol,
789 struct request_queue *q)
790{
791 WARN_ON_ONCE(!rcu_read_lock_held());
792 lockdep_assert_held(q->queue_lock);
793
794 if (!blkcg_policy_enabled(q, pol))
795 return ERR_PTR(-EOPNOTSUPP);
796
797 /*
798 * This could be the first entry point of blkcg implementation and
799 * we shouldn't allow anything to go through for a bypassing queue.
800 */
801 if (unlikely(blk_queue_bypass(q)))
802 return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
803
804 return __blkg_lookup(blkcg, q, true /* update_hint */);
805}
806
807/**
808 * blkg_conf_prep - parse and prepare for per-blkg config update
809 * @blkcg: target block cgroup
810 * @pol: target policy
811 * @input: input string
812 * @ctx: blkg_conf_ctx to be filled
813 *
814 * Parse per-blkg config update from @input and initialize @ctx with the
815 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
816 * part of @input following MAJ:MIN. This function returns with RCU read
817 * lock and queue lock held and must be paired with blkg_conf_finish().
818 */
819int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
820 char *input, struct blkg_conf_ctx *ctx)
821 __acquires(rcu) __acquires(disk->queue->queue_lock)
822{
823 struct gendisk *disk;
824 struct request_queue *q;
825 struct blkcg_gq *blkg;
826 unsigned int major, minor;
827 int key_len, part, ret;
828 char *body;
829
830 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
831 return -EINVAL;
832
833 body = input + key_len;
834 if (!isspace(*body))
835 return -EINVAL;
836 body = skip_spaces(body);
837
838 disk = get_gendisk(MKDEV(major, minor), &part);
839 if (!disk)
840 return -ENODEV;
841 if (part) {
842 ret = -ENODEV;
843 goto fail;
844 }
845
846 q = disk->queue;
847
848 rcu_read_lock();
849 spin_lock_irq(q->queue_lock);
850
851 blkg = blkg_lookup_check(blkcg, pol, q);
852 if (IS_ERR(blkg)) {
853 ret = PTR_ERR(blkg);
854 goto fail_unlock;
855 }
856
857 if (blkg)
858 goto success;
859
860 /*
861 * Create blkgs walking down from blkcg_root to @blkcg, so that all
862 * non-root blkgs have access to their parents.
863 */
864 while (true) {
865 struct blkcg *pos = blkcg;
866 struct blkcg *parent;
867 struct blkcg_gq *new_blkg;
868
869 parent = blkcg_parent(blkcg);
870 while (parent && !__blkg_lookup(parent, q, false)) {
871 pos = parent;
872 parent = blkcg_parent(parent);
873 }
874
875 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
876 spin_unlock_irq(q->queue_lock);
877 rcu_read_unlock();
878
879 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
880 if (unlikely(!new_blkg)) {
881 ret = -ENOMEM;
882 goto fail;
883 }
884
885 rcu_read_lock();
886 spin_lock_irq(q->queue_lock);
887
888 blkg = blkg_lookup_check(pos, pol, q);
889 if (IS_ERR(blkg)) {
890 ret = PTR_ERR(blkg);
891 goto fail_unlock;
892 }
893
894 if (blkg) {
895 blkg_free(new_blkg);
896 } else {
897 blkg = blkg_create(pos, q, new_blkg);
898 if (unlikely(IS_ERR(blkg))) {
899 ret = PTR_ERR(blkg);
900 goto fail_unlock;
901 }
902 }
903
904 if (pos == blkcg)
905 goto success;
906 }
907success:
908 ctx->disk = disk;
909 ctx->blkg = blkg;
910 ctx->body = body;
911 return 0;
912
913fail_unlock:
914 spin_unlock_irq(q->queue_lock);
915 rcu_read_unlock();
916fail:
917 put_disk_and_module(disk);
918 /*
919 * If queue was bypassing, we should retry. Do so after a
920 * short msleep(). It isn't strictly necessary but queue
921 * can be bypassing for some time and it's always nice to
922 * avoid busy looping.
923 */
924 if (ret == -EBUSY) {
925 msleep(10);
926 ret = restart_syscall();
927 }
928 return ret;
929}
930EXPORT_SYMBOL_GPL(blkg_conf_prep);
931
932/**
933 * blkg_conf_finish - finish up per-blkg config update
934 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
935 *
936 * Finish up after per-blkg config update. This function must be paired
937 * with blkg_conf_prep().
938 */
939void blkg_conf_finish(struct blkg_conf_ctx *ctx)
940 __releases(ctx->disk->queue->queue_lock) __releases(rcu)
941{
942 spin_unlock_irq(ctx->disk->queue->queue_lock);
943 rcu_read_unlock();
944 put_disk_and_module(ctx->disk);
945}
946EXPORT_SYMBOL_GPL(blkg_conf_finish);
947
948static int blkcg_print_stat(struct seq_file *sf, void *v)
949{
950 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
951 struct blkcg_gq *blkg;
952
953 rcu_read_lock();
954
955 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
956 const char *dname;
957 struct blkg_rwstat rwstat;
958 u64 rbytes, wbytes, rios, wios;
959
960 dname = blkg_dev_name(blkg);
961 if (!dname)
962 continue;
963
964 spin_lock_irq(blkg->q->queue_lock);
965
966 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
967 offsetof(struct blkcg_gq, stat_bytes));
968 rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
969 wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
970
971 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
972 offsetof(struct blkcg_gq, stat_ios));
973 rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
974 wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
975
976 spin_unlock_irq(blkg->q->queue_lock);
977
978 if (rbytes || wbytes || rios || wios)
979 seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
980 dname, rbytes, wbytes, rios, wios);
981 }
982
983 rcu_read_unlock();
984 return 0;
985}
986
987static struct cftype blkcg_files[] = {
988 {
989 .name = "stat",
990 .flags = CFTYPE_NOT_ON_ROOT,
991 .seq_show = blkcg_print_stat,
992 },
993 { } /* terminate */
994};
995
996static struct cftype blkcg_legacy_files[] = {
997 {
998 .name = "reset_stats",
999 .write_u64 = blkcg_reset_stats,
1000 },
1001 { } /* terminate */
1002};
1003
1004/**
1005 * blkcg_css_offline - cgroup css_offline callback
1006 * @css: css of interest
1007 *
1008 * This function is called when @css is about to go away and responsible
1009 * for offlining all blkgs pd and killing all wbs associated with @css.
1010 * blkgs pd offline should be done while holding both q and blkcg locks.
1011 * As blkcg lock is nested inside q lock, this function performs reverse
1012 * double lock dancing.
1013 *
1014 * This is the blkcg counterpart of ioc_release_fn().
1015 */
1016static void blkcg_css_offline(struct cgroup_subsys_state *css)
1017{
1018 struct blkcg *blkcg = css_to_blkcg(css);
1019 struct blkcg_gq *blkg;
1020
1021 spin_lock_irq(&blkcg->lock);
1022
1023 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
1024 struct request_queue *q = blkg->q;
1025
1026 if (spin_trylock(q->queue_lock)) {
1027 blkg_pd_offline(blkg);
1028 spin_unlock(q->queue_lock);
1029 } else {
1030 spin_unlock_irq(&blkcg->lock);
1031 cpu_relax();
1032 spin_lock_irq(&blkcg->lock);
1033 }
1034 }
1035
1036 spin_unlock_irq(&blkcg->lock);
1037
1038 wb_blkcg_offline(blkcg);
1039}
1040
1041/**
1042 * blkcg_destroy_all_blkgs - destroy all blkgs associated with a blkcg
1043 * @blkcg: blkcg of interest
1044 *
1045 * This function is called when blkcg css is about to free and responsible for
1046 * destroying all blkgs associated with @blkcg.
1047 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1048 * is nested inside q lock, this function performs reverse double lock dancing.
1049 */
1050static void blkcg_destroy_all_blkgs(struct blkcg *blkcg)
1051{
1052 spin_lock_irq(&blkcg->lock);
1053 while (!hlist_empty(&blkcg->blkg_list)) {
1054 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1055 struct blkcg_gq,
1056 blkcg_node);
1057 struct request_queue *q = blkg->q;
1058
1059 if (spin_trylock(q->queue_lock)) {
1060 blkg_destroy(blkg);
1061 spin_unlock(q->queue_lock);
1062 } else {
1063 spin_unlock_irq(&blkcg->lock);
1064 cpu_relax();
1065 spin_lock_irq(&blkcg->lock);
1066 }
1067 }
1068 spin_unlock_irq(&blkcg->lock);
1069}
1070
1071static void blkcg_css_free(struct cgroup_subsys_state *css)
1072{
1073 struct blkcg *blkcg = css_to_blkcg(css);
1074 int i;
1075
1076 blkcg_destroy_all_blkgs(blkcg);
1077
1078 mutex_lock(&blkcg_pol_mutex);
1079
1080 list_del(&blkcg->all_blkcgs_node);
1081
1082 for (i = 0; i < BLKCG_MAX_POLS; i++)
1083 if (blkcg->cpd[i])
1084 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1085
1086 mutex_unlock(&blkcg_pol_mutex);
1087
1088 kfree(blkcg);
1089}
1090
1091static struct cgroup_subsys_state *
1092blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1093{
1094 struct blkcg *blkcg;
1095 struct cgroup_subsys_state *ret;
1096 int i;
1097
1098 mutex_lock(&blkcg_pol_mutex);
1099
1100 if (!parent_css) {
1101 blkcg = &blkcg_root;
1102 } else {
1103 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1104 if (!blkcg) {
1105 ret = ERR_PTR(-ENOMEM);
1106 goto unlock;
1107 }
1108 }
1109
1110 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1111 struct blkcg_policy *pol = blkcg_policy[i];
1112 struct blkcg_policy_data *cpd;
1113
1114 /*
1115 * If the policy hasn't been attached yet, wait for it
1116 * to be attached before doing anything else. Otherwise,
1117 * check if the policy requires any specific per-cgroup
1118 * data: if it does, allocate and initialize it.
1119 */
1120 if (!pol || !pol->cpd_alloc_fn)
1121 continue;
1122
1123 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1124 if (!cpd) {
1125 ret = ERR_PTR(-ENOMEM);
1126 goto free_pd_blkcg;
1127 }
1128 blkcg->cpd[i] = cpd;
1129 cpd->blkcg = blkcg;
1130 cpd->plid = i;
1131 if (pol->cpd_init_fn)
1132 pol->cpd_init_fn(cpd);
1133 }
1134
1135 spin_lock_init(&blkcg->lock);
1136 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1137 INIT_HLIST_HEAD(&blkcg->blkg_list);
1138#ifdef CONFIG_CGROUP_WRITEBACK
1139 INIT_LIST_HEAD(&blkcg->cgwb_list);
1140#endif
1141 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1142
1143 mutex_unlock(&blkcg_pol_mutex);
1144 return &blkcg->css;
1145
1146free_pd_blkcg:
1147 for (i--; i >= 0; i--)
1148 if (blkcg->cpd[i])
1149 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1150
1151 if (blkcg != &blkcg_root)
1152 kfree(blkcg);
1153unlock:
1154 mutex_unlock(&blkcg_pol_mutex);
1155 return ret;
1156}
1157
1158/**
1159 * blkcg_init_queue - initialize blkcg part of request queue
1160 * @q: request_queue to initialize
1161 *
1162 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1163 * part of new request_queue @q.
1164 *
1165 * RETURNS:
1166 * 0 on success, -errno on failure.
1167 */
1168int blkcg_init_queue(struct request_queue *q)
1169{
1170 struct blkcg_gq *new_blkg, *blkg;
1171 bool preloaded;
1172 int ret;
1173
1174 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1175 if (!new_blkg)
1176 return -ENOMEM;
1177
1178 preloaded = !radix_tree_preload(GFP_KERNEL);
1179
1180 /* Make sure the root blkg exists. */
1181 rcu_read_lock();
1182 spin_lock_irq(q->queue_lock);
1183 blkg = blkg_create(&blkcg_root, q, new_blkg);
1184 if (IS_ERR(blkg))
1185 goto err_unlock;
1186 q->root_blkg = blkg;
1187 q->root_rl.blkg = blkg;
1188 spin_unlock_irq(q->queue_lock);
1189 rcu_read_unlock();
1190
1191 if (preloaded)
1192 radix_tree_preload_end();
1193
1194 ret = blk_throtl_init(q);
1195 if (ret) {
1196 spin_lock_irq(q->queue_lock);
1197 blkg_destroy_all(q);
1198 spin_unlock_irq(q->queue_lock);
1199 }
1200 return ret;
1201
1202err_unlock:
1203 spin_unlock_irq(q->queue_lock);
1204 rcu_read_unlock();
1205 if (preloaded)
1206 radix_tree_preload_end();
1207 return PTR_ERR(blkg);
1208}
1209
1210/**
1211 * blkcg_drain_queue - drain blkcg part of request_queue
1212 * @q: request_queue to drain
1213 *
1214 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1215 */
1216void blkcg_drain_queue(struct request_queue *q)
1217{
1218 lockdep_assert_held(q->queue_lock);
1219
1220 /*
1221 * @q could be exiting and already have destroyed all blkgs as
1222 * indicated by NULL root_blkg. If so, don't confuse policies.
1223 */
1224 if (!q->root_blkg)
1225 return;
1226
1227 blk_throtl_drain(q);
1228}
1229
1230/**
1231 * blkcg_exit_queue - exit and release blkcg part of request_queue
1232 * @q: request_queue being released
1233 *
1234 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1235 */
1236void blkcg_exit_queue(struct request_queue *q)
1237{
1238 spin_lock_irq(q->queue_lock);
1239 blkg_destroy_all(q);
1240 spin_unlock_irq(q->queue_lock);
1241
1242 blk_throtl_exit(q);
1243}
1244
1245/*
1246 * We cannot support shared io contexts, as we have no mean to support
1247 * two tasks with the same ioc in two different groups without major rework
1248 * of the main cic data structures. For now we allow a task to change
1249 * its cgroup only if it's the only owner of its ioc.
1250 */
1251static int blkcg_can_attach(struct cgroup_taskset *tset)
1252{
1253 struct task_struct *task;
1254 struct cgroup_subsys_state *dst_css;
1255 struct io_context *ioc;
1256 int ret = 0;
1257
1258 /* task_lock() is needed to avoid races with exit_io_context() */
1259 cgroup_taskset_for_each(task, dst_css, tset) {
1260 task_lock(task);
1261 ioc = task->io_context;
1262 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1263 ret = -EINVAL;
1264 task_unlock(task);
1265 if (ret)
1266 break;
1267 }
1268 return ret;
1269}
1270
1271static void blkcg_bind(struct cgroup_subsys_state *root_css)
1272{
1273 int i;
1274
1275 mutex_lock(&blkcg_pol_mutex);
1276
1277 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1278 struct blkcg_policy *pol = blkcg_policy[i];
1279 struct blkcg *blkcg;
1280
1281 if (!pol || !pol->cpd_bind_fn)
1282 continue;
1283
1284 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1285 if (blkcg->cpd[pol->plid])
1286 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1287 }
1288 mutex_unlock(&blkcg_pol_mutex);
1289}
1290
1291struct cgroup_subsys io_cgrp_subsys = {
1292 .css_alloc = blkcg_css_alloc,
1293 .css_offline = blkcg_css_offline,
1294 .css_free = blkcg_css_free,
1295 .can_attach = blkcg_can_attach,
1296 .bind = blkcg_bind,
1297 .dfl_cftypes = blkcg_files,
1298 .legacy_cftypes = blkcg_legacy_files,
1299 .legacy_name = "blkio",
1300#ifdef CONFIG_MEMCG
1301 /*
1302 * This ensures that, if available, memcg is automatically enabled
1303 * together on the default hierarchy so that the owner cgroup can
1304 * be retrieved from writeback pages.
1305 */
1306 .depends_on = 1 << memory_cgrp_id,
1307#endif
1308};
1309EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1310
1311/**
1312 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1313 * @q: request_queue of interest
1314 * @pol: blkcg policy to activate
1315 *
1316 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1317 * bypass mode to populate its blkgs with policy_data for @pol.
1318 *
1319 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1320 * from IO path. Update of each blkg is protected by both queue and blkcg
1321 * locks so that holding either lock and testing blkcg_policy_enabled() is
1322 * always enough for dereferencing policy data.
1323 *
1324 * The caller is responsible for synchronizing [de]activations and policy
1325 * [un]registerations. Returns 0 on success, -errno on failure.
1326 */
1327int blkcg_activate_policy(struct request_queue *q,
1328 const struct blkcg_policy *pol)
1329{
1330 struct blkg_policy_data *pd_prealloc = NULL;
1331 struct blkcg_gq *blkg;
1332 int ret;
1333
1334 if (blkcg_policy_enabled(q, pol))
1335 return 0;
1336
1337 if (q->mq_ops)
1338 blk_mq_freeze_queue(q);
1339 else
1340 blk_queue_bypass_start(q);
1341pd_prealloc:
1342 if (!pd_prealloc) {
1343 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1344 if (!pd_prealloc) {
1345 ret = -ENOMEM;
1346 goto out_bypass_end;
1347 }
1348 }
1349
1350 spin_lock_irq(q->queue_lock);
1351
1352 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1353 struct blkg_policy_data *pd;
1354
1355 if (blkg->pd[pol->plid])
1356 continue;
1357
1358 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
1359 if (!pd)
1360 swap(pd, pd_prealloc);
1361 if (!pd) {
1362 spin_unlock_irq(q->queue_lock);
1363 goto pd_prealloc;
1364 }
1365
1366 blkg->pd[pol->plid] = pd;
1367 pd->blkg = blkg;
1368 pd->plid = pol->plid;
1369 if (pol->pd_init_fn)
1370 pol->pd_init_fn(pd);
1371 }
1372
1373 __set_bit(pol->plid, q->blkcg_pols);
1374 ret = 0;
1375
1376 spin_unlock_irq(q->queue_lock);
1377out_bypass_end:
1378 if (q->mq_ops)
1379 blk_mq_unfreeze_queue(q);
1380 else
1381 blk_queue_bypass_end(q);
1382 if (pd_prealloc)
1383 pol->pd_free_fn(pd_prealloc);
1384 return ret;
1385}
1386EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1387
1388/**
1389 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1390 * @q: request_queue of interest
1391 * @pol: blkcg policy to deactivate
1392 *
1393 * Deactivate @pol on @q. Follows the same synchronization rules as
1394 * blkcg_activate_policy().
1395 */
1396void blkcg_deactivate_policy(struct request_queue *q,
1397 const struct blkcg_policy *pol)
1398{
1399 struct blkcg_gq *blkg;
1400
1401 if (!blkcg_policy_enabled(q, pol))
1402 return;
1403
1404 if (q->mq_ops)
1405 blk_mq_freeze_queue(q);
1406 else
1407 blk_queue_bypass_start(q);
1408
1409 spin_lock_irq(q->queue_lock);
1410
1411 __clear_bit(pol->plid, q->blkcg_pols);
1412
1413 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1414 if (blkg->pd[pol->plid]) {
1415 if (!blkg->pd[pol->plid]->offline &&
1416 pol->pd_offline_fn) {
1417 pol->pd_offline_fn(blkg->pd[pol->plid]);
1418 blkg->pd[pol->plid]->offline = true;
1419 }
1420 pol->pd_free_fn(blkg->pd[pol->plid]);
1421 blkg->pd[pol->plid] = NULL;
1422 }
1423 }
1424
1425 spin_unlock_irq(q->queue_lock);
1426
1427 if (q->mq_ops)
1428 blk_mq_unfreeze_queue(q);
1429 else
1430 blk_queue_bypass_end(q);
1431}
1432EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1433
1434/**
1435 * blkcg_policy_register - register a blkcg policy
1436 * @pol: blkcg policy to register
1437 *
1438 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1439 * successful registration. Returns 0 on success and -errno on failure.
1440 */
1441int blkcg_policy_register(struct blkcg_policy *pol)
1442{
1443 struct blkcg *blkcg;
1444 int i, ret;
1445
1446 mutex_lock(&blkcg_pol_register_mutex);
1447 mutex_lock(&blkcg_pol_mutex);
1448
1449 /* find an empty slot */
1450 ret = -ENOSPC;
1451 for (i = 0; i < BLKCG_MAX_POLS; i++)
1452 if (!blkcg_policy[i])
1453 break;
1454 if (i >= BLKCG_MAX_POLS)
1455 goto err_unlock;
1456
1457 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1458 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1459 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1460 goto err_unlock;
1461
1462 /* register @pol */
1463 pol->plid = i;
1464 blkcg_policy[pol->plid] = pol;
1465
1466 /* allocate and install cpd's */
1467 if (pol->cpd_alloc_fn) {
1468 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1469 struct blkcg_policy_data *cpd;
1470
1471 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1472 if (!cpd)
1473 goto err_free_cpds;
1474
1475 blkcg->cpd[pol->plid] = cpd;
1476 cpd->blkcg = blkcg;
1477 cpd->plid = pol->plid;
1478 pol->cpd_init_fn(cpd);
1479 }
1480 }
1481
1482 mutex_unlock(&blkcg_pol_mutex);
1483
1484 /* everything is in place, add intf files for the new policy */
1485 if (pol->dfl_cftypes)
1486 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1487 pol->dfl_cftypes));
1488 if (pol->legacy_cftypes)
1489 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1490 pol->legacy_cftypes));
1491 mutex_unlock(&blkcg_pol_register_mutex);
1492 return 0;
1493
1494err_free_cpds:
1495 if (pol->cpd_free_fn) {
1496 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1497 if (blkcg->cpd[pol->plid]) {
1498 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1499 blkcg->cpd[pol->plid] = NULL;
1500 }
1501 }
1502 }
1503 blkcg_policy[pol->plid] = NULL;
1504err_unlock:
1505 mutex_unlock(&blkcg_pol_mutex);
1506 mutex_unlock(&blkcg_pol_register_mutex);
1507 return ret;
1508}
1509EXPORT_SYMBOL_GPL(blkcg_policy_register);
1510
1511/**
1512 * blkcg_policy_unregister - unregister a blkcg policy
1513 * @pol: blkcg policy to unregister
1514 *
1515 * Undo blkcg_policy_register(@pol). Might sleep.
1516 */
1517void blkcg_policy_unregister(struct blkcg_policy *pol)
1518{
1519 struct blkcg *blkcg;
1520
1521 mutex_lock(&blkcg_pol_register_mutex);
1522
1523 if (WARN_ON(blkcg_policy[pol->plid] != pol))
1524 goto out_unlock;
1525
1526 /* kill the intf files first */
1527 if (pol->dfl_cftypes)
1528 cgroup_rm_cftypes(pol->dfl_cftypes);
1529 if (pol->legacy_cftypes)
1530 cgroup_rm_cftypes(pol->legacy_cftypes);
1531
1532 /* remove cpds and unregister */
1533 mutex_lock(&blkcg_pol_mutex);
1534
1535 if (pol->cpd_free_fn) {
1536 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1537 if (blkcg->cpd[pol->plid]) {
1538 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1539 blkcg->cpd[pol->plid] = NULL;
1540 }
1541 }
1542 }
1543 blkcg_policy[pol->plid] = NULL;
1544
1545 mutex_unlock(&blkcg_pol_mutex);
1546out_unlock:
1547 mutex_unlock(&blkcg_pol_register_mutex);
1548}
1549EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common Block IO controller cgroup interface
4 *
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7 *
8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9 * Paolo Valente <paolo.valente@unimore.it>
10 *
11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12 * Nauman Rafique <nauman@google.com>
13 *
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16 * Arianna Avanzini <avanzini.arianna@gmail.com>
17 */
18#include <linux/ioprio.h>
19#include <linux/kdev_t.h>
20#include <linux/module.h>
21#include <linux/sched/signal.h>
22#include <linux/err.h>
23#include <linux/blkdev.h>
24#include <linux/backing-dev.h>
25#include <linux/slab.h>
26#include <linux/delay.h>
27#include <linux/atomic.h>
28#include <linux/ctype.h>
29#include <linux/resume_user_mode.h>
30#include <linux/psi.h>
31#include <linux/part_stat.h>
32#include "blk.h"
33#include "blk-cgroup.h"
34#include "blk-ioprio.h"
35#include "blk-throttle.h"
36
37static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
38
39/*
40 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
41 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
42 * policy [un]register operations including cgroup file additions /
43 * removals. Putting cgroup file registration outside blkcg_pol_mutex
44 * allows grabbing it from cgroup callbacks.
45 */
46static DEFINE_MUTEX(blkcg_pol_register_mutex);
47static DEFINE_MUTEX(blkcg_pol_mutex);
48
49struct blkcg blkcg_root;
50EXPORT_SYMBOL_GPL(blkcg_root);
51
52struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
53EXPORT_SYMBOL_GPL(blkcg_root_css);
54
55static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
56
57static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
58
59bool blkcg_debug_stats = false;
60
61static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
62
63#define BLKG_DESTROY_BATCH_SIZE 64
64
65/*
66 * Lockless lists for tracking IO stats update
67 *
68 * New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg).
69 * There are multiple blkg's (one for each block device) attached to each
70 * blkcg. The rstat code keeps track of which cpu has IO stats updated,
71 * but it doesn't know which blkg has the updated stats. If there are many
72 * block devices in a system, the cost of iterating all the blkg's to flush
73 * out the IO stats can be high. To reduce such overhead, a set of percpu
74 * lockless lists (lhead) per blkcg are used to track the set of recently
75 * updated iostat_cpu's since the last flush. An iostat_cpu will be put
76 * onto the lockless list on the update side [blk_cgroup_bio_start()] if
77 * not there yet and then removed when being flushed [blkcg_rstat_flush()].
78 * References to blkg are gotten and then put back in the process to
79 * protect against blkg removal.
80 *
81 * Return: 0 if successful or -ENOMEM if allocation fails.
82 */
83static int init_blkcg_llists(struct blkcg *blkcg)
84{
85 int cpu;
86
87 blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);
88 if (!blkcg->lhead)
89 return -ENOMEM;
90
91 for_each_possible_cpu(cpu)
92 init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));
93 return 0;
94}
95
96/**
97 * blkcg_css - find the current css
98 *
99 * Find the css associated with either the kthread or the current task.
100 * This may return a dying css, so it is up to the caller to use tryget logic
101 * to confirm it is alive and well.
102 */
103static struct cgroup_subsys_state *blkcg_css(void)
104{
105 struct cgroup_subsys_state *css;
106
107 css = kthread_blkcg();
108 if (css)
109 return css;
110 return task_css(current, io_cgrp_id);
111}
112
113static bool blkcg_policy_enabled(struct request_queue *q,
114 const struct blkcg_policy *pol)
115{
116 return pol && test_bit(pol->plid, q->blkcg_pols);
117}
118
119static void blkg_free_workfn(struct work_struct *work)
120{
121 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
122 free_work);
123 struct request_queue *q = blkg->q;
124 int i;
125
126 /*
127 * pd_free_fn() can also be called from blkcg_deactivate_policy(),
128 * in order to make sure pd_free_fn() is called in order, the deletion
129 * of the list blkg->q_node is delayed to here from blkg_destroy(), and
130 * blkcg_mutex is used to synchronize blkg_free_workfn() and
131 * blkcg_deactivate_policy().
132 */
133 mutex_lock(&q->blkcg_mutex);
134 for (i = 0; i < BLKCG_MAX_POLS; i++)
135 if (blkg->pd[i])
136 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
137 if (blkg->parent)
138 blkg_put(blkg->parent);
139 spin_lock_irq(&q->queue_lock);
140 list_del_init(&blkg->q_node);
141 spin_unlock_irq(&q->queue_lock);
142 mutex_unlock(&q->blkcg_mutex);
143
144 blk_put_queue(q);
145 free_percpu(blkg->iostat_cpu);
146 percpu_ref_exit(&blkg->refcnt);
147 kfree(blkg);
148}
149
150/**
151 * blkg_free - free a blkg
152 * @blkg: blkg to free
153 *
154 * Free @blkg which may be partially allocated.
155 */
156static void blkg_free(struct blkcg_gq *blkg)
157{
158 if (!blkg)
159 return;
160
161 /*
162 * Both ->pd_free_fn() and request queue's release handler may
163 * sleep, so free us by scheduling one work func
164 */
165 INIT_WORK(&blkg->free_work, blkg_free_workfn);
166 schedule_work(&blkg->free_work);
167}
168
169static void __blkg_release(struct rcu_head *rcu)
170{
171 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
172 struct blkcg *blkcg = blkg->blkcg;
173 int cpu;
174
175#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
176 WARN_ON(!bio_list_empty(&blkg->async_bios));
177#endif
178 /*
179 * Flush all the non-empty percpu lockless lists before releasing
180 * us, given these stat belongs to us.
181 *
182 * blkg_stat_lock is for serializing blkg stat update
183 */
184 for_each_possible_cpu(cpu)
185 __blkcg_rstat_flush(blkcg, cpu);
186
187 /* release the blkcg and parent blkg refs this blkg has been holding */
188 css_put(&blkg->blkcg->css);
189 blkg_free(blkg);
190}
191
192/*
193 * A group is RCU protected, but having an rcu lock does not mean that one
194 * can access all the fields of blkg and assume these are valid. For
195 * example, don't try to follow throtl_data and request queue links.
196 *
197 * Having a reference to blkg under an rcu allows accesses to only values
198 * local to groups like group stats and group rate limits.
199 */
200static void blkg_release(struct percpu_ref *ref)
201{
202 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
203
204 call_rcu(&blkg->rcu_head, __blkg_release);
205}
206
207#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
208static struct workqueue_struct *blkcg_punt_bio_wq;
209
210static void blkg_async_bio_workfn(struct work_struct *work)
211{
212 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
213 async_bio_work);
214 struct bio_list bios = BIO_EMPTY_LIST;
215 struct bio *bio;
216 struct blk_plug plug;
217 bool need_plug = false;
218
219 /* as long as there are pending bios, @blkg can't go away */
220 spin_lock(&blkg->async_bio_lock);
221 bio_list_merge(&bios, &blkg->async_bios);
222 bio_list_init(&blkg->async_bios);
223 spin_unlock(&blkg->async_bio_lock);
224
225 /* start plug only when bio_list contains at least 2 bios */
226 if (bios.head && bios.head->bi_next) {
227 need_plug = true;
228 blk_start_plug(&plug);
229 }
230 while ((bio = bio_list_pop(&bios)))
231 submit_bio(bio);
232 if (need_plug)
233 blk_finish_plug(&plug);
234}
235
236/*
237 * When a shared kthread issues a bio for a cgroup, doing so synchronously can
238 * lead to priority inversions as the kthread can be trapped waiting for that
239 * cgroup. Use this helper instead of submit_bio to punt the actual issuing to
240 * a dedicated per-blkcg work item to avoid such priority inversions.
241 */
242void blkcg_punt_bio_submit(struct bio *bio)
243{
244 struct blkcg_gq *blkg = bio->bi_blkg;
245
246 if (blkg->parent) {
247 spin_lock(&blkg->async_bio_lock);
248 bio_list_add(&blkg->async_bios, bio);
249 spin_unlock(&blkg->async_bio_lock);
250 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
251 } else {
252 /* never bounce for the root cgroup */
253 submit_bio(bio);
254 }
255}
256EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);
257
258static int __init blkcg_punt_bio_init(void)
259{
260 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
261 WQ_MEM_RECLAIM | WQ_FREEZABLE |
262 WQ_UNBOUND | WQ_SYSFS, 0);
263 if (!blkcg_punt_bio_wq)
264 return -ENOMEM;
265 return 0;
266}
267subsys_initcall(blkcg_punt_bio_init);
268#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */
269
270/**
271 * bio_blkcg_css - return the blkcg CSS associated with a bio
272 * @bio: target bio
273 *
274 * This returns the CSS for the blkcg associated with a bio, or %NULL if not
275 * associated. Callers are expected to either handle %NULL or know association
276 * has been done prior to calling this.
277 */
278struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
279{
280 if (!bio || !bio->bi_blkg)
281 return NULL;
282 return &bio->bi_blkg->blkcg->css;
283}
284EXPORT_SYMBOL_GPL(bio_blkcg_css);
285
286/**
287 * blkcg_parent - get the parent of a blkcg
288 * @blkcg: blkcg of interest
289 *
290 * Return the parent blkcg of @blkcg. Can be called anytime.
291 */
292static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
293{
294 return css_to_blkcg(blkcg->css.parent);
295}
296
297/**
298 * blkg_alloc - allocate a blkg
299 * @blkcg: block cgroup the new blkg is associated with
300 * @disk: gendisk the new blkg is associated with
301 * @gfp_mask: allocation mask to use
302 *
303 * Allocate a new blkg associating @blkcg and @disk.
304 */
305static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
306 gfp_t gfp_mask)
307{
308 struct blkcg_gq *blkg;
309 int i, cpu;
310
311 /* alloc and init base part */
312 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node);
313 if (!blkg)
314 return NULL;
315 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
316 goto out_free_blkg;
317 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
318 if (!blkg->iostat_cpu)
319 goto out_exit_refcnt;
320 if (!blk_get_queue(disk->queue))
321 goto out_free_iostat;
322
323 blkg->q = disk->queue;
324 INIT_LIST_HEAD(&blkg->q_node);
325 blkg->blkcg = blkcg;
326#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
327 spin_lock_init(&blkg->async_bio_lock);
328 bio_list_init(&blkg->async_bios);
329 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
330#endif
331
332 u64_stats_init(&blkg->iostat.sync);
333 for_each_possible_cpu(cpu) {
334 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
335 per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
336 }
337
338 for (i = 0; i < BLKCG_MAX_POLS; i++) {
339 struct blkcg_policy *pol = blkcg_policy[i];
340 struct blkg_policy_data *pd;
341
342 if (!blkcg_policy_enabled(disk->queue, pol))
343 continue;
344
345 /* alloc per-policy data and attach it to blkg */
346 pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);
347 if (!pd)
348 goto out_free_pds;
349 blkg->pd[i] = pd;
350 pd->blkg = blkg;
351 pd->plid = i;
352 pd->online = false;
353 }
354
355 return blkg;
356
357out_free_pds:
358 while (--i >= 0)
359 if (blkg->pd[i])
360 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
361 blk_put_queue(disk->queue);
362out_free_iostat:
363 free_percpu(blkg->iostat_cpu);
364out_exit_refcnt:
365 percpu_ref_exit(&blkg->refcnt);
366out_free_blkg:
367 kfree(blkg);
368 return NULL;
369}
370
371/*
372 * If @new_blkg is %NULL, this function tries to allocate a new one as
373 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
374 */
375static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
376 struct blkcg_gq *new_blkg)
377{
378 struct blkcg_gq *blkg;
379 int i, ret;
380
381 lockdep_assert_held(&disk->queue->queue_lock);
382
383 /* request_queue is dying, do not create/recreate a blkg */
384 if (blk_queue_dying(disk->queue)) {
385 ret = -ENODEV;
386 goto err_free_blkg;
387 }
388
389 /* blkg holds a reference to blkcg */
390 if (!css_tryget_online(&blkcg->css)) {
391 ret = -ENODEV;
392 goto err_free_blkg;
393 }
394
395 /* allocate */
396 if (!new_blkg) {
397 new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN);
398 if (unlikely(!new_blkg)) {
399 ret = -ENOMEM;
400 goto err_put_css;
401 }
402 }
403 blkg = new_blkg;
404
405 /* link parent */
406 if (blkcg_parent(blkcg)) {
407 blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);
408 if (WARN_ON_ONCE(!blkg->parent)) {
409 ret = -ENODEV;
410 goto err_put_css;
411 }
412 blkg_get(blkg->parent);
413 }
414
415 /* invoke per-policy init */
416 for (i = 0; i < BLKCG_MAX_POLS; i++) {
417 struct blkcg_policy *pol = blkcg_policy[i];
418
419 if (blkg->pd[i] && pol->pd_init_fn)
420 pol->pd_init_fn(blkg->pd[i]);
421 }
422
423 /* insert */
424 spin_lock(&blkcg->lock);
425 ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);
426 if (likely(!ret)) {
427 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
428 list_add(&blkg->q_node, &disk->queue->blkg_list);
429
430 for (i = 0; i < BLKCG_MAX_POLS; i++) {
431 struct blkcg_policy *pol = blkcg_policy[i];
432
433 if (blkg->pd[i]) {
434 if (pol->pd_online_fn)
435 pol->pd_online_fn(blkg->pd[i]);
436 blkg->pd[i]->online = true;
437 }
438 }
439 }
440 blkg->online = true;
441 spin_unlock(&blkcg->lock);
442
443 if (!ret)
444 return blkg;
445
446 /* @blkg failed fully initialized, use the usual release path */
447 blkg_put(blkg);
448 return ERR_PTR(ret);
449
450err_put_css:
451 css_put(&blkcg->css);
452err_free_blkg:
453 if (new_blkg)
454 blkg_free(new_blkg);
455 return ERR_PTR(ret);
456}
457
458/**
459 * blkg_lookup_create - lookup blkg, try to create one if not there
460 * @blkcg: blkcg of interest
461 * @disk: gendisk of interest
462 *
463 * Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to
464 * create one. blkg creation is performed recursively from blkcg_root such
465 * that all non-root blkg's have access to the parent blkg. This function
466 * should be called under RCU read lock and takes @disk->queue->queue_lock.
467 *
468 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
469 * down from root.
470 */
471static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
472 struct gendisk *disk)
473{
474 struct request_queue *q = disk->queue;
475 struct blkcg_gq *blkg;
476 unsigned long flags;
477
478 WARN_ON_ONCE(!rcu_read_lock_held());
479
480 blkg = blkg_lookup(blkcg, q);
481 if (blkg)
482 return blkg;
483
484 spin_lock_irqsave(&q->queue_lock, flags);
485 blkg = blkg_lookup(blkcg, q);
486 if (blkg) {
487 if (blkcg != &blkcg_root &&
488 blkg != rcu_dereference(blkcg->blkg_hint))
489 rcu_assign_pointer(blkcg->blkg_hint, blkg);
490 goto found;
491 }
492
493 /*
494 * Create blkgs walking down from blkcg_root to @blkcg, so that all
495 * non-root blkgs have access to their parents. Returns the closest
496 * blkg to the intended blkg should blkg_create() fail.
497 */
498 while (true) {
499 struct blkcg *pos = blkcg;
500 struct blkcg *parent = blkcg_parent(blkcg);
501 struct blkcg_gq *ret_blkg = q->root_blkg;
502
503 while (parent) {
504 blkg = blkg_lookup(parent, q);
505 if (blkg) {
506 /* remember closest blkg */
507 ret_blkg = blkg;
508 break;
509 }
510 pos = parent;
511 parent = blkcg_parent(parent);
512 }
513
514 blkg = blkg_create(pos, disk, NULL);
515 if (IS_ERR(blkg)) {
516 blkg = ret_blkg;
517 break;
518 }
519 if (pos == blkcg)
520 break;
521 }
522
523found:
524 spin_unlock_irqrestore(&q->queue_lock, flags);
525 return blkg;
526}
527
528static void blkg_destroy(struct blkcg_gq *blkg)
529{
530 struct blkcg *blkcg = blkg->blkcg;
531 int i;
532
533 lockdep_assert_held(&blkg->q->queue_lock);
534 lockdep_assert_held(&blkcg->lock);
535
536 /*
537 * blkg stays on the queue list until blkg_free_workfn(), see details in
538 * blkg_free_workfn(), hence this function can be called from
539 * blkcg_destroy_blkgs() first and again from blkg_destroy_all() before
540 * blkg_free_workfn().
541 */
542 if (hlist_unhashed(&blkg->blkcg_node))
543 return;
544
545 for (i = 0; i < BLKCG_MAX_POLS; i++) {
546 struct blkcg_policy *pol = blkcg_policy[i];
547
548 if (blkg->pd[i] && blkg->pd[i]->online) {
549 blkg->pd[i]->online = false;
550 if (pol->pd_offline_fn)
551 pol->pd_offline_fn(blkg->pd[i]);
552 }
553 }
554
555 blkg->online = false;
556
557 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
558 hlist_del_init_rcu(&blkg->blkcg_node);
559
560 /*
561 * Both setting lookup hint to and clearing it from @blkg are done
562 * under queue_lock. If it's not pointing to @blkg now, it never
563 * will. Hint assignment itself can race safely.
564 */
565 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
566 rcu_assign_pointer(blkcg->blkg_hint, NULL);
567
568 /*
569 * Put the reference taken at the time of creation so that when all
570 * queues are gone, group can be destroyed.
571 */
572 percpu_ref_kill(&blkg->refcnt);
573}
574
575static void blkg_destroy_all(struct gendisk *disk)
576{
577 struct request_queue *q = disk->queue;
578 struct blkcg_gq *blkg;
579 int count = BLKG_DESTROY_BATCH_SIZE;
580 int i;
581
582restart:
583 spin_lock_irq(&q->queue_lock);
584 list_for_each_entry(blkg, &q->blkg_list, q_node) {
585 struct blkcg *blkcg = blkg->blkcg;
586
587 if (hlist_unhashed(&blkg->blkcg_node))
588 continue;
589
590 spin_lock(&blkcg->lock);
591 blkg_destroy(blkg);
592 spin_unlock(&blkcg->lock);
593
594 /*
595 * in order to avoid holding the spin lock for too long, release
596 * it when a batch of blkgs are destroyed.
597 */
598 if (!(--count)) {
599 count = BLKG_DESTROY_BATCH_SIZE;
600 spin_unlock_irq(&q->queue_lock);
601 cond_resched();
602 goto restart;
603 }
604 }
605
606 /*
607 * Mark policy deactivated since policy offline has been done, and
608 * the free is scheduled, so future blkcg_deactivate_policy() can
609 * be bypassed
610 */
611 for (i = 0; i < BLKCG_MAX_POLS; i++) {
612 struct blkcg_policy *pol = blkcg_policy[i];
613
614 if (pol)
615 __clear_bit(pol->plid, q->blkcg_pols);
616 }
617
618 q->root_blkg = NULL;
619 spin_unlock_irq(&q->queue_lock);
620}
621
622static int blkcg_reset_stats(struct cgroup_subsys_state *css,
623 struct cftype *cftype, u64 val)
624{
625 struct blkcg *blkcg = css_to_blkcg(css);
626 struct blkcg_gq *blkg;
627 int i, cpu;
628
629 mutex_lock(&blkcg_pol_mutex);
630 spin_lock_irq(&blkcg->lock);
631
632 /*
633 * Note that stat reset is racy - it doesn't synchronize against
634 * stat updates. This is a debug feature which shouldn't exist
635 * anyway. If you get hit by a race, retry.
636 */
637 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
638 for_each_possible_cpu(cpu) {
639 struct blkg_iostat_set *bis =
640 per_cpu_ptr(blkg->iostat_cpu, cpu);
641 memset(bis, 0, sizeof(*bis));
642
643 /* Re-initialize the cleared blkg_iostat_set */
644 u64_stats_init(&bis->sync);
645 bis->blkg = blkg;
646 }
647 memset(&blkg->iostat, 0, sizeof(blkg->iostat));
648 u64_stats_init(&blkg->iostat.sync);
649
650 for (i = 0; i < BLKCG_MAX_POLS; i++) {
651 struct blkcg_policy *pol = blkcg_policy[i];
652
653 if (blkg->pd[i] && pol->pd_reset_stats_fn)
654 pol->pd_reset_stats_fn(blkg->pd[i]);
655 }
656 }
657
658 spin_unlock_irq(&blkcg->lock);
659 mutex_unlock(&blkcg_pol_mutex);
660 return 0;
661}
662
663const char *blkg_dev_name(struct blkcg_gq *blkg)
664{
665 if (!blkg->q->disk)
666 return NULL;
667 return bdi_dev_name(blkg->q->disk->bdi);
668}
669
670/**
671 * blkcg_print_blkgs - helper for printing per-blkg data
672 * @sf: seq_file to print to
673 * @blkcg: blkcg of interest
674 * @prfill: fill function to print out a blkg
675 * @pol: policy in question
676 * @data: data to be passed to @prfill
677 * @show_total: to print out sum of prfill return values or not
678 *
679 * This function invokes @prfill on each blkg of @blkcg if pd for the
680 * policy specified by @pol exists. @prfill is invoked with @sf, the
681 * policy data and @data and the matching queue lock held. If @show_total
682 * is %true, the sum of the return values from @prfill is printed with
683 * "Total" label at the end.
684 *
685 * This is to be used to construct print functions for
686 * cftype->read_seq_string method.
687 */
688void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
689 u64 (*prfill)(struct seq_file *,
690 struct blkg_policy_data *, int),
691 const struct blkcg_policy *pol, int data,
692 bool show_total)
693{
694 struct blkcg_gq *blkg;
695 u64 total = 0;
696
697 rcu_read_lock();
698 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
699 spin_lock_irq(&blkg->q->queue_lock);
700 if (blkcg_policy_enabled(blkg->q, pol))
701 total += prfill(sf, blkg->pd[pol->plid], data);
702 spin_unlock_irq(&blkg->q->queue_lock);
703 }
704 rcu_read_unlock();
705
706 if (show_total)
707 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
708}
709EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
710
711/**
712 * __blkg_prfill_u64 - prfill helper for a single u64 value
713 * @sf: seq_file to print to
714 * @pd: policy private data of interest
715 * @v: value to print
716 *
717 * Print @v to @sf for the device associated with @pd.
718 */
719u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
720{
721 const char *dname = blkg_dev_name(pd->blkg);
722
723 if (!dname)
724 return 0;
725
726 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
727 return v;
728}
729EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
730
731/**
732 * blkg_conf_init - initialize a blkg_conf_ctx
733 * @ctx: blkg_conf_ctx to initialize
734 * @input: input string
735 *
736 * Initialize @ctx which can be used to parse blkg config input string @input.
737 * Once initialized, @ctx can be used with blkg_conf_open_bdev() and
738 * blkg_conf_prep(), and must be cleaned up with blkg_conf_exit().
739 */
740void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input)
741{
742 *ctx = (struct blkg_conf_ctx){ .input = input };
743}
744EXPORT_SYMBOL_GPL(blkg_conf_init);
745
746/**
747 * blkg_conf_open_bdev - parse and open bdev for per-blkg config update
748 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
749 *
750 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update from
751 * @ctx->input and get and store the matching bdev in @ctx->bdev. @ctx->body is
752 * set to point past the device node prefix.
753 *
754 * This function may be called multiple times on @ctx and the extra calls become
755 * NOOPs. blkg_conf_prep() implicitly calls this function. Use this function
756 * explicitly if bdev access is needed without resolving the blkcg / policy part
757 * of @ctx->input. Returns -errno on error.
758 */
759int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx)
760{
761 char *input = ctx->input;
762 unsigned int major, minor;
763 struct block_device *bdev;
764 int key_len;
765
766 if (ctx->bdev)
767 return 0;
768
769 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
770 return -EINVAL;
771
772 input += key_len;
773 if (!isspace(*input))
774 return -EINVAL;
775 input = skip_spaces(input);
776
777 bdev = blkdev_get_no_open(MKDEV(major, minor));
778 if (!bdev)
779 return -ENODEV;
780 if (bdev_is_partition(bdev)) {
781 blkdev_put_no_open(bdev);
782 return -ENODEV;
783 }
784
785 mutex_lock(&bdev->bd_queue->rq_qos_mutex);
786 if (!disk_live(bdev->bd_disk)) {
787 blkdev_put_no_open(bdev);
788 mutex_unlock(&bdev->bd_queue->rq_qos_mutex);
789 return -ENODEV;
790 }
791
792 ctx->body = input;
793 ctx->bdev = bdev;
794 return 0;
795}
796
797/**
798 * blkg_conf_prep - parse and prepare for per-blkg config update
799 * @blkcg: target block cgroup
800 * @pol: target policy
801 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
802 *
803 * Parse per-blkg config update from @ctx->input and initialize @ctx
804 * accordingly. On success, @ctx->body points to the part of @ctx->input
805 * following MAJ:MIN, @ctx->bdev points to the target block device and
806 * @ctx->blkg to the blkg being configured.
807 *
808 * blkg_conf_open_bdev() may be called on @ctx beforehand. On success, this
809 * function returns with queue lock held and must be followed by
810 * blkg_conf_exit().
811 */
812int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
813 struct blkg_conf_ctx *ctx)
814 __acquires(&bdev->bd_queue->queue_lock)
815{
816 struct gendisk *disk;
817 struct request_queue *q;
818 struct blkcg_gq *blkg;
819 int ret;
820
821 ret = blkg_conf_open_bdev(ctx);
822 if (ret)
823 return ret;
824
825 disk = ctx->bdev->bd_disk;
826 q = disk->queue;
827
828 /*
829 * blkcg_deactivate_policy() requires queue to be frozen, we can grab
830 * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
831 */
832 ret = blk_queue_enter(q, 0);
833 if (ret)
834 goto fail;
835
836 spin_lock_irq(&q->queue_lock);
837
838 if (!blkcg_policy_enabled(q, pol)) {
839 ret = -EOPNOTSUPP;
840 goto fail_unlock;
841 }
842
843 blkg = blkg_lookup(blkcg, q);
844 if (blkg)
845 goto success;
846
847 /*
848 * Create blkgs walking down from blkcg_root to @blkcg, so that all
849 * non-root blkgs have access to their parents.
850 */
851 while (true) {
852 struct blkcg *pos = blkcg;
853 struct blkcg *parent;
854 struct blkcg_gq *new_blkg;
855
856 parent = blkcg_parent(blkcg);
857 while (parent && !blkg_lookup(parent, q)) {
858 pos = parent;
859 parent = blkcg_parent(parent);
860 }
861
862 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
863 spin_unlock_irq(&q->queue_lock);
864
865 new_blkg = blkg_alloc(pos, disk, GFP_KERNEL);
866 if (unlikely(!new_blkg)) {
867 ret = -ENOMEM;
868 goto fail_exit_queue;
869 }
870
871 if (radix_tree_preload(GFP_KERNEL)) {
872 blkg_free(new_blkg);
873 ret = -ENOMEM;
874 goto fail_exit_queue;
875 }
876
877 spin_lock_irq(&q->queue_lock);
878
879 if (!blkcg_policy_enabled(q, pol)) {
880 blkg_free(new_blkg);
881 ret = -EOPNOTSUPP;
882 goto fail_preloaded;
883 }
884
885 blkg = blkg_lookup(pos, q);
886 if (blkg) {
887 blkg_free(new_blkg);
888 } else {
889 blkg = blkg_create(pos, disk, new_blkg);
890 if (IS_ERR(blkg)) {
891 ret = PTR_ERR(blkg);
892 goto fail_preloaded;
893 }
894 }
895
896 radix_tree_preload_end();
897
898 if (pos == blkcg)
899 goto success;
900 }
901success:
902 blk_queue_exit(q);
903 ctx->blkg = blkg;
904 return 0;
905
906fail_preloaded:
907 radix_tree_preload_end();
908fail_unlock:
909 spin_unlock_irq(&q->queue_lock);
910fail_exit_queue:
911 blk_queue_exit(q);
912fail:
913 /*
914 * If queue was bypassing, we should retry. Do so after a
915 * short msleep(). It isn't strictly necessary but queue
916 * can be bypassing for some time and it's always nice to
917 * avoid busy looping.
918 */
919 if (ret == -EBUSY) {
920 msleep(10);
921 ret = restart_syscall();
922 }
923 return ret;
924}
925EXPORT_SYMBOL_GPL(blkg_conf_prep);
926
927/**
928 * blkg_conf_exit - clean up per-blkg config update
929 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
930 *
931 * Clean up after per-blkg config update. This function must be called on all
932 * blkg_conf_ctx's initialized with blkg_conf_init().
933 */
934void blkg_conf_exit(struct blkg_conf_ctx *ctx)
935 __releases(&ctx->bdev->bd_queue->queue_lock)
936 __releases(&ctx->bdev->bd_queue->rq_qos_mutex)
937{
938 if (ctx->blkg) {
939 spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock);
940 ctx->blkg = NULL;
941 }
942
943 if (ctx->bdev) {
944 mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex);
945 blkdev_put_no_open(ctx->bdev);
946 ctx->body = NULL;
947 ctx->bdev = NULL;
948 }
949}
950EXPORT_SYMBOL_GPL(blkg_conf_exit);
951
952static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
953{
954 int i;
955
956 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
957 dst->bytes[i] = src->bytes[i];
958 dst->ios[i] = src->ios[i];
959 }
960}
961
962static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
963{
964 int i;
965
966 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
967 dst->bytes[i] += src->bytes[i];
968 dst->ios[i] += src->ios[i];
969 }
970}
971
972static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
973{
974 int i;
975
976 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
977 dst->bytes[i] -= src->bytes[i];
978 dst->ios[i] -= src->ios[i];
979 }
980}
981
982static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
983 struct blkg_iostat *last)
984{
985 struct blkg_iostat delta;
986 unsigned long flags;
987
988 /* propagate percpu delta to global */
989 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
990 blkg_iostat_set(&delta, cur);
991 blkg_iostat_sub(&delta, last);
992 blkg_iostat_add(&blkg->iostat.cur, &delta);
993 blkg_iostat_add(last, &delta);
994 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
995}
996
997static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
998{
999 struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
1000 struct llist_node *lnode;
1001 struct blkg_iostat_set *bisc, *next_bisc;
1002 unsigned long flags;
1003
1004 rcu_read_lock();
1005
1006 lnode = llist_del_all(lhead);
1007 if (!lnode)
1008 goto out;
1009
1010 /*
1011 * For covering concurrent parent blkg update from blkg_release().
1012 *
1013 * When flushing from cgroup, cgroup_rstat_lock is always held, so
1014 * this lock won't cause contention most of time.
1015 */
1016 raw_spin_lock_irqsave(&blkg_stat_lock, flags);
1017
1018 /*
1019 * Iterate only the iostat_cpu's queued in the lockless list.
1020 */
1021 llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) {
1022 struct blkcg_gq *blkg = bisc->blkg;
1023 struct blkcg_gq *parent = blkg->parent;
1024 struct blkg_iostat cur;
1025 unsigned int seq;
1026
1027 WRITE_ONCE(bisc->lqueued, false);
1028
1029 /* fetch the current per-cpu values */
1030 do {
1031 seq = u64_stats_fetch_begin(&bisc->sync);
1032 blkg_iostat_set(&cur, &bisc->cur);
1033 } while (u64_stats_fetch_retry(&bisc->sync, seq));
1034
1035 blkcg_iostat_update(blkg, &cur, &bisc->last);
1036
1037 /* propagate global delta to parent (unless that's root) */
1038 if (parent && parent->parent)
1039 blkcg_iostat_update(parent, &blkg->iostat.cur,
1040 &blkg->iostat.last);
1041 }
1042 raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
1043out:
1044 rcu_read_unlock();
1045}
1046
1047static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
1048{
1049 /* Root-level stats are sourced from system-wide IO stats */
1050 if (cgroup_parent(css->cgroup))
1051 __blkcg_rstat_flush(css_to_blkcg(css), cpu);
1052}
1053
1054/*
1055 * We source root cgroup stats from the system-wide stats to avoid
1056 * tracking the same information twice and incurring overhead when no
1057 * cgroups are defined. For that reason, cgroup_rstat_flush in
1058 * blkcg_print_stat does not actually fill out the iostat in the root
1059 * cgroup's blkcg_gq.
1060 *
1061 * However, we would like to re-use the printing code between the root and
1062 * non-root cgroups to the extent possible. For that reason, we simulate
1063 * flushing the root cgroup's stats by explicitly filling in the iostat
1064 * with disk level statistics.
1065 */
1066static void blkcg_fill_root_iostats(void)
1067{
1068 struct class_dev_iter iter;
1069 struct device *dev;
1070
1071 class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
1072 while ((dev = class_dev_iter_next(&iter))) {
1073 struct block_device *bdev = dev_to_bdev(dev);
1074 struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;
1075 struct blkg_iostat tmp;
1076 int cpu;
1077 unsigned long flags;
1078
1079 memset(&tmp, 0, sizeof(tmp));
1080 for_each_possible_cpu(cpu) {
1081 struct disk_stats *cpu_dkstats;
1082
1083 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
1084 tmp.ios[BLKG_IOSTAT_READ] +=
1085 cpu_dkstats->ios[STAT_READ];
1086 tmp.ios[BLKG_IOSTAT_WRITE] +=
1087 cpu_dkstats->ios[STAT_WRITE];
1088 tmp.ios[BLKG_IOSTAT_DISCARD] +=
1089 cpu_dkstats->ios[STAT_DISCARD];
1090 // convert sectors to bytes
1091 tmp.bytes[BLKG_IOSTAT_READ] +=
1092 cpu_dkstats->sectors[STAT_READ] << 9;
1093 tmp.bytes[BLKG_IOSTAT_WRITE] +=
1094 cpu_dkstats->sectors[STAT_WRITE] << 9;
1095 tmp.bytes[BLKG_IOSTAT_DISCARD] +=
1096 cpu_dkstats->sectors[STAT_DISCARD] << 9;
1097 }
1098
1099 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
1100 blkg_iostat_set(&blkg->iostat.cur, &tmp);
1101 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
1102 }
1103}
1104
1105static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
1106{
1107 struct blkg_iostat_set *bis = &blkg->iostat;
1108 u64 rbytes, wbytes, rios, wios, dbytes, dios;
1109 const char *dname;
1110 unsigned seq;
1111 int i;
1112
1113 if (!blkg->online)
1114 return;
1115
1116 dname = blkg_dev_name(blkg);
1117 if (!dname)
1118 return;
1119
1120 seq_printf(s, "%s ", dname);
1121
1122 do {
1123 seq = u64_stats_fetch_begin(&bis->sync);
1124
1125 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
1126 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
1127 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
1128 rios = bis->cur.ios[BLKG_IOSTAT_READ];
1129 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
1130 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
1131 } while (u64_stats_fetch_retry(&bis->sync, seq));
1132
1133 if (rbytes || wbytes || rios || wios) {
1134 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
1135 rbytes, wbytes, rios, wios,
1136 dbytes, dios);
1137 }
1138
1139 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
1140 seq_printf(s, " use_delay=%d delay_nsec=%llu",
1141 atomic_read(&blkg->use_delay),
1142 atomic64_read(&blkg->delay_nsec));
1143 }
1144
1145 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1146 struct blkcg_policy *pol = blkcg_policy[i];
1147
1148 if (!blkg->pd[i] || !pol->pd_stat_fn)
1149 continue;
1150
1151 pol->pd_stat_fn(blkg->pd[i], s);
1152 }
1153
1154 seq_puts(s, "\n");
1155}
1156
1157static int blkcg_print_stat(struct seq_file *sf, void *v)
1158{
1159 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
1160 struct blkcg_gq *blkg;
1161
1162 if (!seq_css(sf)->parent)
1163 blkcg_fill_root_iostats();
1164 else
1165 cgroup_rstat_flush(blkcg->css.cgroup);
1166
1167 rcu_read_lock();
1168 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
1169 spin_lock_irq(&blkg->q->queue_lock);
1170 blkcg_print_one_stat(blkg, sf);
1171 spin_unlock_irq(&blkg->q->queue_lock);
1172 }
1173 rcu_read_unlock();
1174 return 0;
1175}
1176
1177static struct cftype blkcg_files[] = {
1178 {
1179 .name = "stat",
1180 .seq_show = blkcg_print_stat,
1181 },
1182 { } /* terminate */
1183};
1184
1185static struct cftype blkcg_legacy_files[] = {
1186 {
1187 .name = "reset_stats",
1188 .write_u64 = blkcg_reset_stats,
1189 },
1190 { } /* terminate */
1191};
1192
1193#ifdef CONFIG_CGROUP_WRITEBACK
1194struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css)
1195{
1196 return &css_to_blkcg(css)->cgwb_list;
1197}
1198#endif
1199
1200/*
1201 * blkcg destruction is a three-stage process.
1202 *
1203 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
1204 * which offlines writeback. Here we tie the next stage of blkg destruction
1205 * to the completion of writeback associated with the blkcg. This lets us
1206 * avoid punting potentially large amounts of outstanding writeback to root
1207 * while maintaining any ongoing policies. The next stage is triggered when
1208 * the nr_cgwbs count goes to zero.
1209 *
1210 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
1211 * and handles the destruction of blkgs. Here the css reference held by
1212 * the blkg is put back eventually allowing blkcg_css_free() to be called.
1213 * This work may occur in cgwb_release_workfn() on the cgwb_release
1214 * workqueue. Any submitted ios that fail to get the blkg ref will be
1215 * punted to the root_blkg.
1216 *
1217 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
1218 * This finally frees the blkcg.
1219 */
1220
1221/**
1222 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1223 * @blkcg: blkcg of interest
1224 *
1225 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1226 * is nested inside q lock, this function performs reverse double lock dancing.
1227 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1228 * blkcg_css_free to eventually be called.
1229 *
1230 * This is the blkcg counterpart of ioc_release_fn().
1231 */
1232static void blkcg_destroy_blkgs(struct blkcg *blkcg)
1233{
1234 might_sleep();
1235
1236 spin_lock_irq(&blkcg->lock);
1237
1238 while (!hlist_empty(&blkcg->blkg_list)) {
1239 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1240 struct blkcg_gq, blkcg_node);
1241 struct request_queue *q = blkg->q;
1242
1243 if (need_resched() || !spin_trylock(&q->queue_lock)) {
1244 /*
1245 * Given that the system can accumulate a huge number
1246 * of blkgs in pathological cases, check to see if we
1247 * need to rescheduling to avoid softlockup.
1248 */
1249 spin_unlock_irq(&blkcg->lock);
1250 cond_resched();
1251 spin_lock_irq(&blkcg->lock);
1252 continue;
1253 }
1254
1255 blkg_destroy(blkg);
1256 spin_unlock(&q->queue_lock);
1257 }
1258
1259 spin_unlock_irq(&blkcg->lock);
1260}
1261
1262/**
1263 * blkcg_pin_online - pin online state
1264 * @blkcg_css: blkcg of interest
1265 *
1266 * While pinned, a blkcg is kept online. This is primarily used to
1267 * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline
1268 * while an associated cgwb is still active.
1269 */
1270void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css)
1271{
1272 refcount_inc(&css_to_blkcg(blkcg_css)->online_pin);
1273}
1274
1275/**
1276 * blkcg_unpin_online - unpin online state
1277 * @blkcg_css: blkcg of interest
1278 *
1279 * This is primarily used to impedance-match blkg and cgwb lifetimes so
1280 * that blkg doesn't go offline while an associated cgwb is still active.
1281 * When this count goes to zero, all active cgwbs have finished so the
1282 * blkcg can continue destruction by calling blkcg_destroy_blkgs().
1283 */
1284void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css)
1285{
1286 struct blkcg *blkcg = css_to_blkcg(blkcg_css);
1287
1288 do {
1289 if (!refcount_dec_and_test(&blkcg->online_pin))
1290 break;
1291 blkcg_destroy_blkgs(blkcg);
1292 blkcg = blkcg_parent(blkcg);
1293 } while (blkcg);
1294}
1295
1296/**
1297 * blkcg_css_offline - cgroup css_offline callback
1298 * @css: css of interest
1299 *
1300 * This function is called when @css is about to go away. Here the cgwbs are
1301 * offlined first and only once writeback associated with the blkcg has
1302 * finished do we start step 2 (see above).
1303 */
1304static void blkcg_css_offline(struct cgroup_subsys_state *css)
1305{
1306 /* this prevents anyone from attaching or migrating to this blkcg */
1307 wb_blkcg_offline(css);
1308
1309 /* put the base online pin allowing step 2 to be triggered */
1310 blkcg_unpin_online(css);
1311}
1312
1313static void blkcg_css_free(struct cgroup_subsys_state *css)
1314{
1315 struct blkcg *blkcg = css_to_blkcg(css);
1316 int i;
1317
1318 mutex_lock(&blkcg_pol_mutex);
1319
1320 list_del(&blkcg->all_blkcgs_node);
1321
1322 for (i = 0; i < BLKCG_MAX_POLS; i++)
1323 if (blkcg->cpd[i])
1324 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1325
1326 mutex_unlock(&blkcg_pol_mutex);
1327
1328 free_percpu(blkcg->lhead);
1329 kfree(blkcg);
1330}
1331
1332static struct cgroup_subsys_state *
1333blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1334{
1335 struct blkcg *blkcg;
1336 int i;
1337
1338 mutex_lock(&blkcg_pol_mutex);
1339
1340 if (!parent_css) {
1341 blkcg = &blkcg_root;
1342 } else {
1343 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1344 if (!blkcg)
1345 goto unlock;
1346 }
1347
1348 if (init_blkcg_llists(blkcg))
1349 goto free_blkcg;
1350
1351 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1352 struct blkcg_policy *pol = blkcg_policy[i];
1353 struct blkcg_policy_data *cpd;
1354
1355 /*
1356 * If the policy hasn't been attached yet, wait for it
1357 * to be attached before doing anything else. Otherwise,
1358 * check if the policy requires any specific per-cgroup
1359 * data: if it does, allocate and initialize it.
1360 */
1361 if (!pol || !pol->cpd_alloc_fn)
1362 continue;
1363
1364 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1365 if (!cpd)
1366 goto free_pd_blkcg;
1367
1368 blkcg->cpd[i] = cpd;
1369 cpd->blkcg = blkcg;
1370 cpd->plid = i;
1371 }
1372
1373 spin_lock_init(&blkcg->lock);
1374 refcount_set(&blkcg->online_pin, 1);
1375 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1376 INIT_HLIST_HEAD(&blkcg->blkg_list);
1377#ifdef CONFIG_CGROUP_WRITEBACK
1378 INIT_LIST_HEAD(&blkcg->cgwb_list);
1379#endif
1380 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1381
1382 mutex_unlock(&blkcg_pol_mutex);
1383 return &blkcg->css;
1384
1385free_pd_blkcg:
1386 for (i--; i >= 0; i--)
1387 if (blkcg->cpd[i])
1388 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1389 free_percpu(blkcg->lhead);
1390free_blkcg:
1391 if (blkcg != &blkcg_root)
1392 kfree(blkcg);
1393unlock:
1394 mutex_unlock(&blkcg_pol_mutex);
1395 return ERR_PTR(-ENOMEM);
1396}
1397
1398static int blkcg_css_online(struct cgroup_subsys_state *css)
1399{
1400 struct blkcg *parent = blkcg_parent(css_to_blkcg(css));
1401
1402 /*
1403 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1404 * don't go offline while cgwbs are still active on them. Pin the
1405 * parent so that offline always happens towards the root.
1406 */
1407 if (parent)
1408 blkcg_pin_online(&parent->css);
1409 return 0;
1410}
1411
1412int blkcg_init_disk(struct gendisk *disk)
1413{
1414 struct request_queue *q = disk->queue;
1415 struct blkcg_gq *new_blkg, *blkg;
1416 bool preloaded;
1417 int ret;
1418
1419 INIT_LIST_HEAD(&q->blkg_list);
1420 mutex_init(&q->blkcg_mutex);
1421
1422 new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL);
1423 if (!new_blkg)
1424 return -ENOMEM;
1425
1426 preloaded = !radix_tree_preload(GFP_KERNEL);
1427
1428 /* Make sure the root blkg exists. */
1429 /* spin_lock_irq can serve as RCU read-side critical section. */
1430 spin_lock_irq(&q->queue_lock);
1431 blkg = blkg_create(&blkcg_root, disk, new_blkg);
1432 if (IS_ERR(blkg))
1433 goto err_unlock;
1434 q->root_blkg = blkg;
1435 spin_unlock_irq(&q->queue_lock);
1436
1437 if (preloaded)
1438 radix_tree_preload_end();
1439
1440 ret = blk_ioprio_init(disk);
1441 if (ret)
1442 goto err_destroy_all;
1443
1444 ret = blk_throtl_init(disk);
1445 if (ret)
1446 goto err_ioprio_exit;
1447
1448 return 0;
1449
1450err_ioprio_exit:
1451 blk_ioprio_exit(disk);
1452err_destroy_all:
1453 blkg_destroy_all(disk);
1454 return ret;
1455err_unlock:
1456 spin_unlock_irq(&q->queue_lock);
1457 if (preloaded)
1458 radix_tree_preload_end();
1459 return PTR_ERR(blkg);
1460}
1461
1462void blkcg_exit_disk(struct gendisk *disk)
1463{
1464 blkg_destroy_all(disk);
1465 blk_throtl_exit(disk);
1466}
1467
1468static void blkcg_exit(struct task_struct *tsk)
1469{
1470 if (tsk->throttle_disk)
1471 put_disk(tsk->throttle_disk);
1472 tsk->throttle_disk = NULL;
1473}
1474
1475struct cgroup_subsys io_cgrp_subsys = {
1476 .css_alloc = blkcg_css_alloc,
1477 .css_online = blkcg_css_online,
1478 .css_offline = blkcg_css_offline,
1479 .css_free = blkcg_css_free,
1480 .css_rstat_flush = blkcg_rstat_flush,
1481 .dfl_cftypes = blkcg_files,
1482 .legacy_cftypes = blkcg_legacy_files,
1483 .legacy_name = "blkio",
1484 .exit = blkcg_exit,
1485#ifdef CONFIG_MEMCG
1486 /*
1487 * This ensures that, if available, memcg is automatically enabled
1488 * together on the default hierarchy so that the owner cgroup can
1489 * be retrieved from writeback pages.
1490 */
1491 .depends_on = 1 << memory_cgrp_id,
1492#endif
1493};
1494EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1495
1496/**
1497 * blkcg_activate_policy - activate a blkcg policy on a gendisk
1498 * @disk: gendisk of interest
1499 * @pol: blkcg policy to activate
1500 *
1501 * Activate @pol on @disk. Requires %GFP_KERNEL context. @disk goes through
1502 * bypass mode to populate its blkgs with policy_data for @pol.
1503 *
1504 * Activation happens with @disk bypassed, so nobody would be accessing blkgs
1505 * from IO path. Update of each blkg is protected by both queue and blkcg
1506 * locks so that holding either lock and testing blkcg_policy_enabled() is
1507 * always enough for dereferencing policy data.
1508 *
1509 * The caller is responsible for synchronizing [de]activations and policy
1510 * [un]registerations. Returns 0 on success, -errno on failure.
1511 */
1512int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol)
1513{
1514 struct request_queue *q = disk->queue;
1515 struct blkg_policy_data *pd_prealloc = NULL;
1516 struct blkcg_gq *blkg, *pinned_blkg = NULL;
1517 int ret;
1518
1519 if (blkcg_policy_enabled(q, pol))
1520 return 0;
1521
1522 if (queue_is_mq(q))
1523 blk_mq_freeze_queue(q);
1524retry:
1525 spin_lock_irq(&q->queue_lock);
1526
1527 /* blkg_list is pushed at the head, reverse walk to initialize parents first */
1528 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
1529 struct blkg_policy_data *pd;
1530
1531 if (blkg->pd[pol->plid])
1532 continue;
1533
1534 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1535 if (blkg == pinned_blkg) {
1536 pd = pd_prealloc;
1537 pd_prealloc = NULL;
1538 } else {
1539 pd = pol->pd_alloc_fn(disk, blkg->blkcg,
1540 GFP_NOWAIT | __GFP_NOWARN);
1541 }
1542
1543 if (!pd) {
1544 /*
1545 * GFP_NOWAIT failed. Free the existing one and
1546 * prealloc for @blkg w/ GFP_KERNEL.
1547 */
1548 if (pinned_blkg)
1549 blkg_put(pinned_blkg);
1550 blkg_get(blkg);
1551 pinned_blkg = blkg;
1552
1553 spin_unlock_irq(&q->queue_lock);
1554
1555 if (pd_prealloc)
1556 pol->pd_free_fn(pd_prealloc);
1557 pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,
1558 GFP_KERNEL);
1559 if (pd_prealloc)
1560 goto retry;
1561 else
1562 goto enomem;
1563 }
1564
1565 spin_lock(&blkg->blkcg->lock);
1566
1567 pd->blkg = blkg;
1568 pd->plid = pol->plid;
1569 blkg->pd[pol->plid] = pd;
1570
1571 if (pol->pd_init_fn)
1572 pol->pd_init_fn(pd);
1573
1574 if (pol->pd_online_fn)
1575 pol->pd_online_fn(pd);
1576 pd->online = true;
1577
1578 spin_unlock(&blkg->blkcg->lock);
1579 }
1580
1581 __set_bit(pol->plid, q->blkcg_pols);
1582 ret = 0;
1583
1584 spin_unlock_irq(&q->queue_lock);
1585out:
1586 if (queue_is_mq(q))
1587 blk_mq_unfreeze_queue(q);
1588 if (pinned_blkg)
1589 blkg_put(pinned_blkg);
1590 if (pd_prealloc)
1591 pol->pd_free_fn(pd_prealloc);
1592 return ret;
1593
1594enomem:
1595 /* alloc failed, take down everything */
1596 spin_lock_irq(&q->queue_lock);
1597 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1598 struct blkcg *blkcg = blkg->blkcg;
1599 struct blkg_policy_data *pd;
1600
1601 spin_lock(&blkcg->lock);
1602 pd = blkg->pd[pol->plid];
1603 if (pd) {
1604 if (pd->online && pol->pd_offline_fn)
1605 pol->pd_offline_fn(pd);
1606 pd->online = false;
1607 pol->pd_free_fn(pd);
1608 blkg->pd[pol->plid] = NULL;
1609 }
1610 spin_unlock(&blkcg->lock);
1611 }
1612 spin_unlock_irq(&q->queue_lock);
1613 ret = -ENOMEM;
1614 goto out;
1615}
1616EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1617
1618/**
1619 * blkcg_deactivate_policy - deactivate a blkcg policy on a gendisk
1620 * @disk: gendisk of interest
1621 * @pol: blkcg policy to deactivate
1622 *
1623 * Deactivate @pol on @disk. Follows the same synchronization rules as
1624 * blkcg_activate_policy().
1625 */
1626void blkcg_deactivate_policy(struct gendisk *disk,
1627 const struct blkcg_policy *pol)
1628{
1629 struct request_queue *q = disk->queue;
1630 struct blkcg_gq *blkg;
1631
1632 if (!blkcg_policy_enabled(q, pol))
1633 return;
1634
1635 if (queue_is_mq(q))
1636 blk_mq_freeze_queue(q);
1637
1638 mutex_lock(&q->blkcg_mutex);
1639 spin_lock_irq(&q->queue_lock);
1640
1641 __clear_bit(pol->plid, q->blkcg_pols);
1642
1643 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1644 struct blkcg *blkcg = blkg->blkcg;
1645
1646 spin_lock(&blkcg->lock);
1647 if (blkg->pd[pol->plid]) {
1648 if (blkg->pd[pol->plid]->online && pol->pd_offline_fn)
1649 pol->pd_offline_fn(blkg->pd[pol->plid]);
1650 pol->pd_free_fn(blkg->pd[pol->plid]);
1651 blkg->pd[pol->plid] = NULL;
1652 }
1653 spin_unlock(&blkcg->lock);
1654 }
1655
1656 spin_unlock_irq(&q->queue_lock);
1657 mutex_unlock(&q->blkcg_mutex);
1658
1659 if (queue_is_mq(q))
1660 blk_mq_unfreeze_queue(q);
1661}
1662EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1663
1664static void blkcg_free_all_cpd(struct blkcg_policy *pol)
1665{
1666 struct blkcg *blkcg;
1667
1668 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1669 if (blkcg->cpd[pol->plid]) {
1670 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1671 blkcg->cpd[pol->plid] = NULL;
1672 }
1673 }
1674}
1675
1676/**
1677 * blkcg_policy_register - register a blkcg policy
1678 * @pol: blkcg policy to register
1679 *
1680 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1681 * successful registration. Returns 0 on success and -errno on failure.
1682 */
1683int blkcg_policy_register(struct blkcg_policy *pol)
1684{
1685 struct blkcg *blkcg;
1686 int i, ret;
1687
1688 mutex_lock(&blkcg_pol_register_mutex);
1689 mutex_lock(&blkcg_pol_mutex);
1690
1691 /* find an empty slot */
1692 ret = -ENOSPC;
1693 for (i = 0; i < BLKCG_MAX_POLS; i++)
1694 if (!blkcg_policy[i])
1695 break;
1696 if (i >= BLKCG_MAX_POLS) {
1697 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1698 goto err_unlock;
1699 }
1700
1701 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1702 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1703 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1704 goto err_unlock;
1705
1706 /* register @pol */
1707 pol->plid = i;
1708 blkcg_policy[pol->plid] = pol;
1709
1710 /* allocate and install cpd's */
1711 if (pol->cpd_alloc_fn) {
1712 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1713 struct blkcg_policy_data *cpd;
1714
1715 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1716 if (!cpd)
1717 goto err_free_cpds;
1718
1719 blkcg->cpd[pol->plid] = cpd;
1720 cpd->blkcg = blkcg;
1721 cpd->plid = pol->plid;
1722 }
1723 }
1724
1725 mutex_unlock(&blkcg_pol_mutex);
1726
1727 /* everything is in place, add intf files for the new policy */
1728 if (pol->dfl_cftypes)
1729 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1730 pol->dfl_cftypes));
1731 if (pol->legacy_cftypes)
1732 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1733 pol->legacy_cftypes));
1734 mutex_unlock(&blkcg_pol_register_mutex);
1735 return 0;
1736
1737err_free_cpds:
1738 if (pol->cpd_free_fn)
1739 blkcg_free_all_cpd(pol);
1740
1741 blkcg_policy[pol->plid] = NULL;
1742err_unlock:
1743 mutex_unlock(&blkcg_pol_mutex);
1744 mutex_unlock(&blkcg_pol_register_mutex);
1745 return ret;
1746}
1747EXPORT_SYMBOL_GPL(blkcg_policy_register);
1748
1749/**
1750 * blkcg_policy_unregister - unregister a blkcg policy
1751 * @pol: blkcg policy to unregister
1752 *
1753 * Undo blkcg_policy_register(@pol). Might sleep.
1754 */
1755void blkcg_policy_unregister(struct blkcg_policy *pol)
1756{
1757 mutex_lock(&blkcg_pol_register_mutex);
1758
1759 if (WARN_ON(blkcg_policy[pol->plid] != pol))
1760 goto out_unlock;
1761
1762 /* kill the intf files first */
1763 if (pol->dfl_cftypes)
1764 cgroup_rm_cftypes(pol->dfl_cftypes);
1765 if (pol->legacy_cftypes)
1766 cgroup_rm_cftypes(pol->legacy_cftypes);
1767
1768 /* remove cpds and unregister */
1769 mutex_lock(&blkcg_pol_mutex);
1770
1771 if (pol->cpd_free_fn)
1772 blkcg_free_all_cpd(pol);
1773
1774 blkcg_policy[pol->plid] = NULL;
1775
1776 mutex_unlock(&blkcg_pol_mutex);
1777out_unlock:
1778 mutex_unlock(&blkcg_pol_register_mutex);
1779}
1780EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1781
1782/*
1783 * Scale the accumulated delay based on how long it has been since we updated
1784 * the delay. We only call this when we are adding delay, in case it's been a
1785 * while since we added delay, and when we are checking to see if we need to
1786 * delay a task, to account for any delays that may have occurred.
1787 */
1788static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1789{
1790 u64 old = atomic64_read(&blkg->delay_start);
1791
1792 /* negative use_delay means no scaling, see blkcg_set_delay() */
1793 if (atomic_read(&blkg->use_delay) < 0)
1794 return;
1795
1796 /*
1797 * We only want to scale down every second. The idea here is that we
1798 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1799 * time window. We only want to throttle tasks for recent delay that
1800 * has occurred, in 1 second time windows since that's the maximum
1801 * things can be throttled. We save the current delay window in
1802 * blkg->last_delay so we know what amount is still left to be charged
1803 * to the blkg from this point onward. blkg->last_use keeps track of
1804 * the use_delay counter. The idea is if we're unthrottling the blkg we
1805 * are ok with whatever is happening now, and we can take away more of
1806 * the accumulated delay as we've already throttled enough that
1807 * everybody is happy with their IO latencies.
1808 */
1809 if (time_before64(old + NSEC_PER_SEC, now) &&
1810 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {
1811 u64 cur = atomic64_read(&blkg->delay_nsec);
1812 u64 sub = min_t(u64, blkg->last_delay, now - old);
1813 int cur_use = atomic_read(&blkg->use_delay);
1814
1815 /*
1816 * We've been unthrottled, subtract a larger chunk of our
1817 * accumulated delay.
1818 */
1819 if (cur_use < blkg->last_use)
1820 sub = max_t(u64, sub, blkg->last_delay >> 1);
1821
1822 /*
1823 * This shouldn't happen, but handle it anyway. Our delay_nsec
1824 * should only ever be growing except here where we subtract out
1825 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1826 * rather not end up with negative numbers.
1827 */
1828 if (unlikely(cur < sub)) {
1829 atomic64_set(&blkg->delay_nsec, 0);
1830 blkg->last_delay = 0;
1831 } else {
1832 atomic64_sub(sub, &blkg->delay_nsec);
1833 blkg->last_delay = cur - sub;
1834 }
1835 blkg->last_use = cur_use;
1836 }
1837}
1838
1839/*
1840 * This is called when we want to actually walk up the hierarchy and check to
1841 * see if we need to throttle, and then actually throttle if there is some
1842 * accumulated delay. This should only be called upon return to user space so
1843 * we're not holding some lock that would induce a priority inversion.
1844 */
1845static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1846{
1847 unsigned long pflags;
1848 bool clamp;
1849 u64 now = ktime_to_ns(ktime_get());
1850 u64 exp;
1851 u64 delay_nsec = 0;
1852 int tok;
1853
1854 while (blkg->parent) {
1855 int use_delay = atomic_read(&blkg->use_delay);
1856
1857 if (use_delay) {
1858 u64 this_delay;
1859
1860 blkcg_scale_delay(blkg, now);
1861 this_delay = atomic64_read(&blkg->delay_nsec);
1862 if (this_delay > delay_nsec) {
1863 delay_nsec = this_delay;
1864 clamp = use_delay > 0;
1865 }
1866 }
1867 blkg = blkg->parent;
1868 }
1869
1870 if (!delay_nsec)
1871 return;
1872
1873 /*
1874 * Let's not sleep for all eternity if we've amassed a huge delay.
1875 * Swapping or metadata IO can accumulate 10's of seconds worth of
1876 * delay, and we want userspace to be able to do _something_ so cap the
1877 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1878 * tasks will be delayed for 0.25 second for every syscall. If
1879 * blkcg_set_delay() was used as indicated by negative use_delay, the
1880 * caller is responsible for regulating the range.
1881 */
1882 if (clamp)
1883 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1884
1885 if (use_memdelay)
1886 psi_memstall_enter(&pflags);
1887
1888 exp = ktime_add_ns(now, delay_nsec);
1889 tok = io_schedule_prepare();
1890 do {
1891 __set_current_state(TASK_KILLABLE);
1892 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1893 break;
1894 } while (!fatal_signal_pending(current));
1895 io_schedule_finish(tok);
1896
1897 if (use_memdelay)
1898 psi_memstall_leave(&pflags);
1899}
1900
1901/**
1902 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1903 *
1904 * This is only called if we've been marked with set_notify_resume(). Obviously
1905 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1906 * check to see if current->throttle_disk is set and if not this doesn't do
1907 * anything. This should only ever be called by the resume code, it's not meant
1908 * to be called by people willy-nilly as it will actually do the work to
1909 * throttle the task if it is setup for throttling.
1910 */
1911void blkcg_maybe_throttle_current(void)
1912{
1913 struct gendisk *disk = current->throttle_disk;
1914 struct blkcg *blkcg;
1915 struct blkcg_gq *blkg;
1916 bool use_memdelay = current->use_memdelay;
1917
1918 if (!disk)
1919 return;
1920
1921 current->throttle_disk = NULL;
1922 current->use_memdelay = false;
1923
1924 rcu_read_lock();
1925 blkcg = css_to_blkcg(blkcg_css());
1926 if (!blkcg)
1927 goto out;
1928 blkg = blkg_lookup(blkcg, disk->queue);
1929 if (!blkg)
1930 goto out;
1931 if (!blkg_tryget(blkg))
1932 goto out;
1933 rcu_read_unlock();
1934
1935 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1936 blkg_put(blkg);
1937 put_disk(disk);
1938 return;
1939out:
1940 rcu_read_unlock();
1941}
1942
1943/**
1944 * blkcg_schedule_throttle - this task needs to check for throttling
1945 * @disk: disk to throttle
1946 * @use_memdelay: do we charge this to memory delay for PSI
1947 *
1948 * This is called by the IO controller when we know there's delay accumulated
1949 * for the blkg for this task. We do not pass the blkg because there are places
1950 * we call this that may not have that information, the swapping code for
1951 * instance will only have a block_device at that point. This set's the
1952 * notify_resume for the task to check and see if it requires throttling before
1953 * returning to user space.
1954 *
1955 * We will only schedule once per syscall. You can call this over and over
1956 * again and it will only do the check once upon return to user space, and only
1957 * throttle once. If the task needs to be throttled again it'll need to be
1958 * re-set at the next time we see the task.
1959 */
1960void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay)
1961{
1962 if (unlikely(current->flags & PF_KTHREAD))
1963 return;
1964
1965 if (current->throttle_disk != disk) {
1966 if (test_bit(GD_DEAD, &disk->state))
1967 return;
1968 get_device(disk_to_dev(disk));
1969
1970 if (current->throttle_disk)
1971 put_disk(current->throttle_disk);
1972 current->throttle_disk = disk;
1973 }
1974
1975 if (use_memdelay)
1976 current->use_memdelay = use_memdelay;
1977 set_notify_resume(current);
1978}
1979
1980/**
1981 * blkcg_add_delay - add delay to this blkg
1982 * @blkg: blkg of interest
1983 * @now: the current time in nanoseconds
1984 * @delta: how many nanoseconds of delay to add
1985 *
1986 * Charge @delta to the blkg's current delay accumulation. This is used to
1987 * throttle tasks if an IO controller thinks we need more throttling.
1988 */
1989void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1990{
1991 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
1992 return;
1993 blkcg_scale_delay(blkg, now);
1994 atomic64_add(delta, &blkg->delay_nsec);
1995}
1996
1997/**
1998 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
1999 * @bio: target bio
2000 * @css: target css
2001 *
2002 * As the failure mode here is to walk up the blkg tree, this ensure that the
2003 * blkg->parent pointers are always valid. This returns the blkg that it ended
2004 * up taking a reference on or %NULL if no reference was taken.
2005 */
2006static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
2007 struct cgroup_subsys_state *css)
2008{
2009 struct blkcg_gq *blkg, *ret_blkg = NULL;
2010
2011 rcu_read_lock();
2012 blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk);
2013 while (blkg) {
2014 if (blkg_tryget(blkg)) {
2015 ret_blkg = blkg;
2016 break;
2017 }
2018 blkg = blkg->parent;
2019 }
2020 rcu_read_unlock();
2021
2022 return ret_blkg;
2023}
2024
2025/**
2026 * bio_associate_blkg_from_css - associate a bio with a specified css
2027 * @bio: target bio
2028 * @css: target css
2029 *
2030 * Associate @bio with the blkg found by combining the css's blkg and the
2031 * request_queue of the @bio. An association failure is handled by walking up
2032 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
2033 * and q->root_blkg. This situation only happens when a cgroup is dying and
2034 * then the remaining bios will spill to the closest alive blkg.
2035 *
2036 * A reference will be taken on the blkg and will be released when @bio is
2037 * freed.
2038 */
2039void bio_associate_blkg_from_css(struct bio *bio,
2040 struct cgroup_subsys_state *css)
2041{
2042 if (bio->bi_blkg)
2043 blkg_put(bio->bi_blkg);
2044
2045 if (css && css->parent) {
2046 bio->bi_blkg = blkg_tryget_closest(bio, css);
2047 } else {
2048 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
2049 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
2050 }
2051}
2052EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
2053
2054/**
2055 * bio_associate_blkg - associate a bio with a blkg
2056 * @bio: target bio
2057 *
2058 * Associate @bio with the blkg found from the bio's css and request_queue.
2059 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
2060 * already associated, the css is reused and association redone as the
2061 * request_queue may have changed.
2062 */
2063void bio_associate_blkg(struct bio *bio)
2064{
2065 struct cgroup_subsys_state *css;
2066
2067 if (blk_op_is_passthrough(bio->bi_opf))
2068 return;
2069
2070 rcu_read_lock();
2071
2072 if (bio->bi_blkg)
2073 css = bio_blkcg_css(bio);
2074 else
2075 css = blkcg_css();
2076
2077 bio_associate_blkg_from_css(bio, css);
2078
2079 rcu_read_unlock();
2080}
2081EXPORT_SYMBOL_GPL(bio_associate_blkg);
2082
2083/**
2084 * bio_clone_blkg_association - clone blkg association from src to dst bio
2085 * @dst: destination bio
2086 * @src: source bio
2087 */
2088void bio_clone_blkg_association(struct bio *dst, struct bio *src)
2089{
2090 if (src->bi_blkg)
2091 bio_associate_blkg_from_css(dst, bio_blkcg_css(src));
2092}
2093EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
2094
2095static int blk_cgroup_io_type(struct bio *bio)
2096{
2097 if (op_is_discard(bio->bi_opf))
2098 return BLKG_IOSTAT_DISCARD;
2099 if (op_is_write(bio->bi_opf))
2100 return BLKG_IOSTAT_WRITE;
2101 return BLKG_IOSTAT_READ;
2102}
2103
2104void blk_cgroup_bio_start(struct bio *bio)
2105{
2106 struct blkcg *blkcg = bio->bi_blkg->blkcg;
2107 int rwd = blk_cgroup_io_type(bio), cpu;
2108 struct blkg_iostat_set *bis;
2109 unsigned long flags;
2110
2111 if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
2112 return;
2113
2114 /* Root-level stats are sourced from system-wide IO stats */
2115 if (!cgroup_parent(blkcg->css.cgroup))
2116 return;
2117
2118 cpu = get_cpu();
2119 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
2120 flags = u64_stats_update_begin_irqsave(&bis->sync);
2121
2122 /*
2123 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
2124 * bio and we would have already accounted for the size of the bio.
2125 */
2126 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
2127 bio_set_flag(bio, BIO_CGROUP_ACCT);
2128 bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
2129 }
2130 bis->cur.ios[rwd]++;
2131
2132 /*
2133 * If the iostat_cpu isn't in a lockless list, put it into the
2134 * list to indicate that a stat update is pending.
2135 */
2136 if (!READ_ONCE(bis->lqueued)) {
2137 struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
2138
2139 llist_add(&bis->lnode, lhead);
2140 WRITE_ONCE(bis->lqueued, true);
2141 }
2142
2143 u64_stats_update_end_irqrestore(&bis->sync, flags);
2144 cgroup_rstat_updated(blkcg->css.cgroup, cpu);
2145 put_cpu();
2146}
2147
2148bool blk_cgroup_congested(void)
2149{
2150 struct cgroup_subsys_state *css;
2151 bool ret = false;
2152
2153 rcu_read_lock();
2154 for (css = blkcg_css(); css; css = css->parent) {
2155 if (atomic_read(&css->cgroup->congestion_count)) {
2156 ret = true;
2157 break;
2158 }
2159 }
2160 rcu_read_unlock();
2161 return ret;
2162}
2163
2164module_param(blkcg_debug_stats, bool, 0644);
2165MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");