Loading...
1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
4 * Do not use in new code.
5 */
6#ifndef _BLK_CGROUP_RWSTAT_H
7#define _BLK_CGROUP_RWSTAT_H
8
9#include "blk-cgroup.h"
10
11enum blkg_rwstat_type {
12 BLKG_RWSTAT_READ,
13 BLKG_RWSTAT_WRITE,
14 BLKG_RWSTAT_SYNC,
15 BLKG_RWSTAT_ASYNC,
16 BLKG_RWSTAT_DISCARD,
17
18 BLKG_RWSTAT_NR,
19 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
20};
21
22/*
23 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
24 * recursive. Used to carry stats of dead children.
25 */
26struct blkg_rwstat {
27 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
28 atomic64_t aux_cnt[BLKG_RWSTAT_NR];
29};
30
31struct blkg_rwstat_sample {
32 u64 cnt[BLKG_RWSTAT_NR];
33};
34
35static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
36 unsigned int idx)
37{
38 return atomic64_read(&rwstat->aux_cnt[idx]) +
39 percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
40}
41
42int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
43void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
44u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
45 const struct blkg_rwstat_sample *rwstat);
46u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
47 int off);
48void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
49 int off, struct blkg_rwstat_sample *sum);
50
51
52/**
53 * blkg_rwstat_add - add a value to a blkg_rwstat
54 * @rwstat: target blkg_rwstat
55 * @op: REQ_OP and flags
56 * @val: value to add
57 *
58 * Add @val to @rwstat. The counters are chosen according to @rw. The
59 * caller is responsible for synchronizing calls to this function.
60 */
61static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
62 blk_opf_t opf, uint64_t val)
63{
64 struct percpu_counter *cnt;
65
66 if (op_is_discard(opf))
67 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
68 else if (op_is_write(opf))
69 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
70 else
71 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
72
73 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
74
75 if (op_is_sync(opf))
76 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
77 else
78 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
79
80 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
81}
82
83/**
84 * blkg_rwstat_read - read the current values of a blkg_rwstat
85 * @rwstat: blkg_rwstat to read
86 *
87 * Read the current snapshot of @rwstat and return it in the aux counts.
88 */
89static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
90 struct blkg_rwstat_sample *result)
91{
92 int i;
93
94 for (i = 0; i < BLKG_RWSTAT_NR; i++)
95 result->cnt[i] =
96 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
97}
98
99/**
100 * blkg_rwstat_total - read the total count of a blkg_rwstat
101 * @rwstat: blkg_rwstat to read
102 *
103 * Return the total count of @rwstat regardless of the IO direction. This
104 * function can be called without synchronization and takes care of u64
105 * atomicity.
106 */
107static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
108{
109 struct blkg_rwstat_sample tmp = { };
110
111 blkg_rwstat_read(rwstat, &tmp);
112 return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
113}
114
115/**
116 * blkg_rwstat_reset - reset a blkg_rwstat
117 * @rwstat: blkg_rwstat to reset
118 */
119static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
120{
121 int i;
122
123 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
124 percpu_counter_set(&rwstat->cpu_cnt[i], 0);
125 atomic64_set(&rwstat->aux_cnt[i], 0);
126 }
127}
128
129/**
130 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
131 * @to: the destination blkg_rwstat
132 * @from: the source
133 *
134 * Add @from's count including the aux one to @to's aux count.
135 */
136static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
137 struct blkg_rwstat *from)
138{
139 u64 sum[BLKG_RWSTAT_NR];
140 int i;
141
142 for (i = 0; i < BLKG_RWSTAT_NR; i++)
143 sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
144
145 for (i = 0; i < BLKG_RWSTAT_NR; i++)
146 atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
147 &to->aux_cnt[i]);
148}
149#endif /* _BLK_CGROUP_RWSTAT_H */
1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
4 * Do not use in new code.
5 */
6#ifndef _BLK_CGROUP_RWSTAT_H
7#define _BLK_CGROUP_RWSTAT_H
8
9#include "blk-cgroup.h"
10
11enum blkg_rwstat_type {
12 BLKG_RWSTAT_READ,
13 BLKG_RWSTAT_WRITE,
14 BLKG_RWSTAT_SYNC,
15 BLKG_RWSTAT_ASYNC,
16 BLKG_RWSTAT_DISCARD,
17
18 BLKG_RWSTAT_NR,
19 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
20};
21
22/*
23 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
24 * recursive. Used to carry stats of dead children.
25 */
26struct blkg_rwstat {
27 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
28 atomic64_t aux_cnt[BLKG_RWSTAT_NR];
29};
30
31struct blkg_rwstat_sample {
32 u64 cnt[BLKG_RWSTAT_NR];
33};
34
35static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
36 unsigned int idx)
37{
38 return atomic64_read(&rwstat->aux_cnt[idx]) +
39 percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
40}
41
42int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
43void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
44u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
45 const struct blkg_rwstat_sample *rwstat);
46u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
47 int off);
48void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
49 int off, struct blkg_rwstat_sample *sum);
50
51
52/**
53 * blkg_rwstat_add - add a value to a blkg_rwstat
54 * @rwstat: target blkg_rwstat
55 * @op: REQ_OP and flags
56 * @val: value to add
57 *
58 * Add @val to @rwstat. The counters are chosen according to @rw. The
59 * caller is responsible for synchronizing calls to this function.
60 */
61static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
62 blk_opf_t opf, uint64_t val)
63{
64 struct percpu_counter *cnt;
65
66 if (op_is_discard(opf))
67 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
68 else if (op_is_write(opf))
69 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
70 else
71 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
72
73 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
74
75 if (op_is_sync(opf))
76 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
77 else
78 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
79
80 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
81}
82
83/**
84 * blkg_rwstat_read - read the current values of a blkg_rwstat
85 * @rwstat: blkg_rwstat to read
86 *
87 * Read the current snapshot of @rwstat and return it in the aux counts.
88 */
89static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
90 struct blkg_rwstat_sample *result)
91{
92 int i;
93
94 for (i = 0; i < BLKG_RWSTAT_NR; i++)
95 result->cnt[i] =
96 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
97}
98
99/**
100 * blkg_rwstat_total - read the total count of a blkg_rwstat
101 * @rwstat: blkg_rwstat to read
102 *
103 * Return the total count of @rwstat regardless of the IO direction. This
104 * function can be called without synchronization and takes care of u64
105 * atomicity.
106 */
107static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
108{
109 struct blkg_rwstat_sample tmp = { };
110
111 blkg_rwstat_read(rwstat, &tmp);
112 return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
113}
114
115/**
116 * blkg_rwstat_reset - reset a blkg_rwstat
117 * @rwstat: blkg_rwstat to reset
118 */
119static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
120{
121 int i;
122
123 for (i = 0; i < BLKG_RWSTAT_NR; i++) {
124 percpu_counter_set(&rwstat->cpu_cnt[i], 0);
125 atomic64_set(&rwstat->aux_cnt[i], 0);
126 }
127}
128
129/**
130 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
131 * @to: the destination blkg_rwstat
132 * @from: the source
133 *
134 * Add @from's count including the aux one to @to's aux count.
135 */
136static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
137 struct blkg_rwstat *from)
138{
139 u64 sum[BLKG_RWSTAT_NR];
140 int i;
141
142 for (i = 0; i < BLKG_RWSTAT_NR; i++)
143 sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
144
145 for (i = 0; i < BLKG_RWSTAT_NR; i++)
146 atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
147 &to->aux_cnt[i]);
148}
149#endif /* _BLK_CGROUP_RWSTAT_H */