Loading...
1#ifdef CONFIG_SMP
2#include "sched-pelt.h"
3
4int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9
10#ifdef CONFIG_SCHED_THERMAL_PRESSURE
11int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
12
13static inline u64 thermal_load_avg(struct rq *rq)
14{
15 return READ_ONCE(rq->avg_thermal.load_avg);
16}
17#else
18static inline int
19update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
20{
21 return 0;
22}
23
24static inline u64 thermal_load_avg(struct rq *rq)
25{
26 return 0;
27}
28#endif
29
30#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31int update_irq_load_avg(struct rq *rq, u64 running);
32#else
33static inline int
34update_irq_load_avg(struct rq *rq, u64 running)
35{
36 return 0;
37}
38#endif
39
40#define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
41
42static inline u32 get_pelt_divider(struct sched_avg *avg)
43{
44 return PELT_MIN_DIVIDER + avg->period_contrib;
45}
46
47static inline void cfs_se_util_change(struct sched_avg *avg)
48{
49 unsigned int enqueued;
50
51 if (!sched_feat(UTIL_EST))
52 return;
53
54 /* Avoid store if the flag has been already reset */
55 enqueued = avg->util_est.enqueued;
56 if (!(enqueued & UTIL_AVG_UNCHANGED))
57 return;
58
59 /* Reset flag to report util_avg has been updated */
60 enqueued &= ~UTIL_AVG_UNCHANGED;
61 WRITE_ONCE(avg->util_est.enqueued, enqueued);
62}
63
64static inline u64 rq_clock_pelt(struct rq *rq)
65{
66 lockdep_assert_rq_held(rq);
67 assert_clock_updated(rq);
68
69 return rq->clock_pelt - rq->lost_idle_time;
70}
71
72/* The rq is idle, we can sync to clock_task */
73static inline void _update_idle_rq_clock_pelt(struct rq *rq)
74{
75 rq->clock_pelt = rq_clock_task(rq);
76
77 u64_u32_store(rq->clock_idle, rq_clock(rq));
78 /* Paired with smp_rmb in migrate_se_pelt_lag() */
79 smp_wmb();
80 u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
81}
82
83/*
84 * The clock_pelt scales the time to reflect the effective amount of
85 * computation done during the running delta time but then sync back to
86 * clock_task when rq is idle.
87 *
88 *
89 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
90 * @ max capacity ------******---------------******---------------
91 * @ half capacity ------************---------************---------
92 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
93 *
94 */
95static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
96{
97 if (unlikely(is_idle_task(rq->curr))) {
98 _update_idle_rq_clock_pelt(rq);
99 return;
100 }
101
102 /*
103 * When a rq runs at a lower compute capacity, it will need
104 * more time to do the same amount of work than at max
105 * capacity. In order to be invariant, we scale the delta to
106 * reflect how much work has been really done.
107 * Running longer results in stealing idle time that will
108 * disturb the load signal compared to max capacity. This
109 * stolen idle time will be automatically reflected when the
110 * rq will be idle and the clock will be synced with
111 * rq_clock_task.
112 */
113
114 /*
115 * Scale the elapsed time to reflect the real amount of
116 * computation
117 */
118 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
119 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
120
121 rq->clock_pelt += delta;
122}
123
124/*
125 * When rq becomes idle, we have to check if it has lost idle time
126 * because it was fully busy. A rq is fully used when the /Sum util_sum
127 * is greater or equal to:
128 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
129 * For optimization and computing rounding purpose, we don't take into account
130 * the position in the current window (period_contrib) and we use the higher
131 * bound of util_sum to decide.
132 */
133static inline void update_idle_rq_clock_pelt(struct rq *rq)
134{
135 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
136 u32 util_sum = rq->cfs.avg.util_sum;
137 util_sum += rq->avg_rt.util_sum;
138 util_sum += rq->avg_dl.util_sum;
139
140 /*
141 * Reflecting stolen time makes sense only if the idle
142 * phase would be present at max capacity. As soon as the
143 * utilization of a rq has reached the maximum value, it is
144 * considered as an always running rq without idle time to
145 * steal. This potential idle time is considered as lost in
146 * this case. We keep track of this lost idle time compare to
147 * rq's clock_task.
148 */
149 if (util_sum >= divider)
150 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
151
152 _update_idle_rq_clock_pelt(rq);
153}
154
155#ifdef CONFIG_CFS_BANDWIDTH
156static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
157{
158 u64 throttled;
159
160 if (unlikely(cfs_rq->throttle_count))
161 throttled = U64_MAX;
162 else
163 throttled = cfs_rq->throttled_clock_pelt_time;
164
165 u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
166}
167
168/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
169static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
170{
171 if (unlikely(cfs_rq->throttle_count))
172 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
173
174 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
175}
176#else
177static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
178static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
179{
180 return rq_clock_pelt(rq_of(cfs_rq));
181}
182#endif
183
184#else
185
186static inline int
187update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
188{
189 return 0;
190}
191
192static inline int
193update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
194{
195 return 0;
196}
197
198static inline int
199update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
200{
201 return 0;
202}
203
204static inline int
205update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
206{
207 return 0;
208}
209
210static inline u64 thermal_load_avg(struct rq *rq)
211{
212 return 0;
213}
214
215static inline int
216update_irq_load_avg(struct rq *rq, u64 running)
217{
218 return 0;
219}
220
221static inline u64 rq_clock_pelt(struct rq *rq)
222{
223 return rq_clock_task(rq);
224}
225
226static inline void
227update_rq_clock_pelt(struct rq *rq, s64 delta) { }
228
229static inline void
230update_idle_rq_clock_pelt(struct rq *rq) { }
231
232static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
233#endif
234
235
1#ifdef CONFIG_SMP
2#include "sched-pelt.h"
3
4int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
5int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
6int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
7int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
9
10#ifdef CONFIG_SCHED_THERMAL_PRESSURE
11int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
12
13static inline u64 thermal_load_avg(struct rq *rq)
14{
15 return READ_ONCE(rq->avg_thermal.load_avg);
16}
17#else
18static inline int
19update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
20{
21 return 0;
22}
23
24static inline u64 thermal_load_avg(struct rq *rq)
25{
26 return 0;
27}
28#endif
29
30#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
31int update_irq_load_avg(struct rq *rq, u64 running);
32#else
33static inline int
34update_irq_load_avg(struct rq *rq, u64 running)
35{
36 return 0;
37}
38#endif
39
40static inline u32 get_pelt_divider(struct sched_avg *avg)
41{
42 return LOAD_AVG_MAX - 1024 + avg->period_contrib;
43}
44
45/*
46 * When a task is dequeued, its estimated utilization should not be update if
47 * its util_avg has not been updated at least once.
48 * This flag is used to synchronize util_avg updates with util_est updates.
49 * We map this information into the LSB bit of the utilization saved at
50 * dequeue time (i.e. util_est.dequeued).
51 */
52#define UTIL_AVG_UNCHANGED 0x1
53
54static inline void cfs_se_util_change(struct sched_avg *avg)
55{
56 unsigned int enqueued;
57
58 if (!sched_feat(UTIL_EST))
59 return;
60
61 /* Avoid store if the flag has been already set */
62 enqueued = avg->util_est.enqueued;
63 if (!(enqueued & UTIL_AVG_UNCHANGED))
64 return;
65
66 /* Reset flag to report util_avg has been updated */
67 enqueued &= ~UTIL_AVG_UNCHANGED;
68 WRITE_ONCE(avg->util_est.enqueued, enqueued);
69}
70
71/*
72 * The clock_pelt scales the time to reflect the effective amount of
73 * computation done during the running delta time but then sync back to
74 * clock_task when rq is idle.
75 *
76 *
77 * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
78 * @ max capacity ------******---------------******---------------
79 * @ half capacity ------************---------************---------
80 * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16
81 *
82 */
83static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
84{
85 if (unlikely(is_idle_task(rq->curr))) {
86 /* The rq is idle, we can sync to clock_task */
87 rq->clock_pelt = rq_clock_task(rq);
88 return;
89 }
90
91 /*
92 * When a rq runs at a lower compute capacity, it will need
93 * more time to do the same amount of work than at max
94 * capacity. In order to be invariant, we scale the delta to
95 * reflect how much work has been really done.
96 * Running longer results in stealing idle time that will
97 * disturb the load signal compared to max capacity. This
98 * stolen idle time will be automatically reflected when the
99 * rq will be idle and the clock will be synced with
100 * rq_clock_task.
101 */
102
103 /*
104 * Scale the elapsed time to reflect the real amount of
105 * computation
106 */
107 delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
108 delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
109
110 rq->clock_pelt += delta;
111}
112
113/*
114 * When rq becomes idle, we have to check if it has lost idle time
115 * because it was fully busy. A rq is fully used when the /Sum util_sum
116 * is greater or equal to:
117 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
118 * For optimization and computing rounding purpose, we don't take into account
119 * the position in the current window (period_contrib) and we use the higher
120 * bound of util_sum to decide.
121 */
122static inline void update_idle_rq_clock_pelt(struct rq *rq)
123{
124 u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
125 u32 util_sum = rq->cfs.avg.util_sum;
126 util_sum += rq->avg_rt.util_sum;
127 util_sum += rq->avg_dl.util_sum;
128
129 /*
130 * Reflecting stolen time makes sense only if the idle
131 * phase would be present at max capacity. As soon as the
132 * utilization of a rq has reached the maximum value, it is
133 * considered as an always runnig rq without idle time to
134 * steal. This potential idle time is considered as lost in
135 * this case. We keep track of this lost idle time compare to
136 * rq's clock_task.
137 */
138 if (util_sum >= divider)
139 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
140}
141
142static inline u64 rq_clock_pelt(struct rq *rq)
143{
144 lockdep_assert_held(&rq->lock);
145 assert_clock_updated(rq);
146
147 return rq->clock_pelt - rq->lost_idle_time;
148}
149
150#ifdef CONFIG_CFS_BANDWIDTH
151/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
152static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
153{
154 if (unlikely(cfs_rq->throttle_count))
155 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
156
157 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
158}
159#else
160static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
161{
162 return rq_clock_pelt(rq_of(cfs_rq));
163}
164#endif
165
166#else
167
168static inline int
169update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
170{
171 return 0;
172}
173
174static inline int
175update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
176{
177 return 0;
178}
179
180static inline int
181update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
182{
183 return 0;
184}
185
186static inline int
187update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
188{
189 return 0;
190}
191
192static inline u64 thermal_load_avg(struct rq *rq)
193{
194 return 0;
195}
196
197static inline int
198update_irq_load_avg(struct rq *rq, u64 running)
199{
200 return 0;
201}
202
203static inline u64 rq_clock_pelt(struct rq *rq)
204{
205 return rq_clock_task(rq);
206}
207
208static inline void
209update_rq_clock_pelt(struct rq *rq, s64 delta) { }
210
211static inline void
212update_idle_rq_clock_pelt(struct rq *rq) { }
213
214#endif
215
216