1#ifdef CONFIG_SMP
  2#include "sched-pelt.h"
  3
  4int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
  5int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
  6int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
  7int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
  8int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
  9
 10#ifdef CONFIG_SCHED_THERMAL_PRESSURE
 11int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
 12
 13static inline u64 thermal_load_avg(struct rq *rq)
 14{
 15	return READ_ONCE(rq->avg_thermal.load_avg);
 16}
 17#else
 18static inline int
 19update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
 20{
 21	return 0;
 22}
 23
 24static inline u64 thermal_load_avg(struct rq *rq)
 25{
 26	return 0;
 27}
 28#endif
 29
 30#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
 31int update_irq_load_avg(struct rq *rq, u64 running);
 32#else
 33static inline int
 34update_irq_load_avg(struct rq *rq, u64 running)
 35{
 36	return 0;
 37}
 38#endif
 39
 40#define PELT_MIN_DIVIDER	(LOAD_AVG_MAX - 1024)
 41
 42static inline u32 get_pelt_divider(struct sched_avg *avg)
 43{
 44	return PELT_MIN_DIVIDER + avg->period_contrib;
 45}
 46
 47static inline void cfs_se_util_change(struct sched_avg *avg)
 48{
 49	unsigned int enqueued;
 50
 51	if (!sched_feat(UTIL_EST))
 52		return;
 53
 54	/* Avoid store if the flag has been already reset */
 55	enqueued = avg->util_est.enqueued;
 56	if (!(enqueued & UTIL_AVG_UNCHANGED))
 57		return;
 58
 59	/* Reset flag to report util_avg has been updated */
 60	enqueued &= ~UTIL_AVG_UNCHANGED;
 61	WRITE_ONCE(avg->util_est.enqueued, enqueued);
 62}
 63
 64static inline u64 rq_clock_pelt(struct rq *rq)
 65{
 66	lockdep_assert_rq_held(rq);
 67	assert_clock_updated(rq);
 68
 69	return rq->clock_pelt - rq->lost_idle_time;
 70}
 71
 72/* The rq is idle, we can sync to clock_task */
 73static inline void _update_idle_rq_clock_pelt(struct rq *rq)
 74{
 75	rq->clock_pelt  = rq_clock_task(rq);
 76
 77	u64_u32_store(rq->clock_idle, rq_clock(rq));
 78	/* Paired with smp_rmb in migrate_se_pelt_lag() */
 79	smp_wmb();
 80	u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
 81}
 82
 83/*
 84 * The clock_pelt scales the time to reflect the effective amount of
 85 * computation done during the running delta time but then sync back to
 86 * clock_task when rq is idle.
 87 *
 88 *
 89 * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
 90 * @ max capacity  ------******---------------******---------------
 91 * @ half capacity ------************---------************---------
 92 * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
 93 *
 94 */
 95static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
 96{
 97	if (unlikely(is_idle_task(rq->curr))) {
 98		_update_idle_rq_clock_pelt(rq);
 99		return;
100	}
101
102	/*
103	 * When a rq runs at a lower compute capacity, it will need
104	 * more time to do the same amount of work than at max
105	 * capacity. In order to be invariant, we scale the delta to
106	 * reflect how much work has been really done.
107	 * Running longer results in stealing idle time that will
108	 * disturb the load signal compared to max capacity. This
109	 * stolen idle time will be automatically reflected when the
110	 * rq will be idle and the clock will be synced with
111	 * rq_clock_task.
112	 */
113
114	/*
115	 * Scale the elapsed time to reflect the real amount of
116	 * computation
117	 */
118	delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
119	delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
120
121	rq->clock_pelt += delta;
122}
123
124/*
125 * When rq becomes idle, we have to check if it has lost idle time
126 * because it was fully busy. A rq is fully used when the /Sum util_sum
127 * is greater or equal to:
128 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
129 * For optimization and computing rounding purpose, we don't take into account
130 * the position in the current window (period_contrib) and we use the higher
131 * bound of util_sum to decide.
132 */
133static inline void update_idle_rq_clock_pelt(struct rq *rq)
134{
135	u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
136	u32 util_sum = rq->cfs.avg.util_sum;
137	util_sum += rq->avg_rt.util_sum;
138	util_sum += rq->avg_dl.util_sum;
139
140	/*
141	 * Reflecting stolen time makes sense only if the idle
142	 * phase would be present at max capacity. As soon as the
143	 * utilization of a rq has reached the maximum value, it is
144	 * considered as an always running rq without idle time to
145	 * steal. This potential idle time is considered as lost in
146	 * this case. We keep track of this lost idle time compare to
147	 * rq's clock_task.
148	 */
149	if (util_sum >= divider)
150		rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
151
152	_update_idle_rq_clock_pelt(rq);
153}
154
155#ifdef CONFIG_CFS_BANDWIDTH
156static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
157{
158	u64 throttled;
159
160	if (unlikely(cfs_rq->throttle_count))
161		throttled = U64_MAX;
162	else
163		throttled = cfs_rq->throttled_clock_pelt_time;
164
165	u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
166}
167
168/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
169static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
170{
171	if (unlikely(cfs_rq->throttle_count))
172		return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
173
174	return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
175}
176#else
177static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
178static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
179{
180	return rq_clock_pelt(rq_of(cfs_rq));
181}
182#endif
183
184#else
185
186static inline int
187update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
188{
189	return 0;
190}
191
192static inline int
193update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
194{
195	return 0;
196}
197
198static inline int
199update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
200{
201	return 0;
202}
203
204static inline int
205update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
206{
207	return 0;
208}
209
210static inline u64 thermal_load_avg(struct rq *rq)
211{
212	return 0;
213}
214
215static inline int
216update_irq_load_avg(struct rq *rq, u64 running)
217{
218	return 0;
219}
220
221static inline u64 rq_clock_pelt(struct rq *rq)
222{
223	return rq_clock_task(rq);
224}
225
226static inline void
227update_rq_clock_pelt(struct rq *rq, s64 delta) { }
228
229static inline void
230update_idle_rq_clock_pelt(struct rq *rq) { }
231
232static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
233#endif
234
235