1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  kernel/sched/core.c
   4 *
   5 *  Core kernel scheduler code and related syscalls
   6 *
   7 *  Copyright (C) 1991-2002  Linus Torvalds
   8 */
   9#include "sched.h"
  10
 
  11#include <linux/nospec.h>
  12
  13#include <linux/kcov.h>
  14
  15#include <asm/switch_to.h>
  16#include <asm/tlb.h>
  17
  18#include "../workqueue_internal.h"
  19#include "../smpboot.h"
  20
  21#include "pelt.h"
  22
  23#define CREATE_TRACE_POINTS
  24#include <trace/events/sched.h>
  25
  26/*
  27 * Export tracepoints that act as a bare tracehook (ie: have no trace event
  28 * associated with them) to allow external modules to probe them.
  29 */
  30EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
  31EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
  32EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
  33EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
  34EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
  35EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
  36
  37DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  38
  39#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL)
  40/*
  41 * Debugging: various feature bits
  42 *
  43 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of
  44 * sysctl_sched_features, defined in sched.h, to allow constants propagation
  45 * at compile time and compiler optimization based on features default.
  46 */
  47#define SCHED_FEAT(name, enabled)	\
  48	(1UL << __SCHED_FEAT_##name) * enabled |
  49const_debug unsigned int sysctl_sched_features =
  50#include "features.h"
  51	0;
  52#undef SCHED_FEAT
  53#endif
  54
  55/*
  56 * Number of tasks to iterate in a single balance run.
  57 * Limited because this is done with IRQs disabled.
  58 */
  59const_debug unsigned int sysctl_sched_nr_migrate = 32;
  60
  61/*
 
 
 
 
 
 
 
 
  62 * period over which we measure -rt task CPU usage in us.
  63 * default: 1s
  64 */
  65unsigned int sysctl_sched_rt_period = 1000000;
  66
  67__read_mostly int scheduler_running;
  68
  69/*
  70 * part of the period that we allow rt tasks to run in us.
  71 * default: 0.95s
  72 */
  73int sysctl_sched_rt_runtime = 950000;
  74
  75/*
  76 * __task_rq_lock - lock the rq @p resides on.
  77 */
  78struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
  79	__acquires(rq->lock)
  80{
  81	struct rq *rq;
  82
  83	lockdep_assert_held(&p->pi_lock);
  84
  85	for (;;) {
  86		rq = task_rq(p);
  87		raw_spin_lock(&rq->lock);
  88		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
  89			rq_pin_lock(rq, rf);
  90			return rq;
  91		}
  92		raw_spin_unlock(&rq->lock);
  93
  94		while (unlikely(task_on_rq_migrating(p)))
  95			cpu_relax();
  96	}
  97}
  98
  99/*
 100 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
 101 */
 102struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
 103	__acquires(p->pi_lock)
 104	__acquires(rq->lock)
 105{
 106	struct rq *rq;
 107
 108	for (;;) {
 109		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
 110		rq = task_rq(p);
 111		raw_spin_lock(&rq->lock);
 112		/*
 113		 *	move_queued_task()		task_rq_lock()
 114		 *
 115		 *	ACQUIRE (rq->lock)
 116		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()
 117		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);
 118		 *	[S] ->cpu = new_cpu		[L] task_rq()
 119		 *					[L] ->on_rq
 120		 *	RELEASE (rq->lock)
 121		 *
 122		 * If we observe the old CPU in task_rq_lock(), the acquire of
 123		 * the old rq->lock will fully serialize against the stores.
 124		 *
 125		 * If we observe the new CPU in task_rq_lock(), the address
 126		 * dependency headed by '[L] rq = task_rq()' and the acquire
 127		 * will pair with the WMB to ensure we then also see migrating.
 128		 */
 129		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
 130			rq_pin_lock(rq, rf);
 131			return rq;
 132		}
 133		raw_spin_unlock(&rq->lock);
 134		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
 135
 136		while (unlikely(task_on_rq_migrating(p)))
 137			cpu_relax();
 138	}
 139}
 140
 141/*
 142 * RQ-clock updating methods:
 143 */
 144
 145static void update_rq_clock_task(struct rq *rq, s64 delta)
 146{
 147/*
 148 * In theory, the compile should just see 0 here, and optimize out the call
 149 * to sched_rt_avg_update. But I don't trust it...
 150 */
 151	s64 __maybe_unused steal = 0, irq_delta = 0;
 152
 
 153#ifdef CONFIG_IRQ_TIME_ACCOUNTING
 154	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
 155
 156	/*
 157	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
 158	 * this case when a previous update_rq_clock() happened inside a
 159	 * {soft,}irq region.
 160	 *
 161	 * When this happens, we stop ->clock_task and only update the
 162	 * prev_irq_time stamp to account for the part that fit, so that a next
 163	 * update will consume the rest. This ensures ->clock_task is
 164	 * monotonic.
 165	 *
 166	 * It does however cause some slight miss-attribution of {soft,}irq
 167	 * time, a more accurate solution would be to update the irq_time using
 168	 * the current rq->clock timestamp, except that would require using
 169	 * atomic ops.
 170	 */
 171	if (irq_delta > delta)
 172		irq_delta = delta;
 173
 174	rq->prev_irq_time += irq_delta;
 175	delta -= irq_delta;
 176#endif
 177#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
 178	if (static_key_false((&paravirt_steal_rq_enabled))) {
 179		steal = paravirt_steal_clock(cpu_of(rq));
 180		steal -= rq->prev_steal_time_rq;
 181
 182		if (unlikely(steal > delta))
 183			steal = delta;
 184
 185		rq->prev_steal_time_rq += steal;
 186		delta -= steal;
 187	}
 188#endif
 189
 190	rq->clock_task += delta;
 191
 192#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
 193	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
 194		update_irq_load_avg(rq, irq_delta + steal);
 195#endif
 196	update_rq_clock_pelt(rq, delta);
 197}
 198
 199void update_rq_clock(struct rq *rq)
 200{
 201	s64 delta;
 202
 203	lockdep_assert_held(&rq->lock);
 204
 205	if (rq->clock_update_flags & RQCF_ACT_SKIP)
 206		return;
 207
 208#ifdef CONFIG_SCHED_DEBUG
 209	if (sched_feat(WARN_DOUBLE_CLOCK))
 210		SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED);
 211	rq->clock_update_flags |= RQCF_UPDATED;
 212#endif
 213
 214	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
 215	if (delta < 0)
 216		return;
 217	rq->clock += delta;
 218	update_rq_clock_task(rq, delta);
 219}
 220
 221
 222#ifdef CONFIG_SCHED_HRTICK
 223/*
 224 * Use HR-timers to deliver accurate preemption points.
 225 */
 226
 227static void hrtick_clear(struct rq *rq)
 228{
 229	if (hrtimer_active(&rq->hrtick_timer))
 230		hrtimer_cancel(&rq->hrtick_timer);
 231}
 232
 233/*
 234 * High-resolution timer tick.
 235 * Runs from hardirq context with interrupts disabled.
 236 */
 237static enum hrtimer_restart hrtick(struct hrtimer *timer)
 238{
 239	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
 240	struct rq_flags rf;
 241
 242	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
 243
 244	rq_lock(rq, &rf);
 245	update_rq_clock(rq);
 246	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
 247	rq_unlock(rq, &rf);
 248
 249	return HRTIMER_NORESTART;
 250}
 251
 252#ifdef CONFIG_SMP
 253
 254static void __hrtick_restart(struct rq *rq)
 255{
 256	struct hrtimer *timer = &rq->hrtick_timer;
 257
 258	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
 259}
 260
 261/*
 262 * called from hardirq (IPI) context
 263 */
 264static void __hrtick_start(void *arg)
 265{
 266	struct rq *rq = arg;
 267	struct rq_flags rf;
 268
 269	rq_lock(rq, &rf);
 270	__hrtick_restart(rq);
 271	rq->hrtick_csd_pending = 0;
 272	rq_unlock(rq, &rf);
 273}
 274
 275/*
 276 * Called to set the hrtick timer state.
 277 *
 278 * called with rq->lock held and irqs disabled
 279 */
 280void hrtick_start(struct rq *rq, u64 delay)
 281{
 282	struct hrtimer *timer = &rq->hrtick_timer;
 283	ktime_t time;
 284	s64 delta;
 285
 286	/*
 287	 * Don't schedule slices shorter than 10000ns, that just
 288	 * doesn't make sense and can cause timer DoS.
 289	 */
 290	delta = max_t(s64, delay, 10000LL);
 291	time = ktime_add_ns(timer->base->get_time(), delta);
 292
 293	hrtimer_set_expires(timer, time);
 294
 295	if (rq == this_rq()) {
 296		__hrtick_restart(rq);
 297	} else if (!rq->hrtick_csd_pending) {
 298		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
 299		rq->hrtick_csd_pending = 1;
 300	}
 301}
 302
 303#else
 304/*
 305 * Called to set the hrtick timer state.
 306 *
 307 * called with rq->lock held and irqs disabled
 308 */
 309void hrtick_start(struct rq *rq, u64 delay)
 310{
 311	/*
 312	 * Don't schedule slices shorter than 10000ns, that just
 313	 * doesn't make sense. Rely on vruntime for fairness.
 314	 */
 315	delay = max_t(u64, delay, 10000LL);
 316	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
 317		      HRTIMER_MODE_REL_PINNED_HARD);
 318}
 319#endif /* CONFIG_SMP */
 320
 321static void hrtick_rq_init(struct rq *rq)
 322{
 323#ifdef CONFIG_SMP
 324	rq->hrtick_csd_pending = 0;
 325
 326	rq->hrtick_csd.flags = 0;
 327	rq->hrtick_csd.func = __hrtick_start;
 328	rq->hrtick_csd.info = rq;
 329#endif
 330
 331	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
 332	rq->hrtick_timer.function = hrtick;
 333}
 334#else	/* CONFIG_SCHED_HRTICK */
 335static inline void hrtick_clear(struct rq *rq)
 336{
 337}
 338
 339static inline void hrtick_rq_init(struct rq *rq)
 340{
 341}
 342#endif	/* CONFIG_SCHED_HRTICK */
 343
 344/*
 345 * cmpxchg based fetch_or, macro so it works for different integer types
 346 */
 347#define fetch_or(ptr, mask)						\
 348	({								\
 349		typeof(ptr) _ptr = (ptr);				\
 350		typeof(mask) _mask = (mask);				\
 351		typeof(*_ptr) _old, _val = *_ptr;			\
 352									\
 353		for (;;) {						\
 354			_old = cmpxchg(_ptr, _val, _val | _mask);	\
 355			if (_old == _val)				\
 356				break;					\
 357			_val = _old;					\
 358		}							\
 359	_old;								\
 360})
 361
 362#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
 363/*
 364 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
 365 * this avoids any races wrt polling state changes and thereby avoids
 366 * spurious IPIs.
 367 */
 368static bool set_nr_and_not_polling(struct task_struct *p)
 369{
 370	struct thread_info *ti = task_thread_info(p);
 371	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
 372}
 373
 374/*
 375 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
 376 *
 377 * If this returns true, then the idle task promises to call
 378 * sched_ttwu_pending() and reschedule soon.
 379 */
 380static bool set_nr_if_polling(struct task_struct *p)
 381{
 382	struct thread_info *ti = task_thread_info(p);
 383	typeof(ti->flags) old, val = READ_ONCE(ti->flags);
 384
 385	for (;;) {
 386		if (!(val & _TIF_POLLING_NRFLAG))
 387			return false;
 388		if (val & _TIF_NEED_RESCHED)
 389			return true;
 390		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
 391		if (old == val)
 392			break;
 393		val = old;
 394	}
 395	return true;
 396}
 397
 398#else
 399static bool set_nr_and_not_polling(struct task_struct *p)
 400{
 401	set_tsk_need_resched(p);
 402	return true;
 403}
 404
 405#ifdef CONFIG_SMP
 406static bool set_nr_if_polling(struct task_struct *p)
 407{
 408	return false;
 409}
 410#endif
 411#endif
 412
 413static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
 414{
 415	struct wake_q_node *node = &task->wake_q;
 416
 417	/*
 418	 * Atomically grab the task, if ->wake_q is !nil already it means
 419	 * its already queued (either by us or someone else) and will get the
 420	 * wakeup due to that.
 421	 *
 422	 * In order to ensure that a pending wakeup will observe our pending
 423	 * state, even in the failed case, an explicit smp_mb() must be used.
 424	 */
 425	smp_mb__before_atomic();
 426	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
 427		return false;
 
 428
 429	/*
 430	 * The head is context local, there can be no concurrency.
 431	 */
 432	*head->lastp = node;
 433	head->lastp = &node->next;
 434	return true;
 435}
 436
 437/**
 438 * wake_q_add() - queue a wakeup for 'later' waking.
 439 * @head: the wake_q_head to add @task to
 440 * @task: the task to queue for 'later' wakeup
 441 *
 442 * Queue a task for later wakeup, most likely by the wake_up_q() call in the
 443 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
 444 * instantly.
 445 *
 446 * This function must be used as-if it were wake_up_process(); IOW the task
 447 * must be ready to be woken at this location.
 448 */
 449void wake_q_add(struct wake_q_head *head, struct task_struct *task)
 450{
 451	if (__wake_q_add(head, task))
 452		get_task_struct(task);
 453}
 454
 455/**
 456 * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
 457 * @head: the wake_q_head to add @task to
 458 * @task: the task to queue for 'later' wakeup
 459 *
 460 * Queue a task for later wakeup, most likely by the wake_up_q() call in the
 461 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
 462 * instantly.
 463 *
 464 * This function must be used as-if it were wake_up_process(); IOW the task
 465 * must be ready to be woken at this location.
 466 *
 467 * This function is essentially a task-safe equivalent to wake_q_add(). Callers
 468 * that already hold reference to @task can call the 'safe' version and trust
 469 * wake_q to do the right thing depending whether or not the @task is already
 470 * queued for wakeup.
 471 */
 472void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
 473{
 474	if (!__wake_q_add(head, task))
 475		put_task_struct(task);
 476}
 477
 478void wake_up_q(struct wake_q_head *head)
 479{
 480	struct wake_q_node *node = head->first;
 481
 482	while (node != WAKE_Q_TAIL) {
 483		struct task_struct *task;
 484
 485		task = container_of(node, struct task_struct, wake_q);
 486		BUG_ON(!task);
 487		/* Task can safely be re-inserted now: */
 488		node = node->next;
 489		task->wake_q.next = NULL;
 490
 491		/*
 492		 * wake_up_process() executes a full barrier, which pairs with
 493		 * the queueing in wake_q_add() so as not to miss wakeups.
 494		 */
 495		wake_up_process(task);
 496		put_task_struct(task);
 497	}
 498}
 499
 500/*
 501 * resched_curr - mark rq's current task 'to be rescheduled now'.
 502 *
 503 * On UP this means the setting of the need_resched flag, on SMP it
 504 * might also involve a cross-CPU call to trigger the scheduler on
 505 * the target CPU.
 506 */
 507void resched_curr(struct rq *rq)
 508{
 509	struct task_struct *curr = rq->curr;
 510	int cpu;
 511
 512	lockdep_assert_held(&rq->lock);
 513
 514	if (test_tsk_need_resched(curr))
 515		return;
 516
 517	cpu = cpu_of(rq);
 518
 519	if (cpu == smp_processor_id()) {
 520		set_tsk_need_resched(curr);
 521		set_preempt_need_resched();
 522		return;
 523	}
 524
 525	if (set_nr_and_not_polling(curr))
 526		smp_send_reschedule(cpu);
 527	else
 528		trace_sched_wake_idle_without_ipi(cpu);
 529}
 530
 531void resched_cpu(int cpu)
 532{
 533	struct rq *rq = cpu_rq(cpu);
 534	unsigned long flags;
 535
 536	raw_spin_lock_irqsave(&rq->lock, flags);
 537	if (cpu_online(cpu) || cpu == smp_processor_id())
 538		resched_curr(rq);
 539	raw_spin_unlock_irqrestore(&rq->lock, flags);
 540}
 541
 542#ifdef CONFIG_SMP
 543#ifdef CONFIG_NO_HZ_COMMON
 544/*
 545 * In the semi idle case, use the nearest busy CPU for migrating timers
 546 * from an idle CPU.  This is good for power-savings.
 547 *
 548 * We don't do similar optimization for completely idle system, as
 549 * selecting an idle CPU will add more delays to the timers than intended
 550 * (as that CPU's timer base may not be uptodate wrt jiffies etc).
 551 */
 552int get_nohz_timer_target(void)
 553{
 554	int i, cpu = smp_processor_id();
 555	struct sched_domain *sd;
 556
 557	if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER))
 558		return cpu;
 559
 560	rcu_read_lock();
 561	for_each_domain(cpu, sd) {
 562		for_each_cpu(i, sched_domain_span(sd)) {
 563			if (cpu == i)
 564				continue;
 565
 566			if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) {
 567				cpu = i;
 568				goto unlock;
 569			}
 570		}
 571	}
 572
 573	if (!housekeeping_cpu(cpu, HK_FLAG_TIMER))
 574		cpu = housekeeping_any_cpu(HK_FLAG_TIMER);
 575unlock:
 576	rcu_read_unlock();
 577	return cpu;
 578}
 579
 580/*
 581 * When add_timer_on() enqueues a timer into the timer wheel of an
 582 * idle CPU then this timer might expire before the next timer event
 583 * which is scheduled to wake up that CPU. In case of a completely
 584 * idle system the next event might even be infinite time into the
 585 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 586 * leaves the inner idle loop so the newly added timer is taken into
 587 * account when the CPU goes back to idle and evaluates the timer
 588 * wheel for the next timer event.
 589 */
 590static void wake_up_idle_cpu(int cpu)
 591{
 592	struct rq *rq = cpu_rq(cpu);
 593
 594	if (cpu == smp_processor_id())
 595		return;
 596
 597	if (set_nr_and_not_polling(rq->idle))
 598		smp_send_reschedule(cpu);
 599	else
 600		trace_sched_wake_idle_without_ipi(cpu);
 601}
 602
 603static bool wake_up_full_nohz_cpu(int cpu)
 604{
 605	/*
 606	 * We just need the target to call irq_exit() and re-evaluate
 607	 * the next tick. The nohz full kick at least implies that.
 608	 * If needed we can still optimize that later with an
 609	 * empty IRQ.
 610	 */
 611	if (cpu_is_offline(cpu))
 612		return true;  /* Don't try to wake offline CPUs. */
 613	if (tick_nohz_full_cpu(cpu)) {
 614		if (cpu != smp_processor_id() ||
 615		    tick_nohz_tick_stopped())
 616			tick_nohz_full_kick_cpu(cpu);
 617		return true;
 618	}
 619
 620	return false;
 621}
 622
 623/*
 624 * Wake up the specified CPU.  If the CPU is going offline, it is the
 625 * caller's responsibility to deal with the lost wakeup, for example,
 626 * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
 627 */
 628void wake_up_nohz_cpu(int cpu)
 629{
 630	if (!wake_up_full_nohz_cpu(cpu))
 631		wake_up_idle_cpu(cpu);
 632}
 633
 634static inline bool got_nohz_idle_kick(void)
 635{
 636	int cpu = smp_processor_id();
 637
 638	if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
 639		return false;
 640
 641	if (idle_cpu(cpu) && !need_resched())
 642		return true;
 643
 644	/*
 645	 * We can't run Idle Load Balance on this CPU for this time so we
 646	 * cancel it and clear NOHZ_BALANCE_KICK
 647	 */
 648	atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
 649	return false;
 650}
 651
 652#else /* CONFIG_NO_HZ_COMMON */
 653
 654static inline bool got_nohz_idle_kick(void)
 655{
 656	return false;
 657}
 658
 659#endif /* CONFIG_NO_HZ_COMMON */
 660
 661#ifdef CONFIG_NO_HZ_FULL
 662bool sched_can_stop_tick(struct rq *rq)
 663{
 664	int fifo_nr_running;
 665
 666	/* Deadline tasks, even if single, need the tick */
 667	if (rq->dl.dl_nr_running)
 668		return false;
 669
 670	/*
 671	 * If there are more than one RR tasks, we need the tick to effect the
 672	 * actual RR behaviour.
 673	 */
 674	if (rq->rt.rr_nr_running) {
 675		if (rq->rt.rr_nr_running == 1)
 676			return true;
 677		else
 678			return false;
 679	}
 680
 681	/*
 682	 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
 683	 * forced preemption between FIFO tasks.
 684	 */
 685	fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
 686	if (fifo_nr_running)
 687		return true;
 688
 689	/*
 690	 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
 691	 * if there's more than one we need the tick for involuntary
 692	 * preemption.
 693	 */
 694	if (rq->nr_running > 1)
 695		return false;
 696
 697	return true;
 698}
 699#endif /* CONFIG_NO_HZ_FULL */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 700#endif /* CONFIG_SMP */
 701
 702#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
 703			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
 704/*
 705 * Iterate task_group tree rooted at *from, calling @down when first entering a
 706 * node and @up when leaving it for the final time.
 707 *
 708 * Caller must hold rcu_lock or sufficient equivalent.
 709 */
 710int walk_tg_tree_from(struct task_group *from,
 711			     tg_visitor down, tg_visitor up, void *data)
 712{
 713	struct task_group *parent, *child;
 714	int ret;
 715
 716	parent = from;
 717
 718down:
 719	ret = (*down)(parent, data);
 720	if (ret)
 721		goto out;
 722	list_for_each_entry_rcu(child, &parent->children, siblings) {
 723		parent = child;
 724		goto down;
 725
 726up:
 727		continue;
 728	}
 729	ret = (*up)(parent, data);
 730	if (ret || parent == from)
 731		goto out;
 732
 733	child = parent;
 734	parent = parent->parent;
 735	if (parent)
 736		goto up;
 737out:
 738	return ret;
 739}
 740
 741int tg_nop(struct task_group *tg, void *data)
 742{
 743	return 0;
 744}
 745#endif
 746
 747static void set_load_weight(struct task_struct *p, bool update_load)
 748{
 749	int prio = p->static_prio - MAX_RT_PRIO;
 750	struct load_weight *load = &p->se.load;
 751
 752	/*
 753	 * SCHED_IDLE tasks get minimal weight:
 754	 */
 755	if (task_has_idle_policy(p)) {
 756		load->weight = scale_load(WEIGHT_IDLEPRIO);
 757		load->inv_weight = WMULT_IDLEPRIO;
 758		p->se.runnable_weight = load->weight;
 759		return;
 760	}
 761
 762	/*
 763	 * SCHED_OTHER tasks have to update their load when changing their
 764	 * weight
 765	 */
 766	if (update_load && p->sched_class == &fair_sched_class) {
 767		reweight_task(p, prio);
 768	} else {
 769		load->weight = scale_load(sched_prio_to_weight[prio]);
 770		load->inv_weight = sched_prio_to_wmult[prio];
 771		p->se.runnable_weight = load->weight;
 772	}
 773}
 774
 775#ifdef CONFIG_UCLAMP_TASK
 776/*
 777 * Serializes updates of utilization clamp values
 778 *
 779 * The (slow-path) user-space triggers utilization clamp value updates which
 780 * can require updates on (fast-path) scheduler's data structures used to
 781 * support enqueue/dequeue operations.
 782 * While the per-CPU rq lock protects fast-path update operations, user-space
 783 * requests are serialized using a mutex to reduce the risk of conflicting
 784 * updates or API abuses.
 785 */
 786static DEFINE_MUTEX(uclamp_mutex);
 787
 788/* Max allowed minimum utilization */
 789unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
 790
 791/* Max allowed maximum utilization */
 792unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
 793
 794/* All clamps are required to be less or equal than these values */
 795static struct uclamp_se uclamp_default[UCLAMP_CNT];
 796
 797/* Integer rounded range for each bucket */
 798#define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS)
 799
 800#define for_each_clamp_id(clamp_id) \
 801	for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)
 802
 803static inline unsigned int uclamp_bucket_id(unsigned int clamp_value)
 804{
 805	return clamp_value / UCLAMP_BUCKET_DELTA;
 806}
 807
 808static inline unsigned int uclamp_bucket_base_value(unsigned int clamp_value)
 809{
 810	return UCLAMP_BUCKET_DELTA * uclamp_bucket_id(clamp_value);
 811}
 812
 813static inline enum uclamp_id uclamp_none(enum uclamp_id clamp_id)
 814{
 815	if (clamp_id == UCLAMP_MIN)
 816		return 0;
 817	return SCHED_CAPACITY_SCALE;
 818}
 819
 820static inline void uclamp_se_set(struct uclamp_se *uc_se,
 821				 unsigned int value, bool user_defined)
 822{
 823	uc_se->value = value;
 824	uc_se->bucket_id = uclamp_bucket_id(value);
 825	uc_se->user_defined = user_defined;
 826}
 827
 828static inline unsigned int
 829uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id,
 830		  unsigned int clamp_value)
 831{
 832	/*
 833	 * Avoid blocked utilization pushing up the frequency when we go
 834	 * idle (which drops the max-clamp) by retaining the last known
 835	 * max-clamp.
 836	 */
 837	if (clamp_id == UCLAMP_MAX) {
 838		rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
 839		return clamp_value;
 840	}
 841
 842	return uclamp_none(UCLAMP_MIN);
 843}
 844
 845static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id,
 846				     unsigned int clamp_value)
 847{
 848	/* Reset max-clamp retention only on idle exit */
 849	if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
 850		return;
 851
 852	WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value);
 853}
 854
 855static inline
 856enum uclamp_id uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id,
 857				   unsigned int clamp_value)
 858{
 859	struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
 860	int bucket_id = UCLAMP_BUCKETS - 1;
 861
 862	/*
 863	 * Since both min and max clamps are max aggregated, find the
 864	 * top most bucket with tasks in.
 865	 */
 866	for ( ; bucket_id >= 0; bucket_id--) {
 867		if (!bucket[bucket_id].tasks)
 868			continue;
 869		return bucket[bucket_id].value;
 870	}
 871
 872	/* No tasks -- default clamp values */
 873	return uclamp_idle_value(rq, clamp_id, clamp_value);
 874}
 875
 876static inline struct uclamp_se
 877uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
 878{
 879	struct uclamp_se uc_req = p->uclamp_req[clamp_id];
 880#ifdef CONFIG_UCLAMP_TASK_GROUP
 881	struct uclamp_se uc_max;
 882
 883	/*
 884	 * Tasks in autogroups or root task group will be
 885	 * restricted by system defaults.
 886	 */
 887	if (task_group_is_autogroup(task_group(p)))
 888		return uc_req;
 889	if (task_group(p) == &root_task_group)
 890		return uc_req;
 891
 892	uc_max = task_group(p)->uclamp[clamp_id];
 893	if (uc_req.value > uc_max.value || !uc_req.user_defined)
 894		return uc_max;
 895#endif
 896
 897	return uc_req;
 898}
 899
 900/*
 901 * The effective clamp bucket index of a task depends on, by increasing
 902 * priority:
 903 * - the task specific clamp value, when explicitly requested from userspace
 904 * - the task group effective clamp value, for tasks not either in the root
 905 *   group or in an autogroup
 906 * - the system default clamp value, defined by the sysadmin
 907 */
 908static inline struct uclamp_se
 909uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
 910{
 911	struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
 912	struct uclamp_se uc_max = uclamp_default[clamp_id];
 913
 914	/* System default restrictions always apply */
 915	if (unlikely(uc_req.value > uc_max.value))
 916		return uc_max;
 917
 918	return uc_req;
 919}
 920
 921enum uclamp_id uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
 922{
 923	struct uclamp_se uc_eff;
 924
 925	/* Task currently refcounted: use back-annotated (effective) value */
 926	if (p->uclamp[clamp_id].active)
 927		return p->uclamp[clamp_id].value;
 928
 929	uc_eff = uclamp_eff_get(p, clamp_id);
 930
 931	return uc_eff.value;
 932}
 933
 934/*
 935 * When a task is enqueued on a rq, the clamp bucket currently defined by the
 936 * task's uclamp::bucket_id is refcounted on that rq. This also immediately
 937 * updates the rq's clamp value if required.
 938 *
 939 * Tasks can have a task-specific value requested from user-space, track
 940 * within each bucket the maximum value for tasks refcounted in it.
 941 * This "local max aggregation" allows to track the exact "requested" value
 942 * for each bucket when all its RUNNABLE tasks require the same clamp.
 943 */
 944static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
 945				    enum uclamp_id clamp_id)
 946{
 947	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
 948	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
 949	struct uclamp_bucket *bucket;
 950
 951	lockdep_assert_held(&rq->lock);
 952
 953	/* Update task effective clamp */
 954	p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
 955
 956	bucket = &uc_rq->bucket[uc_se->bucket_id];
 957	bucket->tasks++;
 958	uc_se->active = true;
 959
 960	uclamp_idle_reset(rq, clamp_id, uc_se->value);
 961
 962	/*
 963	 * Local max aggregation: rq buckets always track the max
 964	 * "requested" clamp value of its RUNNABLE tasks.
 965	 */
 966	if (bucket->tasks == 1 || uc_se->value > bucket->value)
 967		bucket->value = uc_se->value;
 968
 969	if (uc_se->value > READ_ONCE(uc_rq->value))
 970		WRITE_ONCE(uc_rq->value, uc_se->value);
 971}
 972
 973/*
 974 * When a task is dequeued from a rq, the clamp bucket refcounted by the task
 975 * is released. If this is the last task reference counting the rq's max
 976 * active clamp value, then the rq's clamp value is updated.
 977 *
 978 * Both refcounted tasks and rq's cached clamp values are expected to be
 979 * always valid. If it's detected they are not, as defensive programming,
 980 * enforce the expected state and warn.
 981 */
 982static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
 983				    enum uclamp_id clamp_id)
 984{
 985	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
 986	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
 987	struct uclamp_bucket *bucket;
 988	unsigned int bkt_clamp;
 989	unsigned int rq_clamp;
 990
 991	lockdep_assert_held(&rq->lock);
 992
 993	bucket = &uc_rq->bucket[uc_se->bucket_id];
 994	SCHED_WARN_ON(!bucket->tasks);
 995	if (likely(bucket->tasks))
 996		bucket->tasks--;
 997	uc_se->active = false;
 998
 999	/*
1000	 * Keep "local max aggregation" simple and accept to (possibly)
1001	 * overboost some RUNNABLE tasks in the same bucket.
1002	 * The rq clamp bucket value is reset to its base value whenever
1003	 * there are no more RUNNABLE tasks refcounting it.
1004	 */
1005	if (likely(bucket->tasks))
1006		return;
1007
1008	rq_clamp = READ_ONCE(uc_rq->value);
1009	/*
1010	 * Defensive programming: this should never happen. If it happens,
1011	 * e.g. due to future modification, warn and fixup the expected value.
1012	 */
1013	SCHED_WARN_ON(bucket->value > rq_clamp);
1014	if (bucket->value >= rq_clamp) {
1015		bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
1016		WRITE_ONCE(uc_rq->value, bkt_clamp);
1017	}
1018}
1019
1020static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
1021{
1022	enum uclamp_id clamp_id;
1023
1024	if (unlikely(!p->sched_class->uclamp_enabled))
1025		return;
1026
1027	for_each_clamp_id(clamp_id)
1028		uclamp_rq_inc_id(rq, p, clamp_id);
1029
1030	/* Reset clamp idle holding when there is one RUNNABLE task */
1031	if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
1032		rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1033}
1034
1035static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
1036{
1037	enum uclamp_id clamp_id;
1038
1039	if (unlikely(!p->sched_class->uclamp_enabled))
1040		return;
1041
1042	for_each_clamp_id(clamp_id)
1043		uclamp_rq_dec_id(rq, p, clamp_id);
1044}
1045
1046static inline void
1047uclamp_update_active(struct task_struct *p, enum uclamp_id clamp_id)
1048{
1049	struct rq_flags rf;
1050	struct rq *rq;
1051
1052	/*
1053	 * Lock the task and the rq where the task is (or was) queued.
1054	 *
1055	 * We might lock the (previous) rq of a !RUNNABLE task, but that's the
1056	 * price to pay to safely serialize util_{min,max} updates with
1057	 * enqueues, dequeues and migration operations.
1058	 * This is the same locking schema used by __set_cpus_allowed_ptr().
1059	 */
1060	rq = task_rq_lock(p, &rf);
1061
1062	/*
1063	 * Setting the clamp bucket is serialized by task_rq_lock().
1064	 * If the task is not yet RUNNABLE and its task_struct is not
1065	 * affecting a valid clamp bucket, the next time it's enqueued,
1066	 * it will already see the updated clamp bucket value.
1067	 */
1068	if (p->uclamp[clamp_id].active) {
1069		uclamp_rq_dec_id(rq, p, clamp_id);
1070		uclamp_rq_inc_id(rq, p, clamp_id);
1071	}
1072
1073	task_rq_unlock(rq, p, &rf);
1074}
1075
1076#ifdef CONFIG_UCLAMP_TASK_GROUP
1077static inline void
1078uclamp_update_active_tasks(struct cgroup_subsys_state *css,
1079			   unsigned int clamps)
1080{
1081	enum uclamp_id clamp_id;
1082	struct css_task_iter it;
1083	struct task_struct *p;
1084
1085	css_task_iter_start(css, 0, &it);
1086	while ((p = css_task_iter_next(&it))) {
1087		for_each_clamp_id(clamp_id) {
1088			if ((0x1 << clamp_id) & clamps)
1089				uclamp_update_active(p, clamp_id);
1090		}
1091	}
1092	css_task_iter_end(&it);
1093}
1094
1095static void cpu_util_update_eff(struct cgroup_subsys_state *css);
1096static void uclamp_update_root_tg(void)
1097{
1098	struct task_group *tg = &root_task_group;
1099
1100	uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN],
1101		      sysctl_sched_uclamp_util_min, false);
1102	uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX],
1103		      sysctl_sched_uclamp_util_max, false);
1104
1105	rcu_read_lock();
1106	cpu_util_update_eff(&root_task_group.css);
1107	rcu_read_unlock();
1108}
1109#else
1110static void uclamp_update_root_tg(void) { }
1111#endif
1112
1113int sysctl_sched_uclamp_handler(struct ctl_table *table, int write,
1114				void __user *buffer, size_t *lenp,
1115				loff_t *ppos)
1116{
1117	bool update_root_tg = false;
1118	int old_min, old_max;
1119	int result;
1120
1121	mutex_lock(&uclamp_mutex);
1122	old_min = sysctl_sched_uclamp_util_min;
1123	old_max = sysctl_sched_uclamp_util_max;
1124
1125	result = proc_dointvec(table, write, buffer, lenp, ppos);
1126	if (result)
1127		goto undo;
1128	if (!write)
1129		goto done;
1130
1131	if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1132	    sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE) {
1133		result = -EINVAL;
1134		goto undo;
1135	}
1136
1137	if (old_min != sysctl_sched_uclamp_util_min) {
1138		uclamp_se_set(&uclamp_default[UCLAMP_MIN],
1139			      sysctl_sched_uclamp_util_min, false);
1140		update_root_tg = true;
1141	}
1142	if (old_max != sysctl_sched_uclamp_util_max) {
1143		uclamp_se_set(&uclamp_default[UCLAMP_MAX],
1144			      sysctl_sched_uclamp_util_max, false);
1145		update_root_tg = true;
1146	}
1147
1148	if (update_root_tg)
1149		uclamp_update_root_tg();
1150
1151	/*
1152	 * We update all RUNNABLE tasks only when task groups are in use.
1153	 * Otherwise, keep it simple and do just a lazy update at each next
1154	 * task enqueue time.
1155	 */
1156
1157	goto done;
1158
1159undo:
1160	sysctl_sched_uclamp_util_min = old_min;
1161	sysctl_sched_uclamp_util_max = old_max;
1162done:
1163	mutex_unlock(&uclamp_mutex);
1164
1165	return result;
1166}
1167
1168static int uclamp_validate(struct task_struct *p,
1169			   const struct sched_attr *attr)
1170{
1171	unsigned int lower_bound = p->uclamp_req[UCLAMP_MIN].value;
1172	unsigned int upper_bound = p->uclamp_req[UCLAMP_MAX].value;
1173
1174	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN)
1175		lower_bound = attr->sched_util_min;
1176	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX)
1177		upper_bound = attr->sched_util_max;
1178
1179	if (lower_bound > upper_bound)
1180		return -EINVAL;
1181	if (upper_bound > SCHED_CAPACITY_SCALE)
1182		return -EINVAL;
1183
1184	return 0;
1185}
1186
1187static void __setscheduler_uclamp(struct task_struct *p,
1188				  const struct sched_attr *attr)
1189{
1190	enum uclamp_id clamp_id;
1191
1192	/*
1193	 * On scheduling class change, reset to default clamps for tasks
1194	 * without a task-specific value.
1195	 */
1196	for_each_clamp_id(clamp_id) {
1197		struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];
1198		unsigned int clamp_value = uclamp_none(clamp_id);
1199
1200		/* Keep using defined clamps across class changes */
1201		if (uc_se->user_defined)
1202			continue;
1203
1204		/* By default, RT tasks always get 100% boost */
1205		if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1206			clamp_value = uclamp_none(UCLAMP_MAX);
1207
1208		uclamp_se_set(uc_se, clamp_value, false);
1209	}
1210
1211	if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))
1212		return;
1213
1214	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) {
1215		uclamp_se_set(&p->uclamp_req[UCLAMP_MIN],
1216			      attr->sched_util_min, true);
1217	}
1218
1219	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) {
1220		uclamp_se_set(&p->uclamp_req[UCLAMP_MAX],
1221			      attr->sched_util_max, true);
1222	}
1223}
1224
1225static void uclamp_fork(struct task_struct *p)
1226{
1227	enum uclamp_id clamp_id;
1228
1229	for_each_clamp_id(clamp_id)
1230		p->uclamp[clamp_id].active = false;
1231
1232	if (likely(!p->sched_reset_on_fork))
1233		return;
1234
1235	for_each_clamp_id(clamp_id) {
1236		unsigned int clamp_value = uclamp_none(clamp_id);
1237
1238		/* By default, RT tasks always get 100% boost */
1239		if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1240			clamp_value = uclamp_none(UCLAMP_MAX);
1241
1242		uclamp_se_set(&p->uclamp_req[clamp_id], clamp_value, false);
1243	}
1244}
1245
1246static void __init init_uclamp(void)
1247{
1248	struct uclamp_se uc_max = {};
1249	enum uclamp_id clamp_id;
1250	int cpu;
1251
1252	mutex_init(&uclamp_mutex);
1253
1254	for_each_possible_cpu(cpu) {
1255		memset(&cpu_rq(cpu)->uclamp, 0, sizeof(struct uclamp_rq));
1256		cpu_rq(cpu)->uclamp_flags = 0;
1257	}
1258
1259	for_each_clamp_id(clamp_id) {
1260		uclamp_se_set(&init_task.uclamp_req[clamp_id],
1261			      uclamp_none(clamp_id), false);
1262	}
1263
1264	/* System defaults allow max clamp values for both indexes */
1265	uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false);
1266	for_each_clamp_id(clamp_id) {
1267		uclamp_default[clamp_id] = uc_max;
1268#ifdef CONFIG_UCLAMP_TASK_GROUP
1269		root_task_group.uclamp_req[clamp_id] = uc_max;
1270		root_task_group.uclamp[clamp_id] = uc_max;
1271#endif
1272	}
1273}
1274
1275#else /* CONFIG_UCLAMP_TASK */
1276static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { }
1277static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
1278static inline int uclamp_validate(struct task_struct *p,
1279				  const struct sched_attr *attr)
1280{
1281	return -EOPNOTSUPP;
1282}
1283static void __setscheduler_uclamp(struct task_struct *p,
1284				  const struct sched_attr *attr) { }
1285static inline void uclamp_fork(struct task_struct *p) { }
1286static inline void init_uclamp(void) { }
1287#endif /* CONFIG_UCLAMP_TASK */
1288
1289static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
1290{
1291	if (!(flags & ENQUEUE_NOCLOCK))
1292		update_rq_clock(rq);
1293
1294	if (!(flags & ENQUEUE_RESTORE)) {
1295		sched_info_queued(rq, p);
1296		psi_enqueue(p, flags & ENQUEUE_WAKEUP);
1297	}
1298
1299	uclamp_rq_inc(rq, p);
1300	p->sched_class->enqueue_task(rq, p, flags);
1301}
1302
1303static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
1304{
1305	if (!(flags & DEQUEUE_NOCLOCK))
1306		update_rq_clock(rq);
1307
1308	if (!(flags & DEQUEUE_SAVE)) {
1309		sched_info_dequeued(rq, p);
1310		psi_dequeue(p, flags & DEQUEUE_SLEEP);
1311	}
1312
1313	uclamp_rq_dec(rq, p);
1314	p->sched_class->dequeue_task(rq, p, flags);
1315}
1316
1317void activate_task(struct rq *rq, struct task_struct *p, int flags)
1318{
1319	if (task_contributes_to_load(p))
1320		rq->nr_uninterruptible--;
1321
1322	enqueue_task(rq, p, flags);
1323
1324	p->on_rq = TASK_ON_RQ_QUEUED;
1325}
1326
1327void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1328{
1329	p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
1330
1331	if (task_contributes_to_load(p))
1332		rq->nr_uninterruptible++;
1333
1334	dequeue_task(rq, p, flags);
1335}
1336
1337/*
1338 * __normal_prio - return the priority that is based on the static prio
1339 */
1340static inline int __normal_prio(struct task_struct *p)
1341{
1342	return p->static_prio;
1343}
1344
1345/*
1346 * Calculate the expected normal priority: i.e. priority
1347 * without taking RT-inheritance into account. Might be
1348 * boosted by interactivity modifiers. Changes upon fork,
1349 * setprio syscalls, and whenever the interactivity
1350 * estimator recalculates.
1351 */
1352static inline int normal_prio(struct task_struct *p)
1353{
1354	int prio;
1355
1356	if (task_has_dl_policy(p))
1357		prio = MAX_DL_PRIO-1;
1358	else if (task_has_rt_policy(p))
1359		prio = MAX_RT_PRIO-1 - p->rt_priority;
1360	else
1361		prio = __normal_prio(p);
1362	return prio;
1363}
1364
1365/*
1366 * Calculate the current priority, i.e. the priority
1367 * taken into account by the scheduler. This value might
1368 * be boosted by RT tasks, or might be boosted by
1369 * interactivity modifiers. Will be RT if the task got
1370 * RT-boosted. If not then it returns p->normal_prio.
1371 */
1372static int effective_prio(struct task_struct *p)
1373{
1374	p->normal_prio = normal_prio(p);
1375	/*
1376	 * If we are RT tasks or we were boosted to RT priority,
1377	 * keep the priority unchanged. Otherwise, update priority
1378	 * to the normal priority:
1379	 */
1380	if (!rt_prio(p->prio))
1381		return p->normal_prio;
1382	return p->prio;
1383}
1384
1385/**
1386 * task_curr - is this task currently executing on a CPU?
1387 * @p: the task in question.
1388 *
1389 * Return: 1 if the task is currently executing. 0 otherwise.
1390 */
1391inline int task_curr(const struct task_struct *p)
1392{
1393	return cpu_curr(task_cpu(p)) == p;
1394}
1395
1396/*
1397 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
1398 * use the balance_callback list if you want balancing.
1399 *
1400 * this means any call to check_class_changed() must be followed by a call to
1401 * balance_callback().
1402 */
1403static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1404				       const struct sched_class *prev_class,
1405				       int oldprio)
1406{
1407	if (prev_class != p->sched_class) {
1408		if (prev_class->switched_from)
1409			prev_class->switched_from(rq, p);
1410
1411		p->sched_class->switched_to(rq, p);
1412	} else if (oldprio != p->prio || dl_task(p))
1413		p->sched_class->prio_changed(rq, p, oldprio);
1414}
1415
1416void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1417{
1418	const struct sched_class *class;
1419
1420	if (p->sched_class == rq->curr->sched_class) {
1421		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
1422	} else {
1423		for_each_class(class) {
1424			if (class == rq->curr->sched_class)
1425				break;
1426			if (class == p->sched_class) {
1427				resched_curr(rq);
1428				break;
1429			}
1430		}
1431	}
1432
1433	/*
1434	 * A queue event has occurred, and we're going to schedule.  In
1435	 * this case, we can save a useless back to back clock update.
1436	 */
1437	if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
1438		rq_clock_skip_update(rq);
1439}
1440
1441#ifdef CONFIG_SMP
1442
1443static inline bool is_per_cpu_kthread(struct task_struct *p)
1444{
1445	if (!(p->flags & PF_KTHREAD))
1446		return false;
1447
1448	if (p->nr_cpus_allowed != 1)
1449		return false;
1450
1451	return true;
1452}
1453
1454/*
1455 * Per-CPU kthreads are allowed to run on !active && online CPUs, see
1456 * __set_cpus_allowed_ptr() and select_fallback_rq().
1457 */
1458static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
1459{
1460	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
1461		return false;
1462
1463	if (is_per_cpu_kthread(p))
1464		return cpu_online(cpu);
1465
1466	return cpu_active(cpu);
1467}
1468
1469/*
1470 * This is how migration works:
1471 *
1472 * 1) we invoke migration_cpu_stop() on the target CPU using
1473 *    stop_one_cpu().
1474 * 2) stopper starts to run (implicitly forcing the migrated thread
1475 *    off the CPU)
1476 * 3) it checks whether the migrated task is still in the wrong runqueue.
1477 * 4) if it's in the wrong runqueue then the migration thread removes
1478 *    it and puts it into the right queue.
1479 * 5) stopper completes and stop_one_cpu() returns and the migration
1480 *    is done.
1481 */
1482
1483/*
1484 * move_queued_task - move a queued task to new rq.
1485 *
1486 * Returns (locked) new rq. Old rq's lock is released.
1487 */
1488static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
1489				   struct task_struct *p, int new_cpu)
1490{
1491	lockdep_assert_held(&rq->lock);
1492
1493	WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
1494	dequeue_task(rq, p, DEQUEUE_NOCLOCK);
1495	set_task_cpu(p, new_cpu);
1496	rq_unlock(rq, rf);
1497
1498	rq = cpu_rq(new_cpu);
1499
1500	rq_lock(rq, rf);
1501	BUG_ON(task_cpu(p) != new_cpu);
1502	enqueue_task(rq, p, 0);
1503	p->on_rq = TASK_ON_RQ_QUEUED;
1504	check_preempt_curr(rq, p, 0);
1505
1506	return rq;
1507}
1508
1509struct migration_arg {
1510	struct task_struct *task;
1511	int dest_cpu;
1512};
1513
1514/*
1515 * Move (not current) task off this CPU, onto the destination CPU. We're doing
1516 * this because either it can't run here any more (set_cpus_allowed()
1517 * away from this CPU, or CPU going down), or because we're
1518 * attempting to rebalance this task on exec (sched_exec).
1519 *
1520 * So we race with normal scheduler movements, but that's OK, as long
1521 * as the task is no longer on this CPU.
1522 */
1523static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
1524				 struct task_struct *p, int dest_cpu)
1525{
1526	/* Affinity changed (again). */
1527	if (!is_cpu_allowed(p, dest_cpu))
1528		return rq;
1529
1530	update_rq_clock(rq);
1531	rq = move_queued_task(rq, rf, p, dest_cpu);
1532
1533	return rq;
1534}
1535
1536/*
1537 * migration_cpu_stop - this will be executed by a highprio stopper thread
1538 * and performs thread migration by bumping thread off CPU then
1539 * 'pushing' onto another runqueue.
1540 */
1541static int migration_cpu_stop(void *data)
1542{
1543	struct migration_arg *arg = data;
1544	struct task_struct *p = arg->task;
1545	struct rq *rq = this_rq();
1546	struct rq_flags rf;
1547
1548	/*
1549	 * The original target CPU might have gone down and we might
1550	 * be on another CPU but it doesn't matter.
1551	 */
1552	local_irq_disable();
1553	/*
1554	 * We need to explicitly wake pending tasks before running
1555	 * __migrate_task() such that we will not miss enforcing cpus_ptr
1556	 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1557	 */
1558	sched_ttwu_pending();
1559
1560	raw_spin_lock(&p->pi_lock);
1561	rq_lock(rq, &rf);
1562	/*
1563	 * If task_rq(p) != rq, it cannot be migrated here, because we're
1564	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1565	 * we're holding p->pi_lock.
1566	 */
1567	if (task_rq(p) == rq) {
1568		if (task_on_rq_queued(p))
1569			rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
1570		else
1571			p->wake_cpu = arg->dest_cpu;
1572	}
1573	rq_unlock(rq, &rf);
1574	raw_spin_unlock(&p->pi_lock);
1575
1576	local_irq_enable();
1577	return 0;
1578}
1579
1580/*
1581 * sched_class::set_cpus_allowed must do the below, but is not required to
1582 * actually call this function.
1583 */
1584void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
1585{
1586	cpumask_copy(&p->cpus_mask, new_mask);
1587	p->nr_cpus_allowed = cpumask_weight(new_mask);
1588}
1589
1590void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1591{
1592	struct rq *rq = task_rq(p);
1593	bool queued, running;
1594
1595	lockdep_assert_held(&p->pi_lock);
1596
1597	queued = task_on_rq_queued(p);
1598	running = task_current(rq, p);
1599
1600	if (queued) {
1601		/*
1602		 * Because __kthread_bind() calls this on blocked tasks without
1603		 * holding rq->lock.
1604		 */
1605		lockdep_assert_held(&rq->lock);
1606		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
1607	}
1608	if (running)
1609		put_prev_task(rq, p);
1610
1611	p->sched_class->set_cpus_allowed(p, new_mask);
1612
1613	if (queued)
1614		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
1615	if (running)
1616		set_next_task(rq, p);
1617}
1618
1619/*
1620 * Change a given task's CPU affinity. Migrate the thread to a
1621 * proper CPU and schedule it away if the CPU it's executing on
1622 * is removed from the allowed bitmask.
1623 *
1624 * NOTE: the caller must have a valid reference to the task, the
1625 * task must not exit() & deallocate itself prematurely. The
1626 * call is not atomic; no spinlocks may be held.
1627 */
1628static int __set_cpus_allowed_ptr(struct task_struct *p,
1629				  const struct cpumask *new_mask, bool check)
1630{
1631	const struct cpumask *cpu_valid_mask = cpu_active_mask;
1632	unsigned int dest_cpu;
1633	struct rq_flags rf;
1634	struct rq *rq;
1635	int ret = 0;
1636
1637	rq = task_rq_lock(p, &rf);
1638	update_rq_clock(rq);
1639
1640	if (p->flags & PF_KTHREAD) {
1641		/*
1642		 * Kernel threads are allowed on online && !active CPUs
1643		 */
1644		cpu_valid_mask = cpu_online_mask;
1645	}
1646
1647	/*
1648	 * Must re-check here, to close a race against __kthread_bind(),
1649	 * sched_setaffinity() is not guaranteed to observe the flag.
1650	 */
1651	if (check && (p->flags & PF_NO_SETAFFINITY)) {
1652		ret = -EINVAL;
1653		goto out;
1654	}
1655
1656	if (cpumask_equal(p->cpus_ptr, new_mask))
1657		goto out;
1658
1659	dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
1660	if (dest_cpu >= nr_cpu_ids) {
1661		ret = -EINVAL;
1662		goto out;
1663	}
1664
1665	do_set_cpus_allowed(p, new_mask);
1666
1667	if (p->flags & PF_KTHREAD) {
1668		/*
1669		 * For kernel threads that do indeed end up on online &&
1670		 * !active we want to ensure they are strict per-CPU threads.
1671		 */
1672		WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
1673			!cpumask_intersects(new_mask, cpu_active_mask) &&
1674			p->nr_cpus_allowed != 1);
1675	}
1676
1677	/* Can the task run on the task's current CPU? If so, we're done */
1678	if (cpumask_test_cpu(task_cpu(p), new_mask))
1679		goto out;
1680
 
1681	if (task_running(rq, p) || p->state == TASK_WAKING) {
1682		struct migration_arg arg = { p, dest_cpu };
1683		/* Need help from migration thread: drop lock and wait. */
1684		task_rq_unlock(rq, p, &rf);
1685		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
 
1686		return 0;
1687	} else if (task_on_rq_queued(p)) {
1688		/*
1689		 * OK, since we're going to drop the lock immediately
1690		 * afterwards anyway.
1691		 */
1692		rq = move_queued_task(rq, &rf, p, dest_cpu);
1693	}
1694out:
1695	task_rq_unlock(rq, p, &rf);
1696
1697	return ret;
1698}
1699
1700int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1701{
1702	return __set_cpus_allowed_ptr(p, new_mask, false);
1703}
1704EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1705
1706void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1707{
1708#ifdef CONFIG_SCHED_DEBUG
1709	/*
1710	 * We should never call set_task_cpu() on a blocked task,
1711	 * ttwu() will sort out the placement.
1712	 */
1713	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
1714			!p->on_rq);
1715
1716	/*
1717	 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
1718	 * because schedstat_wait_{start,end} rebase migrating task's wait_start
1719	 * time relying on p->on_rq.
1720	 */
1721	WARN_ON_ONCE(p->state == TASK_RUNNING &&
1722		     p->sched_class == &fair_sched_class &&
1723		     (p->on_rq && !task_on_rq_migrating(p)));
1724
1725#ifdef CONFIG_LOCKDEP
1726	/*
1727	 * The caller should hold either p->pi_lock or rq->lock, when changing
1728	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1729	 *
1730	 * sched_move_task() holds both and thus holding either pins the cgroup,
1731	 * see task_group().
1732	 *
1733	 * Furthermore, all task_rq users should acquire both locks, see
1734	 * task_rq_lock().
1735	 */
1736	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1737				      lockdep_is_held(&task_rq(p)->lock)));
1738#endif
1739	/*
1740	 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.
1741	 */
1742	WARN_ON_ONCE(!cpu_online(new_cpu));
1743#endif
1744
1745	trace_sched_migrate_task(p, new_cpu);
1746
1747	if (task_cpu(p) != new_cpu) {
1748		if (p->sched_class->migrate_task_rq)
1749			p->sched_class->migrate_task_rq(p, new_cpu);
1750		p->se.nr_migrations++;
1751		rseq_migrate(p);
1752		perf_event_task_migrate(p);
1753	}
1754
1755	__set_task_cpu(p, new_cpu);
1756}
1757
1758#ifdef CONFIG_NUMA_BALANCING
1759static void __migrate_swap_task(struct task_struct *p, int cpu)
1760{
1761	if (task_on_rq_queued(p)) {
1762		struct rq *src_rq, *dst_rq;
1763		struct rq_flags srf, drf;
1764
1765		src_rq = task_rq(p);
1766		dst_rq = cpu_rq(cpu);
1767
1768		rq_pin_lock(src_rq, &srf);
1769		rq_pin_lock(dst_rq, &drf);
1770
 
1771		deactivate_task(src_rq, p, 0);
1772		set_task_cpu(p, cpu);
1773		activate_task(dst_rq, p, 0);
 
1774		check_preempt_curr(dst_rq, p, 0);
1775
1776		rq_unpin_lock(dst_rq, &drf);
1777		rq_unpin_lock(src_rq, &srf);
1778
1779	} else {
1780		/*
1781		 * Task isn't running anymore; make it appear like we migrated
1782		 * it before it went to sleep. This means on wakeup we make the
1783		 * previous CPU our target instead of where it really is.
1784		 */
1785		p->wake_cpu = cpu;
1786	}
1787}
1788
1789struct migration_swap_arg {
1790	struct task_struct *src_task, *dst_task;
1791	int src_cpu, dst_cpu;
1792};
1793
1794static int migrate_swap_stop(void *data)
1795{
1796	struct migration_swap_arg *arg = data;
1797	struct rq *src_rq, *dst_rq;
1798	int ret = -EAGAIN;
1799
1800	if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
1801		return -EAGAIN;
1802
1803	src_rq = cpu_rq(arg->src_cpu);
1804	dst_rq = cpu_rq(arg->dst_cpu);
1805
1806	double_raw_lock(&arg->src_task->pi_lock,
1807			&arg->dst_task->pi_lock);
1808	double_rq_lock(src_rq, dst_rq);
1809
1810	if (task_cpu(arg->dst_task) != arg->dst_cpu)
1811		goto unlock;
1812
1813	if (task_cpu(arg->src_task) != arg->src_cpu)
1814		goto unlock;
1815
1816	if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
1817		goto unlock;
1818
1819	if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
1820		goto unlock;
1821
1822	__migrate_swap_task(arg->src_task, arg->dst_cpu);
1823	__migrate_swap_task(arg->dst_task, arg->src_cpu);
1824
1825	ret = 0;
1826
1827unlock:
1828	double_rq_unlock(src_rq, dst_rq);
1829	raw_spin_unlock(&arg->dst_task->pi_lock);
1830	raw_spin_unlock(&arg->src_task->pi_lock);
1831
1832	return ret;
1833}
1834
1835/*
1836 * Cross migrate two tasks
1837 */
1838int migrate_swap(struct task_struct *cur, struct task_struct *p,
1839		int target_cpu, int curr_cpu)
1840{
1841	struct migration_swap_arg arg;
1842	int ret = -EINVAL;
1843
1844	arg = (struct migration_swap_arg){
1845		.src_task = cur,
1846		.src_cpu = curr_cpu,
1847		.dst_task = p,
1848		.dst_cpu = target_cpu,
1849	};
1850
1851	if (arg.src_cpu == arg.dst_cpu)
1852		goto out;
1853
1854	/*
1855	 * These three tests are all lockless; this is OK since all of them
1856	 * will be re-checked with proper locks held further down the line.
1857	 */
1858	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1859		goto out;
1860
1861	if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
1862		goto out;
1863
1864	if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
1865		goto out;
1866
1867	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1868	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1869
1870out:
1871	return ret;
1872}
1873#endif /* CONFIG_NUMA_BALANCING */
1874
1875/*
1876 * wait_task_inactive - wait for a thread to unschedule.
1877 *
1878 * If @match_state is nonzero, it's the @p->state value just checked and
1879 * not expected to change.  If it changes, i.e. @p might have woken up,
1880 * then return zero.  When we succeed in waiting for @p to be off its CPU,
1881 * we return a positive number (its total switch count).  If a second call
1882 * a short while later returns the same number, the caller can be sure that
1883 * @p has remained unscheduled the whole time.
1884 *
1885 * The caller must ensure that the task *will* unschedule sometime soon,
1886 * else this function might spin for a *long* time. This function can't
1887 * be called with interrupts off, or it may introduce deadlock with
1888 * smp_call_function() if an IPI is sent by the same process we are
1889 * waiting to become inactive.
1890 */
1891unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1892{
1893	int running, queued;
1894	struct rq_flags rf;
1895	unsigned long ncsw;
1896	struct rq *rq;
1897
1898	for (;;) {
1899		/*
1900		 * We do the initial early heuristics without holding
1901		 * any task-queue locks at all. We'll only try to get
1902		 * the runqueue lock when things look like they will
1903		 * work out!
1904		 */
1905		rq = task_rq(p);
1906
1907		/*
1908		 * If the task is actively running on another CPU
1909		 * still, just relax and busy-wait without holding
1910		 * any locks.
1911		 *
1912		 * NOTE! Since we don't hold any locks, it's not
1913		 * even sure that "rq" stays as the right runqueue!
1914		 * But we don't care, since "task_running()" will
1915		 * return false if the runqueue has changed and p
1916		 * is actually now running somewhere else!
1917		 */
1918		while (task_running(rq, p)) {
1919			if (match_state && unlikely(p->state != match_state))
1920				return 0;
1921			cpu_relax();
1922		}
1923
1924		/*
1925		 * Ok, time to look more closely! We need the rq
1926		 * lock now, to be *sure*. If we're wrong, we'll
1927		 * just go back and repeat.
1928		 */
1929		rq = task_rq_lock(p, &rf);
1930		trace_sched_wait_task(p);
1931		running = task_running(rq, p);
1932		queued = task_on_rq_queued(p);
1933		ncsw = 0;
1934		if (!match_state || p->state == match_state)
1935			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1936		task_rq_unlock(rq, p, &rf);
1937
1938		/*
1939		 * If it changed from the expected state, bail out now.
1940		 */
1941		if (unlikely(!ncsw))
1942			break;
1943
1944		/*
1945		 * Was it really running after all now that we
1946		 * checked with the proper locks actually held?
1947		 *
1948		 * Oops. Go back and try again..
1949		 */
1950		if (unlikely(running)) {
1951			cpu_relax();
1952			continue;
1953		}
1954
1955		/*
1956		 * It's not enough that it's not actively running,
1957		 * it must be off the runqueue _entirely_, and not
1958		 * preempted!
1959		 *
1960		 * So if it was still runnable (but just not actively
1961		 * running right now), it's preempted, and we should
1962		 * yield - it could be a while.
1963		 */
1964		if (unlikely(queued)) {
1965			ktime_t to = NSEC_PER_SEC / HZ;
1966
1967			set_current_state(TASK_UNINTERRUPTIBLE);
1968			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1969			continue;
1970		}
1971
1972		/*
1973		 * Ahh, all good. It wasn't running, and it wasn't
1974		 * runnable, which means that it will never become
1975		 * running in the future either. We're all done!
1976		 */
1977		break;
1978	}
1979
1980	return ncsw;
1981}
1982
1983/***
1984 * kick_process - kick a running thread to enter/exit the kernel
1985 * @p: the to-be-kicked thread
1986 *
1987 * Cause a process which is running on another CPU to enter
1988 * kernel-mode, without any delay. (to get signals handled.)
1989 *
1990 * NOTE: this function doesn't have to take the runqueue lock,
1991 * because all it wants to ensure is that the remote task enters
1992 * the kernel. If the IPI races and the task has been migrated
1993 * to another CPU then no harm is done and the purpose has been
1994 * achieved as well.
1995 */
1996void kick_process(struct task_struct *p)
1997{
1998	int cpu;
1999
2000	preempt_disable();
2001	cpu = task_cpu(p);
2002	if ((cpu != smp_processor_id()) && task_curr(p))
2003		smp_send_reschedule(cpu);
2004	preempt_enable();
2005}
2006EXPORT_SYMBOL_GPL(kick_process);
2007
2008/*
2009 * ->cpus_ptr is protected by both rq->lock and p->pi_lock
2010 *
2011 * A few notes on cpu_active vs cpu_online:
2012 *
2013 *  - cpu_active must be a subset of cpu_online
2014 *
2015 *  - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
2016 *    see __set_cpus_allowed_ptr(). At this point the newly online
2017 *    CPU isn't yet part of the sched domains, and balancing will not
2018 *    see it.
2019 *
2020 *  - on CPU-down we clear cpu_active() to mask the sched domains and
2021 *    avoid the load balancer to place new tasks on the to be removed
2022 *    CPU. Existing tasks will remain running there and will be taken
2023 *    off.
2024 *
2025 * This means that fallback selection must not select !active CPUs.
2026 * And can assume that any active CPU must be online. Conversely
2027 * select_task_rq() below may allow selection of !active CPUs in order
2028 * to satisfy the above rules.
2029 */
2030static int select_fallback_rq(int cpu, struct task_struct *p)
2031{
2032	int nid = cpu_to_node(cpu);
2033	const struct cpumask *nodemask = NULL;
2034	enum { cpuset, possible, fail } state = cpuset;
2035	int dest_cpu;
2036
2037	/*
2038	 * If the node that the CPU is on has been offlined, cpu_to_node()
2039	 * will return -1. There is no CPU on the node, and we should
2040	 * select the CPU on the other node.
2041	 */
2042	if (nid != -1) {
2043		nodemask = cpumask_of_node(nid);
2044
2045		/* Look for allowed, online CPU in same node. */
2046		for_each_cpu(dest_cpu, nodemask) {
2047			if (!cpu_active(dest_cpu))
2048				continue;
2049			if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
2050				return dest_cpu;
2051		}
2052	}
2053
2054	for (;;) {
2055		/* Any allowed, online CPU? */
2056		for_each_cpu(dest_cpu, p->cpus_ptr) {
2057			if (!is_cpu_allowed(p, dest_cpu))
2058				continue;
2059
2060			goto out;
2061		}
2062
2063		/* No more Mr. Nice Guy. */
2064		switch (state) {
2065		case cpuset:
2066			if (IS_ENABLED(CONFIG_CPUSETS)) {
2067				cpuset_cpus_allowed_fallback(p);
2068				state = possible;
2069				break;
2070			}
2071			/* Fall-through */
2072		case possible:
2073			do_set_cpus_allowed(p, cpu_possible_mask);
2074			state = fail;
2075			break;
2076
2077		case fail:
2078			BUG();
2079			break;
2080		}
2081	}
2082
2083out:
2084	if (state != cpuset) {
2085		/*
2086		 * Don't tell them about moving exiting tasks or
2087		 * kernel threads (both mm NULL), since they never
2088		 * leave kernel.
2089		 */
2090		if (p->mm && printk_ratelimit()) {
2091			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
2092					task_pid_nr(p), p->comm, cpu);
2093		}
2094	}
2095
2096	return dest_cpu;
2097}
2098
2099/*
2100 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
2101 */
2102static inline
2103int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
2104{
2105	lockdep_assert_held(&p->pi_lock);
2106
2107	if (p->nr_cpus_allowed > 1)
2108		cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
2109	else
2110		cpu = cpumask_any(p->cpus_ptr);
2111
2112	/*
2113	 * In order not to call set_task_cpu() on a blocking task we need
2114	 * to rely on ttwu() to place the task on a valid ->cpus_ptr
2115	 * CPU.
2116	 *
2117	 * Since this is common to all placement strategies, this lives here.
2118	 *
2119	 * [ this allows ->select_task() to simply return task_cpu(p) and
2120	 *   not worry about this generic constraint ]
2121	 */
2122	if (unlikely(!is_cpu_allowed(p, cpu)))
2123		cpu = select_fallback_rq(task_cpu(p), p);
2124
2125	return cpu;
2126}
2127
2128static void update_avg(u64 *avg, u64 sample)
2129{
2130	s64 diff = sample - *avg;
2131	*avg += diff >> 3;
2132}
2133
2134void sched_set_stop_task(int cpu, struct task_struct *stop)
2135{
2136	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
2137	struct task_struct *old_stop = cpu_rq(cpu)->stop;
2138
2139	if (stop) {
2140		/*
2141		 * Make it appear like a SCHED_FIFO task, its something
2142		 * userspace knows about and won't get confused about.
2143		 *
2144		 * Also, it will make PI more or less work without too
2145		 * much confusion -- but then, stop work should not
2146		 * rely on PI working anyway.
2147		 */
2148		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
2149
2150		stop->sched_class = &stop_sched_class;
2151	}
2152
2153	cpu_rq(cpu)->stop = stop;
2154
2155	if (old_stop) {
2156		/*
2157		 * Reset it back to a normal scheduling class so that
2158		 * it can die in pieces.
2159		 */
2160		old_stop->sched_class = &rt_sched_class;
2161	}
2162}
2163
2164#else
2165
2166static inline int __set_cpus_allowed_ptr(struct task_struct *p,
2167					 const struct cpumask *new_mask, bool check)
2168{
2169	return set_cpus_allowed_ptr(p, new_mask);
2170}
2171
2172#endif /* CONFIG_SMP */
2173
2174static void
2175ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
2176{
2177	struct rq *rq;
2178
2179	if (!schedstat_enabled())
2180		return;
2181
2182	rq = this_rq();
2183
2184#ifdef CONFIG_SMP
2185	if (cpu == rq->cpu) {
2186		__schedstat_inc(rq->ttwu_local);
2187		__schedstat_inc(p->se.statistics.nr_wakeups_local);
2188	} else {
2189		struct sched_domain *sd;
2190
2191		__schedstat_inc(p->se.statistics.nr_wakeups_remote);
2192		rcu_read_lock();
2193		for_each_domain(rq->cpu, sd) {
2194			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2195				__schedstat_inc(sd->ttwu_wake_remote);
2196				break;
2197			}
2198		}
2199		rcu_read_unlock();
2200	}
2201
2202	if (wake_flags & WF_MIGRATED)
2203		__schedstat_inc(p->se.statistics.nr_wakeups_migrate);
2204#endif /* CONFIG_SMP */
2205
2206	__schedstat_inc(rq->ttwu_count);
2207	__schedstat_inc(p->se.statistics.nr_wakeups);
2208
2209	if (wake_flags & WF_SYNC)
2210		__schedstat_inc(p->se.statistics.nr_wakeups_sync);
2211}
2212
 
 
 
 
 
 
 
 
 
 
2213/*
2214 * Mark the task runnable and perform wakeup-preemption.
2215 */
2216static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
2217			   struct rq_flags *rf)
2218{
2219	check_preempt_curr(rq, p, wake_flags);
2220	p->state = TASK_RUNNING;
2221	trace_sched_wakeup(p);
2222
2223#ifdef CONFIG_SMP
2224	if (p->sched_class->task_woken) {
2225		/*
2226		 * Our task @p is fully woken up and running; so its safe to
2227		 * drop the rq->lock, hereafter rq is only used for statistics.
2228		 */
2229		rq_unpin_lock(rq, rf);
2230		p->sched_class->task_woken(rq, p);
2231		rq_repin_lock(rq, rf);
2232	}
2233
2234	if (rq->idle_stamp) {
2235		u64 delta = rq_clock(rq) - rq->idle_stamp;
2236		u64 max = 2*rq->max_idle_balance_cost;
2237
2238		update_avg(&rq->avg_idle, delta);
2239
2240		if (rq->avg_idle > max)
2241			rq->avg_idle = max;
2242
2243		rq->idle_stamp = 0;
2244	}
2245#endif
2246}
2247
2248static void
2249ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
2250		 struct rq_flags *rf)
2251{
2252	int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
2253
2254	lockdep_assert_held(&rq->lock);
2255
2256#ifdef CONFIG_SMP
2257	if (p->sched_contributes_to_load)
2258		rq->nr_uninterruptible--;
2259
2260	if (wake_flags & WF_MIGRATED)
2261		en_flags |= ENQUEUE_MIGRATED;
2262#endif
2263
2264	activate_task(rq, p, en_flags);
2265	ttwu_do_wakeup(rq, p, wake_flags, rf);
2266}
2267
2268/*
2269 * Called in case the task @p isn't fully descheduled from its runqueue,
2270 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
2271 * since all we need to do is flip p->state to TASK_RUNNING, since
2272 * the task is still ->on_rq.
2273 */
2274static int ttwu_remote(struct task_struct *p, int wake_flags)
2275{
2276	struct rq_flags rf;
2277	struct rq *rq;
2278	int ret = 0;
2279
2280	rq = __task_rq_lock(p, &rf);
2281	if (task_on_rq_queued(p)) {
2282		/* check_preempt_curr() may use rq clock */
2283		update_rq_clock(rq);
2284		ttwu_do_wakeup(rq, p, wake_flags, &rf);
2285		ret = 1;
2286	}
2287	__task_rq_unlock(rq, &rf);
2288
2289	return ret;
2290}
2291
2292#ifdef CONFIG_SMP
2293void sched_ttwu_pending(void)
2294{
2295	struct rq *rq = this_rq();
2296	struct llist_node *llist = llist_del_all(&rq->wake_list);
2297	struct task_struct *p, *t;
2298	struct rq_flags rf;
2299
2300	if (!llist)
2301		return;
2302
2303	rq_lock_irqsave(rq, &rf);
2304	update_rq_clock(rq);
2305
2306	llist_for_each_entry_safe(p, t, llist, wake_entry)
2307		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
2308
2309	rq_unlock_irqrestore(rq, &rf);
2310}
2311
2312void scheduler_ipi(void)
2313{
2314	/*
2315	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
2316	 * TIF_NEED_RESCHED remotely (for the first time) will also send
2317	 * this IPI.
2318	 */
2319	preempt_fold_need_resched();
2320
2321	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
2322		return;
2323
2324	/*
2325	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
2326	 * traditionally all their work was done from the interrupt return
2327	 * path. Now that we actually do some work, we need to make sure
2328	 * we do call them.
2329	 *
2330	 * Some archs already do call them, luckily irq_enter/exit nest
2331	 * properly.
2332	 *
2333	 * Arguably we should visit all archs and update all handlers,
2334	 * however a fair share of IPIs are still resched only so this would
2335	 * somewhat pessimize the simple resched case.
2336	 */
2337	irq_enter();
2338	sched_ttwu_pending();
2339
2340	/*
2341	 * Check if someone kicked us for doing the nohz idle load balance.
2342	 */
2343	if (unlikely(got_nohz_idle_kick())) {
2344		this_rq()->idle_balance = 1;
2345		raise_softirq_irqoff(SCHED_SOFTIRQ);
2346	}
2347	irq_exit();
2348}
2349
2350static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
2351{
2352	struct rq *rq = cpu_rq(cpu);
2353
2354	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
2355
2356	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
2357		if (!set_nr_if_polling(rq->idle))
2358			smp_send_reschedule(cpu);
2359		else
2360			trace_sched_wake_idle_without_ipi(cpu);
2361	}
2362}
2363
2364void wake_up_if_idle(int cpu)
2365{
2366	struct rq *rq = cpu_rq(cpu);
2367	struct rq_flags rf;
2368
2369	rcu_read_lock();
2370
2371	if (!is_idle_task(rcu_dereference(rq->curr)))
2372		goto out;
2373
2374	if (set_nr_if_polling(rq->idle)) {
2375		trace_sched_wake_idle_without_ipi(cpu);
2376	} else {
2377		rq_lock_irqsave(rq, &rf);
2378		if (is_idle_task(rq->curr))
2379			smp_send_reschedule(cpu);
2380		/* Else CPU is not idle, do nothing here: */
2381		rq_unlock_irqrestore(rq, &rf);
2382	}
2383
2384out:
2385	rcu_read_unlock();
2386}
2387
2388bool cpus_share_cache(int this_cpu, int that_cpu)
2389{
2390	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
2391}
2392#endif /* CONFIG_SMP */
2393
2394static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
2395{
2396	struct rq *rq = cpu_rq(cpu);
2397	struct rq_flags rf;
2398
2399#if defined(CONFIG_SMP)
2400	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
2401		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
2402		ttwu_queue_remote(p, cpu, wake_flags);
2403		return;
2404	}
2405#endif
2406
2407	rq_lock(rq, &rf);
2408	update_rq_clock(rq);
2409	ttwu_do_activate(rq, p, wake_flags, &rf);
2410	rq_unlock(rq, &rf);
2411}
2412
2413/*
2414 * Notes on Program-Order guarantees on SMP systems.
2415 *
2416 *  MIGRATION
2417 *
2418 * The basic program-order guarantee on SMP systems is that when a task [t]
2419 * migrates, all its activity on its old CPU [c0] happens-before any subsequent
2420 * execution on its new CPU [c1].
2421 *
2422 * For migration (of runnable tasks) this is provided by the following means:
2423 *
2424 *  A) UNLOCK of the rq(c0)->lock scheduling out task t
2425 *  B) migration for t is required to synchronize *both* rq(c0)->lock and
2426 *     rq(c1)->lock (if not at the same time, then in that order).
2427 *  C) LOCK of the rq(c1)->lock scheduling in task
2428 *
2429 * Release/acquire chaining guarantees that B happens after A and C after B.
 
2430 * Note: the CPU doing B need not be c0 or c1
2431 *
2432 * Example:
2433 *
2434 *   CPU0            CPU1            CPU2
2435 *
2436 *   LOCK rq(0)->lock
2437 *   sched-out X
2438 *   sched-in Y
2439 *   UNLOCK rq(0)->lock
2440 *
2441 *                                   LOCK rq(0)->lock // orders against CPU0
2442 *                                   dequeue X
2443 *                                   UNLOCK rq(0)->lock
2444 *
2445 *                                   LOCK rq(1)->lock
2446 *                                   enqueue X
2447 *                                   UNLOCK rq(1)->lock
2448 *
2449 *                   LOCK rq(1)->lock // orders against CPU2
2450 *                   sched-out Z
2451 *                   sched-in X
2452 *                   UNLOCK rq(1)->lock
2453 *
2454 *
2455 *  BLOCKING -- aka. SLEEP + WAKEUP
2456 *
2457 * For blocking we (obviously) need to provide the same guarantee as for
2458 * migration. However the means are completely different as there is no lock
2459 * chain to provide order. Instead we do:
2460 *
2461 *   1) smp_store_release(X->on_cpu, 0)
2462 *   2) smp_cond_load_acquire(!X->on_cpu)
2463 *
2464 * Example:
2465 *
2466 *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
2467 *
2468 *   LOCK rq(0)->lock LOCK X->pi_lock
2469 *   dequeue X
2470 *   sched-out X
2471 *   smp_store_release(X->on_cpu, 0);
2472 *
2473 *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
2474 *                    X->state = WAKING
2475 *                    set_task_cpu(X,2)
2476 *
2477 *                    LOCK rq(2)->lock
2478 *                    enqueue X
2479 *                    X->state = RUNNING
2480 *                    UNLOCK rq(2)->lock
2481 *
2482 *                                          LOCK rq(2)->lock // orders against CPU1
2483 *                                          sched-out Z
2484 *                                          sched-in X
2485 *                                          UNLOCK rq(2)->lock
2486 *
2487 *                    UNLOCK X->pi_lock
2488 *   UNLOCK rq(0)->lock
2489 *
2490 *
2491 * However, for wakeups there is a second guarantee we must provide, namely we
2492 * must ensure that CONDITION=1 done by the caller can not be reordered with
2493 * accesses to the task state; see try_to_wake_up() and set_current_state().
 
 
 
 
 
 
 
2494 */
2495
2496/**
2497 * try_to_wake_up - wake up a thread
2498 * @p: the thread to be awakened
2499 * @state: the mask of task states that can be woken
2500 * @wake_flags: wake modifier flags (WF_*)
2501 *
2502 * If (@state & @p->state) @p->state = TASK_RUNNING.
2503 *
2504 * If the task was not queued/runnable, also place it back on a runqueue.
2505 *
2506 * Atomic against schedule() which would dequeue a task, also see
2507 * set_current_state().
2508 *
2509 * This function executes a full memory barrier before accessing the task
2510 * state; see set_current_state().
2511 *
2512 * Return: %true if @p->state changes (an actual wakeup was done),
2513 *	   %false otherwise.
2514 */
2515static int
2516try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2517{
2518	unsigned long flags;
2519	int cpu, success = 0;
2520
2521	preempt_disable();
2522	if (p == current) {
2523		/*
2524		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
2525		 * == smp_processor_id()'. Together this means we can special
2526		 * case the whole 'p->on_rq && ttwu_remote()' case below
2527		 * without taking any locks.
2528		 *
2529		 * In particular:
2530		 *  - we rely on Program-Order guarantees for all the ordering,
2531		 *  - we're serialized against set_special_state() by virtue of
2532		 *    it disabling IRQs (this allows not taking ->pi_lock).
2533		 */
2534		if (!(p->state & state))
2535			goto out;
2536
2537		success = 1;
2538		cpu = task_cpu(p);
2539		trace_sched_waking(p);
2540		p->state = TASK_RUNNING;
2541		trace_sched_wakeup(p);
2542		goto out;
2543	}
2544
2545	/*
2546	 * If we are going to wake up a thread waiting for CONDITION we
2547	 * need to ensure that CONDITION=1 done by the caller can not be
2548	 * reordered with p->state check below. This pairs with mb() in
2549	 * set_current_state() the waiting thread does.
2550	 */
2551	raw_spin_lock_irqsave(&p->pi_lock, flags);
2552	smp_mb__after_spinlock();
2553	if (!(p->state & state))
2554		goto unlock;
2555
2556	trace_sched_waking(p);
2557
2558	/* We're going to change ->state: */
2559	success = 1;
2560	cpu = task_cpu(p);
2561
2562	/*
2563	 * Ensure we load p->on_rq _after_ p->state, otherwise it would
2564	 * be possible to, falsely, observe p->on_rq == 0 and get stuck
2565	 * in smp_cond_load_acquire() below.
2566	 *
2567	 * sched_ttwu_pending()			try_to_wake_up()
2568	 *   STORE p->on_rq = 1			  LOAD p->state
2569	 *   UNLOCK rq->lock
2570	 *
2571	 * __schedule() (switch to task 'p')
2572	 *   LOCK rq->lock			  smp_rmb();
2573	 *   smp_mb__after_spinlock();
2574	 *   UNLOCK rq->lock
2575	 *
2576	 * [task p]
2577	 *   STORE p->state = UNINTERRUPTIBLE	  LOAD p->on_rq
2578	 *
2579	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
2580	 * __schedule().  See the comment for smp_mb__after_spinlock().
 
2581	 */
2582	smp_rmb();
2583	if (p->on_rq && ttwu_remote(p, wake_flags))
2584		goto unlock;
2585
2586#ifdef CONFIG_SMP
2587	/*
2588	 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
2589	 * possible to, falsely, observe p->on_cpu == 0.
2590	 *
2591	 * One must be running (->on_cpu == 1) in order to remove oneself
2592	 * from the runqueue.
2593	 *
2594	 * __schedule() (switch to task 'p')	try_to_wake_up()
2595	 *   STORE p->on_cpu = 1		  LOAD p->on_rq
2596	 *   UNLOCK rq->lock
2597	 *
2598	 * __schedule() (put 'p' to sleep)
2599	 *   LOCK rq->lock			  smp_rmb();
2600	 *   smp_mb__after_spinlock();
2601	 *   STORE p->on_rq = 0			  LOAD p->on_cpu
2602	 *
2603	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
2604	 * __schedule().  See the comment for smp_mb__after_spinlock().
2605	 */
2606	smp_rmb();
2607
2608	/*
2609	 * If the owning (remote) CPU is still in the middle of schedule() with
2610	 * this task as prev, wait until its done referencing the task.
2611	 *
2612	 * Pairs with the smp_store_release() in finish_task().
2613	 *
2614	 * This ensures that tasks getting woken will be fully ordered against
2615	 * their previous state and preserve Program Order.
2616	 */
2617	smp_cond_load_acquire(&p->on_cpu, !VAL);
2618
2619	p->sched_contributes_to_load = !!task_contributes_to_load(p);
2620	p->state = TASK_WAKING;
2621
2622	if (p->in_iowait) {
2623		delayacct_blkio_end(p);
2624		atomic_dec(&task_rq(p)->nr_iowait);
2625	}
2626
2627	cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
2628	if (task_cpu(p) != cpu) {
2629		wake_flags |= WF_MIGRATED;
2630		psi_ttwu_dequeue(p);
2631		set_task_cpu(p, cpu);
2632	}
2633
2634#else /* CONFIG_SMP */
2635
2636	if (p->in_iowait) {
2637		delayacct_blkio_end(p);
2638		atomic_dec(&task_rq(p)->nr_iowait);
2639	}
2640
2641#endif /* CONFIG_SMP */
2642
2643	ttwu_queue(p, cpu, wake_flags);
2644unlock:
2645	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2646out:
2647	if (success)
2648		ttwu_stat(p, cpu, wake_flags);
2649	preempt_enable();
2650
2651	return success;
2652}
2653
2654/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2655 * wake_up_process - Wake up a specific process
2656 * @p: The process to be woken up.
2657 *
2658 * Attempt to wake up the nominated process and move it to the set of runnable
2659 * processes.
2660 *
2661 * Return: 1 if the process was woken up, 0 if it was already running.
2662 *
2663 * This function executes a full memory barrier before accessing the task state.
 
2664 */
2665int wake_up_process(struct task_struct *p)
2666{
2667	return try_to_wake_up(p, TASK_NORMAL, 0);
2668}
2669EXPORT_SYMBOL(wake_up_process);
2670
2671int wake_up_state(struct task_struct *p, unsigned int state)
2672{
2673	return try_to_wake_up(p, state, 0);
2674}
2675
2676/*
2677 * Perform scheduler related setup for a newly forked process p.
2678 * p is forked by current.
2679 *
2680 * __sched_fork() is basic setup used by init_idle() too:
2681 */
2682static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
2683{
2684	p->on_rq			= 0;
2685
2686	p->se.on_rq			= 0;
2687	p->se.exec_start		= 0;
2688	p->se.sum_exec_runtime		= 0;
2689	p->se.prev_sum_exec_runtime	= 0;
2690	p->se.nr_migrations		= 0;
2691	p->se.vruntime			= 0;
2692	INIT_LIST_HEAD(&p->se.group_node);
2693
2694#ifdef CONFIG_FAIR_GROUP_SCHED
2695	p->se.cfs_rq			= NULL;
2696#endif
2697
2698#ifdef CONFIG_SCHEDSTATS
2699	/* Even if schedstat is disabled, there should not be garbage */
2700	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
2701#endif
2702
2703	RB_CLEAR_NODE(&p->dl.rb_node);
2704	init_dl_task_timer(&p->dl);
2705	init_dl_inactive_task_timer(&p->dl);
2706	__dl_clear_params(p);
2707
2708	INIT_LIST_HEAD(&p->rt.run_list);
2709	p->rt.timeout		= 0;
2710	p->rt.time_slice	= sched_rr_timeslice;
2711	p->rt.on_rq		= 0;
2712	p->rt.on_list		= 0;
2713
2714#ifdef CONFIG_PREEMPT_NOTIFIERS
2715	INIT_HLIST_HEAD(&p->preempt_notifiers);
2716#endif
2717
2718#ifdef CONFIG_COMPACTION
2719	p->capture_control = NULL;
2720#endif
2721	init_numa_balancing(clone_flags, p);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2722}
2723
2724DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
2725
2726#ifdef CONFIG_NUMA_BALANCING
2727
2728void set_numabalancing_state(bool enabled)
2729{
2730	if (enabled)
2731		static_branch_enable(&sched_numa_balancing);
2732	else
2733		static_branch_disable(&sched_numa_balancing);
2734}
2735
2736#ifdef CONFIG_PROC_SYSCTL
2737int sysctl_numa_balancing(struct ctl_table *table, int write,
2738			 void __user *buffer, size_t *lenp, loff_t *ppos)
2739{
2740	struct ctl_table t;
2741	int err;
2742	int state = static_branch_likely(&sched_numa_balancing);
2743
2744	if (write && !capable(CAP_SYS_ADMIN))
2745		return -EPERM;
2746
2747	t = *table;
2748	t.data = &state;
2749	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2750	if (err < 0)
2751		return err;
2752	if (write)
2753		set_numabalancing_state(state);
2754	return err;
2755}
2756#endif
2757#endif
2758
2759#ifdef CONFIG_SCHEDSTATS
2760
2761DEFINE_STATIC_KEY_FALSE(sched_schedstats);
2762static bool __initdata __sched_schedstats = false;
2763
2764static void set_schedstats(bool enabled)
2765{
2766	if (enabled)
2767		static_branch_enable(&sched_schedstats);
2768	else
2769		static_branch_disable(&sched_schedstats);
2770}
2771
2772void force_schedstat_enabled(void)
2773{
2774	if (!schedstat_enabled()) {
2775		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
2776		static_branch_enable(&sched_schedstats);
2777	}
2778}
2779
2780static int __init setup_schedstats(char *str)
2781{
2782	int ret = 0;
2783	if (!str)
2784		goto out;
2785
2786	/*
2787	 * This code is called before jump labels have been set up, so we can't
2788	 * change the static branch directly just yet.  Instead set a temporary
2789	 * variable so init_schedstats() can do it later.
2790	 */
2791	if (!strcmp(str, "enable")) {
2792		__sched_schedstats = true;
2793		ret = 1;
2794	} else if (!strcmp(str, "disable")) {
2795		__sched_schedstats = false;
2796		ret = 1;
2797	}
2798out:
2799	if (!ret)
2800		pr_warn("Unable to parse schedstats=\n");
2801
2802	return ret;
2803}
2804__setup("schedstats=", setup_schedstats);
2805
2806static void __init init_schedstats(void)
2807{
2808	set_schedstats(__sched_schedstats);
2809}
2810
2811#ifdef CONFIG_PROC_SYSCTL
2812int sysctl_schedstats(struct ctl_table *table, int write,
2813			 void __user *buffer, size_t *lenp, loff_t *ppos)
2814{
2815	struct ctl_table t;
2816	int err;
2817	int state = static_branch_likely(&sched_schedstats);
2818
2819	if (write && !capable(CAP_SYS_ADMIN))
2820		return -EPERM;
2821
2822	t = *table;
2823	t.data = &state;
2824	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2825	if (err < 0)
2826		return err;
2827	if (write)
2828		set_schedstats(state);
2829	return err;
2830}
2831#endif /* CONFIG_PROC_SYSCTL */
2832#else  /* !CONFIG_SCHEDSTATS */
2833static inline void init_schedstats(void) {}
2834#endif /* CONFIG_SCHEDSTATS */
2835
2836/*
2837 * fork()/clone()-time setup:
2838 */
2839int sched_fork(unsigned long clone_flags, struct task_struct *p)
2840{
2841	unsigned long flags;
 
2842
2843	__sched_fork(clone_flags, p);
2844	/*
2845	 * We mark the process as NEW here. This guarantees that
2846	 * nobody will actually run it, and a signal or other external
2847	 * event cannot wake it up and insert it on the runqueue either.
2848	 */
2849	p->state = TASK_NEW;
2850
2851	/*
2852	 * Make sure we do not leak PI boosting priority to the child.
2853	 */
2854	p->prio = current->normal_prio;
2855
2856	uclamp_fork(p);
2857
2858	/*
2859	 * Revert to default priority/policy on fork if requested.
2860	 */
2861	if (unlikely(p->sched_reset_on_fork)) {
2862		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
2863			p->policy = SCHED_NORMAL;
2864			p->static_prio = NICE_TO_PRIO(0);
2865			p->rt_priority = 0;
2866		} else if (PRIO_TO_NICE(p->static_prio) < 0)
2867			p->static_prio = NICE_TO_PRIO(0);
2868
2869		p->prio = p->normal_prio = __normal_prio(p);
2870		set_load_weight(p, false);
2871
2872		/*
2873		 * We don't need the reset flag anymore after the fork. It has
2874		 * fulfilled its duty:
2875		 */
2876		p->sched_reset_on_fork = 0;
2877	}
2878
2879	if (dl_prio(p->prio))
 
2880		return -EAGAIN;
2881	else if (rt_prio(p->prio))
2882		p->sched_class = &rt_sched_class;
2883	else
2884		p->sched_class = &fair_sched_class;
 
2885
2886	init_entity_runnable_average(&p->se);
2887
2888	/*
2889	 * The child is not yet in the pid-hash so no cgroup attach races,
2890	 * and the cgroup is pinned to this child due to cgroup_fork()
2891	 * is ran before sched_fork().
2892	 *
2893	 * Silence PROVE_RCU.
2894	 */
2895	raw_spin_lock_irqsave(&p->pi_lock, flags);
2896	/*
2897	 * We're setting the CPU for the first time, we don't migrate,
2898	 * so use __set_task_cpu().
2899	 */
2900	__set_task_cpu(p, smp_processor_id());
2901	if (p->sched_class->task_fork)
2902		p->sched_class->task_fork(p);
2903	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2904
2905#ifdef CONFIG_SCHED_INFO
2906	if (likely(sched_info_on()))
2907		memset(&p->sched_info, 0, sizeof(p->sched_info));
2908#endif
2909#if defined(CONFIG_SMP)
2910	p->on_cpu = 0;
2911#endif
2912	init_task_preempt_count(p);
2913#ifdef CONFIG_SMP
2914	plist_node_init(&p->pushable_tasks, MAX_PRIO);
2915	RB_CLEAR_NODE(&p->pushable_dl_tasks);
2916#endif
 
 
2917	return 0;
2918}
2919
2920unsigned long to_ratio(u64 period, u64 runtime)
2921{
2922	if (runtime == RUNTIME_INF)
2923		return BW_UNIT;
2924
2925	/*
2926	 * Doing this here saves a lot of checks in all
2927	 * the calling paths, and returning zero seems
2928	 * safe for them anyway.
2929	 */
2930	if (period == 0)
2931		return 0;
2932
2933	return div64_u64(runtime << BW_SHIFT, period);
2934}
2935
2936/*
2937 * wake_up_new_task - wake up a newly created task for the first time.
2938 *
2939 * This function will do some initial scheduler statistics housekeeping
2940 * that must be done for every newly created context, then puts the task
2941 * on the runqueue and wakes it.
2942 */
2943void wake_up_new_task(struct task_struct *p)
2944{
2945	struct rq_flags rf;
2946	struct rq *rq;
2947
2948	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
2949	p->state = TASK_RUNNING;
2950#ifdef CONFIG_SMP
2951	/*
2952	 * Fork balancing, do it here and not earlier because:
2953	 *  - cpus_ptr can change in the fork path
2954	 *  - any previously selected CPU might disappear through hotplug
2955	 *
2956	 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
2957	 * as we're not fully set-up yet.
2958	 */
2959	p->recent_used_cpu = task_cpu(p);
2960	__set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2961#endif
2962	rq = __task_rq_lock(p, &rf);
2963	update_rq_clock(rq);
2964	post_init_entity_util_avg(p);
2965
2966	activate_task(rq, p, ENQUEUE_NOCLOCK);
 
2967	trace_sched_wakeup_new(p);
2968	check_preempt_curr(rq, p, WF_FORK);
2969#ifdef CONFIG_SMP
2970	if (p->sched_class->task_woken) {
2971		/*
2972		 * Nothing relies on rq->lock after this, so its fine to
2973		 * drop it.
2974		 */
2975		rq_unpin_lock(rq, &rf);
2976		p->sched_class->task_woken(rq, p);
2977		rq_repin_lock(rq, &rf);
2978	}
2979#endif
2980	task_rq_unlock(rq, p, &rf);
2981}
2982
2983#ifdef CONFIG_PREEMPT_NOTIFIERS
2984
2985static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
2986
2987void preempt_notifier_inc(void)
2988{
2989	static_branch_inc(&preempt_notifier_key);
2990}
2991EXPORT_SYMBOL_GPL(preempt_notifier_inc);
2992
2993void preempt_notifier_dec(void)
2994{
2995	static_branch_dec(&preempt_notifier_key);
2996}
2997EXPORT_SYMBOL_GPL(preempt_notifier_dec);
2998
2999/**
3000 * preempt_notifier_register - tell me when current is being preempted & rescheduled
3001 * @notifier: notifier struct to register
3002 */
3003void preempt_notifier_register(struct preempt_notifier *notifier)
3004{
3005	if (!static_branch_unlikely(&preempt_notifier_key))
3006		WARN(1, "registering preempt_notifier while notifiers disabled\n");
3007
3008	hlist_add_head(&notifier->link, &current->preempt_notifiers);
3009}
3010EXPORT_SYMBOL_GPL(preempt_notifier_register);
3011
3012/**
3013 * preempt_notifier_unregister - no longer interested in preemption notifications
3014 * @notifier: notifier struct to unregister
3015 *
3016 * This is *not* safe to call from within a preemption notifier.
3017 */
3018void preempt_notifier_unregister(struct preempt_notifier *notifier)
3019{
3020	hlist_del(&notifier->link);
3021}
3022EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
3023
3024static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
3025{
3026	struct preempt_notifier *notifier;
3027
3028	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3029		notifier->ops->sched_in(notifier, raw_smp_processor_id());
3030}
3031
3032static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3033{
3034	if (static_branch_unlikely(&preempt_notifier_key))
3035		__fire_sched_in_preempt_notifiers(curr);
3036}
3037
3038static void
3039__fire_sched_out_preempt_notifiers(struct task_struct *curr,
3040				   struct task_struct *next)
3041{
3042	struct preempt_notifier *notifier;
3043
3044	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
3045		notifier->ops->sched_out(notifier, next);
3046}
3047
3048static __always_inline void
3049fire_sched_out_preempt_notifiers(struct task_struct *curr,
3050				 struct task_struct *next)
3051{
3052	if (static_branch_unlikely(&preempt_notifier_key))
3053		__fire_sched_out_preempt_notifiers(curr, next);
3054}
3055
3056#else /* !CONFIG_PREEMPT_NOTIFIERS */
3057
3058static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
3059{
3060}
3061
3062static inline void
3063fire_sched_out_preempt_notifiers(struct task_struct *curr,
3064				 struct task_struct *next)
3065{
3066}
3067
3068#endif /* CONFIG_PREEMPT_NOTIFIERS */
3069
3070static inline void prepare_task(struct task_struct *next)
3071{
3072#ifdef CONFIG_SMP
3073	/*
3074	 * Claim the task as running, we do this before switching to it
3075	 * such that any running task will have this set.
3076	 */
3077	next->on_cpu = 1;
3078#endif
3079}
3080
3081static inline void finish_task(struct task_struct *prev)
3082{
3083#ifdef CONFIG_SMP
3084	/*
3085	 * After ->on_cpu is cleared, the task can be moved to a different CPU.
3086	 * We must ensure this doesn't happen until the switch is completely
3087	 * finished.
3088	 *
3089	 * In particular, the load of prev->state in finish_task_switch() must
3090	 * happen before this.
3091	 *
3092	 * Pairs with the smp_cond_load_acquire() in try_to_wake_up().
3093	 */
3094	smp_store_release(&prev->on_cpu, 0);
3095#endif
3096}
3097
3098static inline void
3099prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
3100{
3101	/*
3102	 * Since the runqueue lock will be released by the next
3103	 * task (which is an invalid locking op but in the case
3104	 * of the scheduler it's an obvious special-case), so we
3105	 * do an early lockdep release here:
3106	 */
3107	rq_unpin_lock(rq, rf);
3108	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
3109#ifdef CONFIG_DEBUG_SPINLOCK
3110	/* this is a valid case when another task releases the spinlock */
3111	rq->lock.owner = next;
3112#endif
3113}
3114
3115static inline void finish_lock_switch(struct rq *rq)
3116{
3117	/*
3118	 * If we are tracking spinlock dependencies then we have to
3119	 * fix up the runqueue lock - which gets 'carried over' from
3120	 * prev into current:
3121	 */
3122	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
3123	raw_spin_unlock_irq(&rq->lock);
3124}
3125
3126/*
3127 * NOP if the arch has not defined these:
3128 */
3129
3130#ifndef prepare_arch_switch
3131# define prepare_arch_switch(next)	do { } while (0)
3132#endif
3133
3134#ifndef finish_arch_post_lock_switch
3135# define finish_arch_post_lock_switch()	do { } while (0)
3136#endif
3137
3138/**
3139 * prepare_task_switch - prepare to switch tasks
3140 * @rq: the runqueue preparing to switch
3141 * @prev: the current task that is being switched out
3142 * @next: the task we are going to switch to.
3143 *
3144 * This is called with the rq lock held and interrupts off. It must
3145 * be paired with a subsequent finish_task_switch after the context
3146 * switch.
3147 *
3148 * prepare_task_switch sets up locking and calls architecture specific
3149 * hooks.
3150 */
3151static inline void
3152prepare_task_switch(struct rq *rq, struct task_struct *prev,
3153		    struct task_struct *next)
3154{
3155	kcov_prepare_switch(prev);
3156	sched_info_switch(rq, prev, next);
3157	perf_event_task_sched_out(prev, next);
3158	rseq_preempt(prev);
3159	fire_sched_out_preempt_notifiers(prev, next);
3160	prepare_task(next);
3161	prepare_arch_switch(next);
3162}
3163
3164/**
3165 * finish_task_switch - clean up after a task-switch
3166 * @prev: the thread we just switched away from.
3167 *
3168 * finish_task_switch must be called after the context switch, paired
3169 * with a prepare_task_switch call before the context switch.
3170 * finish_task_switch will reconcile locking set up by prepare_task_switch,
3171 * and do any other architecture-specific cleanup actions.
3172 *
3173 * Note that we may have delayed dropping an mm in context_switch(). If
3174 * so, we finish that here outside of the runqueue lock. (Doing it
3175 * with the lock held can cause deadlocks; see schedule() for
3176 * details.)
3177 *
3178 * The context switch have flipped the stack from under us and restored the
3179 * local variables which were saved when this task called schedule() in the
3180 * past. prev == current is still correct but we need to recalculate this_rq
3181 * because prev may have moved to another CPU.
3182 */
3183static struct rq *finish_task_switch(struct task_struct *prev)
3184	__releases(rq->lock)
3185{
3186	struct rq *rq = this_rq();
3187	struct mm_struct *mm = rq->prev_mm;
3188	long prev_state;
3189
3190	/*
3191	 * The previous task will have left us with a preempt_count of 2
3192	 * because it left us after:
3193	 *
3194	 *	schedule()
3195	 *	  preempt_disable();			// 1
3196	 *	  __schedule()
3197	 *	    raw_spin_lock_irq(&rq->lock)	// 2
3198	 *
3199	 * Also, see FORK_PREEMPT_COUNT.
3200	 */
3201	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
3202		      "corrupted preempt_count: %s/%d/0x%x\n",
3203		      current->comm, current->pid, preempt_count()))
3204		preempt_count_set(FORK_PREEMPT_COUNT);
3205
3206	rq->prev_mm = NULL;
3207
3208	/*
3209	 * A task struct has one reference for the use as "current".
3210	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
3211	 * schedule one last time. The schedule call will never return, and
3212	 * the scheduled task must drop that reference.
3213	 *
3214	 * We must observe prev->state before clearing prev->on_cpu (in
3215	 * finish_task), otherwise a concurrent wakeup can get prev
3216	 * running on another CPU and we could rave with its RUNNING -> DEAD
3217	 * transition, resulting in a double drop.
3218	 */
3219	prev_state = prev->state;
3220	vtime_task_switch(prev);
3221	perf_event_task_sched_in(prev, current);
3222	finish_task(prev);
3223	finish_lock_switch(rq);
3224	finish_arch_post_lock_switch();
3225	kcov_finish_switch(current);
3226
3227	fire_sched_in_preempt_notifiers(current);
3228	/*
3229	 * When switching through a kernel thread, the loop in
3230	 * membarrier_{private,global}_expedited() may have observed that
3231	 * kernel thread and not issued an IPI. It is therefore possible to
3232	 * schedule between user->kernel->user threads without passing though
3233	 * switch_mm(). Membarrier requires a barrier after storing to
3234	 * rq->curr, before returning to userspace, so provide them here:
3235	 *
3236	 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
3237	 *   provided by mmdrop(),
3238	 * - a sync_core for SYNC_CORE.
3239	 */
3240	if (mm) {
3241		membarrier_mm_sync_core_before_usermode(mm);
3242		mmdrop(mm);
3243	}
3244	if (unlikely(prev_state == TASK_DEAD)) {
3245		if (prev->sched_class->task_dead)
3246			prev->sched_class->task_dead(prev);
 
 
3247
3248		/*
3249		 * Remove function-return probe instances associated with this
3250		 * task and put them back on the free list.
3251		 */
3252		kprobe_flush_task(prev);
3253
3254		/* Task is done with its stack. */
3255		put_task_stack(prev);
3256
3257		put_task_struct_rcu_user(prev);
 
 
 
 
 
 
3258	}
3259
3260	tick_nohz_task_switch();
3261	return rq;
3262}
3263
3264#ifdef CONFIG_SMP
3265
3266/* rq->lock is NOT held, but preemption is disabled */
3267static void __balance_callback(struct rq *rq)
3268{
3269	struct callback_head *head, *next;
3270	void (*func)(struct rq *rq);
3271	unsigned long flags;
3272
3273	raw_spin_lock_irqsave(&rq->lock, flags);
3274	head = rq->balance_callback;
3275	rq->balance_callback = NULL;
3276	while (head) {
3277		func = (void (*)(struct rq *))head->func;
3278		next = head->next;
3279		head->next = NULL;
3280		head = next;
3281
3282		func(rq);
3283	}
3284	raw_spin_unlock_irqrestore(&rq->lock, flags);
3285}
3286
3287static inline void balance_callback(struct rq *rq)
3288{
3289	if (unlikely(rq->balance_callback))
3290		__balance_callback(rq);
3291}
3292
3293#else
3294
3295static inline void balance_callback(struct rq *rq)
3296{
3297}
3298
3299#endif
3300
3301/**
3302 * schedule_tail - first thing a freshly forked thread must call.
3303 * @prev: the thread we just switched away from.
3304 */
3305asmlinkage __visible void schedule_tail(struct task_struct *prev)
3306	__releases(rq->lock)
3307{
3308	struct rq *rq;
3309
3310	/*
3311	 * New tasks start with FORK_PREEMPT_COUNT, see there and
3312	 * finish_task_switch() for details.
3313	 *
3314	 * finish_task_switch() will drop rq->lock() and lower preempt_count
3315	 * and the preempt_enable() will end up enabling preemption (on
3316	 * PREEMPT_COUNT kernels).
3317	 */
3318
3319	rq = finish_task_switch(prev);
3320	balance_callback(rq);
3321	preempt_enable();
3322
3323	if (current->set_child_tid)
3324		put_user(task_pid_vnr(current), current->set_child_tid);
3325
3326	calculate_sigpending();
3327}
3328
3329/*
3330 * context_switch - switch to the new MM and the new thread's register state.
3331 */
3332static __always_inline struct rq *
3333context_switch(struct rq *rq, struct task_struct *prev,
3334	       struct task_struct *next, struct rq_flags *rf)
3335{
 
 
3336	prepare_task_switch(rq, prev, next);
3337
 
 
3338	/*
3339	 * For paravirt, this is coupled with an exit in switch_to to
3340	 * combine the page table reload and the switch backend into
3341	 * one hypercall.
3342	 */
3343	arch_start_context_switch(prev);
3344
3345	/*
3346	 * kernel -> kernel   lazy + transfer active
3347	 *   user -> kernel   lazy + mmgrab() active
3348	 *
3349	 * kernel ->   user   switch + mmdrop() active
3350	 *   user ->   user   switch
3351	 */
3352	if (!next->mm) {                                // to kernel
3353		enter_lazy_tlb(prev->active_mm, next);
3354
3355		next->active_mm = prev->active_mm;
3356		if (prev->mm)                           // from user
3357			mmgrab(prev->active_mm);
3358		else
3359			prev->active_mm = NULL;
3360	} else {                                        // to user
3361		membarrier_switch_mm(rq, prev->active_mm, next->mm);
3362		/*
3363		 * sys_membarrier() requires an smp_mb() between setting
3364		 * rq->curr / membarrier_switch_mm() and returning to userspace.
3365		 *
3366		 * The below provides this either through switch_mm(), or in
3367		 * case 'prev->active_mm == next->mm' through
3368		 * finish_task_switch()'s mmdrop().
3369		 */
3370		switch_mm_irqs_off(prev->active_mm, next->mm, next);
3371
3372		if (!prev->mm) {                        // from kernel
3373			/* will mmdrop() in finish_task_switch(). */
3374			rq->prev_mm = prev->active_mm;
3375			prev->active_mm = NULL;
3376		}
3377	}
3378
3379	rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
3380
3381	prepare_lock_switch(rq, next, rf);
3382
3383	/* Here we just switch the register state and the stack. */
3384	switch_to(prev, next, prev);
3385	barrier();
3386
3387	return finish_task_switch(prev);
3388}
3389
3390/*
3391 * nr_running and nr_context_switches:
3392 *
3393 * externally visible scheduler statistics: current number of runnable
3394 * threads, total number of context switches performed since bootup.
3395 */
3396unsigned long nr_running(void)
3397{
3398	unsigned long i, sum = 0;
3399
3400	for_each_online_cpu(i)
3401		sum += cpu_rq(i)->nr_running;
3402
3403	return sum;
3404}
3405
3406/*
3407 * Check if only the current task is running on the CPU.
3408 *
3409 * Caution: this function does not check that the caller has disabled
3410 * preemption, thus the result might have a time-of-check-to-time-of-use
3411 * race.  The caller is responsible to use it correctly, for example:
3412 *
3413 * - from a non-preemptible section (of course)
3414 *
3415 * - from a thread that is bound to a single CPU
3416 *
3417 * - in a loop with very short iterations (e.g. a polling loop)
3418 */
3419bool single_task_running(void)
3420{
3421	return raw_rq()->nr_running == 1;
3422}
3423EXPORT_SYMBOL(single_task_running);
3424
3425unsigned long long nr_context_switches(void)
3426{
3427	int i;
3428	unsigned long long sum = 0;
3429
3430	for_each_possible_cpu(i)
3431		sum += cpu_rq(i)->nr_switches;
3432
3433	return sum;
3434}
3435
3436/*
3437 * Consumers of these two interfaces, like for example the cpuidle menu
3438 * governor, are using nonsensical data. Preferring shallow idle state selection
3439 * for a CPU that has IO-wait which might not even end up running the task when
3440 * it does become runnable.
3441 */
3442
3443unsigned long nr_iowait_cpu(int cpu)
3444{
3445	return atomic_read(&cpu_rq(cpu)->nr_iowait);
3446}
3447
3448/*
3449 * IO-wait accounting, and how its mostly bollocks (on SMP).
3450 *
3451 * The idea behind IO-wait account is to account the idle time that we could
3452 * have spend running if it were not for IO. That is, if we were to improve the
3453 * storage performance, we'd have a proportional reduction in IO-wait time.
3454 *
3455 * This all works nicely on UP, where, when a task blocks on IO, we account
3456 * idle time as IO-wait, because if the storage were faster, it could've been
3457 * running and we'd not be idle.
3458 *
3459 * This has been extended to SMP, by doing the same for each CPU. This however
3460 * is broken.
3461 *
3462 * Imagine for instance the case where two tasks block on one CPU, only the one
3463 * CPU will have IO-wait accounted, while the other has regular idle. Even
3464 * though, if the storage were faster, both could've ran at the same time,
3465 * utilising both CPUs.
3466 *
3467 * This means, that when looking globally, the current IO-wait accounting on
3468 * SMP is a lower bound, by reason of under accounting.
3469 *
3470 * Worse, since the numbers are provided per CPU, they are sometimes
3471 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
3472 * associated with any one particular CPU, it can wake to another CPU than it
3473 * blocked on. This means the per CPU IO-wait number is meaningless.
3474 *
3475 * Task CPU affinities can make all that even more 'interesting'.
3476 */
3477
3478unsigned long nr_iowait(void)
3479{
3480	unsigned long i, sum = 0;
3481
3482	for_each_possible_cpu(i)
3483		sum += nr_iowait_cpu(i);
3484
3485	return sum;
3486}
3487
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3488#ifdef CONFIG_SMP
3489
3490/*
3491 * sched_exec - execve() is a valuable balancing opportunity, because at
3492 * this point the task has the smallest effective memory and cache footprint.
3493 */
3494void sched_exec(void)
3495{
3496	struct task_struct *p = current;
3497	unsigned long flags;
3498	int dest_cpu;
3499
3500	raw_spin_lock_irqsave(&p->pi_lock, flags);
3501	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
3502	if (dest_cpu == smp_processor_id())
3503		goto unlock;
3504
3505	if (likely(cpu_active(dest_cpu))) {
3506		struct migration_arg arg = { p, dest_cpu };
3507
3508		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3509		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
3510		return;
3511	}
3512unlock:
3513	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3514}
3515
3516#endif
3517
3518DEFINE_PER_CPU(struct kernel_stat, kstat);
3519DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
3520
3521EXPORT_PER_CPU_SYMBOL(kstat);
3522EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
3523
3524/*
3525 * The function fair_sched_class.update_curr accesses the struct curr
3526 * and its field curr->exec_start; when called from task_sched_runtime(),
3527 * we observe a high rate of cache misses in practice.
3528 * Prefetching this data results in improved performance.
3529 */
3530static inline void prefetch_curr_exec_start(struct task_struct *p)
3531{
3532#ifdef CONFIG_FAIR_GROUP_SCHED
3533	struct sched_entity *curr = (&p->se)->cfs_rq->curr;
3534#else
3535	struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
3536#endif
3537	prefetch(curr);
3538	prefetch(&curr->exec_start);
3539}
3540
3541/*
3542 * Return accounted runtime for the task.
3543 * In case the task is currently running, return the runtime plus current's
3544 * pending runtime that have not been accounted yet.
3545 */
3546unsigned long long task_sched_runtime(struct task_struct *p)
3547{
3548	struct rq_flags rf;
3549	struct rq *rq;
3550	u64 ns;
3551
3552#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
3553	/*
3554	 * 64-bit doesn't need locks to atomically read a 64-bit value.
3555	 * So we have a optimization chance when the task's delta_exec is 0.
3556	 * Reading ->on_cpu is racy, but this is ok.
3557	 *
3558	 * If we race with it leaving CPU, we'll take a lock. So we're correct.
3559	 * If we race with it entering CPU, unaccounted time is 0. This is
3560	 * indistinguishable from the read occurring a few cycles earlier.
3561	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
3562	 * been accounted, so we're correct here as well.
3563	 */
3564	if (!p->on_cpu || !task_on_rq_queued(p))
3565		return p->se.sum_exec_runtime;
3566#endif
3567
3568	rq = task_rq_lock(p, &rf);
3569	/*
3570	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
3571	 * project cycles that may never be accounted to this
3572	 * thread, breaking clock_gettime().
3573	 */
3574	if (task_current(rq, p) && task_on_rq_queued(p)) {
3575		prefetch_curr_exec_start(p);
3576		update_rq_clock(rq);
3577		p->sched_class->update_curr(rq);
3578	}
3579	ns = p->se.sum_exec_runtime;
3580	task_rq_unlock(rq, p, &rf);
3581
3582	return ns;
3583}
3584
3585/*
3586 * This function gets called by the timer code, with HZ frequency.
3587 * We call it with interrupts disabled.
3588 */
3589void scheduler_tick(void)
3590{
3591	int cpu = smp_processor_id();
3592	struct rq *rq = cpu_rq(cpu);
3593	struct task_struct *curr = rq->curr;
3594	struct rq_flags rf;
3595
3596	sched_clock_tick();
3597
3598	rq_lock(rq, &rf);
3599
3600	update_rq_clock(rq);
3601	curr->sched_class->task_tick(rq, curr, 0);
 
3602	calc_global_load_tick(rq);
3603	psi_task_tick(rq);
3604
3605	rq_unlock(rq, &rf);
3606
3607	perf_event_task_tick();
3608
3609#ifdef CONFIG_SMP
3610	rq->idle_balance = idle_cpu(cpu);
3611	trigger_load_balance(rq);
3612#endif
3613}
3614
3615#ifdef CONFIG_NO_HZ_FULL
3616
3617struct tick_work {
3618	int			cpu;
3619	atomic_t		state;
3620	struct delayed_work	work;
3621};
3622/* Values for ->state, see diagram below. */
3623#define TICK_SCHED_REMOTE_OFFLINE	0
3624#define TICK_SCHED_REMOTE_OFFLINING	1
3625#define TICK_SCHED_REMOTE_RUNNING	2
3626
3627/*
3628 * State diagram for ->state:
3629 *
3630 *
3631 *          TICK_SCHED_REMOTE_OFFLINE
3632 *                    |   ^
3633 *                    |   |
3634 *                    |   | sched_tick_remote()
3635 *                    |   |
3636 *                    |   |
3637 *                    +--TICK_SCHED_REMOTE_OFFLINING
3638 *                    |   ^
3639 *                    |   |
3640 * sched_tick_start() |   | sched_tick_stop()
3641 *                    |   |
3642 *                    V   |
3643 *          TICK_SCHED_REMOTE_RUNNING
3644 *
3645 *
3646 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
3647 * and sched_tick_start() are happy to leave the state in RUNNING.
3648 */
3649
3650static struct tick_work __percpu *tick_work_cpu;
3651
3652static void sched_tick_remote(struct work_struct *work)
3653{
3654	struct delayed_work *dwork = to_delayed_work(work);
3655	struct tick_work *twork = container_of(dwork, struct tick_work, work);
3656	int cpu = twork->cpu;
3657	struct rq *rq = cpu_rq(cpu);
3658	struct task_struct *curr;
3659	struct rq_flags rf;
3660	u64 delta;
3661	int os;
3662
3663	/*
3664	 * Handle the tick only if it appears the remote CPU is running in full
3665	 * dynticks mode. The check is racy by nature, but missing a tick or
3666	 * having one too much is no big deal because the scheduler tick updates
3667	 * statistics and checks timeslices in a time-independent way, regardless
3668	 * of when exactly it is running.
3669	 */
3670	if (idle_cpu(cpu) || !tick_nohz_tick_stopped_cpu(cpu))
3671		goto out_requeue;
3672
3673	rq_lock_irq(rq, &rf);
3674	curr = rq->curr;
3675	if (is_idle_task(curr) || cpu_is_offline(cpu))
3676		goto out_unlock;
3677
3678	update_rq_clock(rq);
3679	delta = rq_clock_task(rq) - curr->se.exec_start;
3680
3681	/*
3682	 * Make sure the next tick runs within a reasonable
3683	 * amount of time.
3684	 */
3685	WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3);
3686	curr->sched_class->task_tick(rq, curr, 0);
3687
3688out_unlock:
3689	rq_unlock_irq(rq, &rf);
 
 
 
 
 
 
3690
3691out_requeue:
3692	/*
3693	 * Run the remote tick once per second (1Hz). This arbitrary
3694	 * frequency is large enough to avoid overload but short enough
3695	 * to keep scheduler internal stats reasonably up to date.  But
3696	 * first update state to reflect hotplug activity if required.
3697	 */
3698	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
3699	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
3700	if (os == TICK_SCHED_REMOTE_RUNNING)
3701		queue_delayed_work(system_unbound_wq, dwork, HZ);
3702}
3703
3704static void sched_tick_start(int cpu)
3705{
3706	int os;
3707	struct tick_work *twork;
3708
3709	if (housekeeping_cpu(cpu, HK_FLAG_TICK))
3710		return;
3711
3712	WARN_ON_ONCE(!tick_work_cpu);
3713
3714	twork = per_cpu_ptr(tick_work_cpu, cpu);
3715	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
3716	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
3717	if (os == TICK_SCHED_REMOTE_OFFLINE) {
3718		twork->cpu = cpu;
3719		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
3720		queue_delayed_work(system_unbound_wq, &twork->work, HZ);
3721	}
3722}
3723
3724#ifdef CONFIG_HOTPLUG_CPU
3725static void sched_tick_stop(int cpu)
3726{
3727	struct tick_work *twork;
3728	int os;
3729
3730	if (housekeeping_cpu(cpu, HK_FLAG_TICK))
3731		return;
3732
3733	WARN_ON_ONCE(!tick_work_cpu);
3734
3735	twork = per_cpu_ptr(tick_work_cpu, cpu);
3736	/* There cannot be competing actions, but don't rely on stop-machine. */
3737	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
3738	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
3739	/* Don't cancel, as this would mess up the state machine. */
3740}
3741#endif /* CONFIG_HOTPLUG_CPU */
3742
3743int __init sched_tick_offload_init(void)
3744{
3745	tick_work_cpu = alloc_percpu(struct tick_work);
3746	BUG_ON(!tick_work_cpu);
 
3747	return 0;
3748}
3749
3750#else /* !CONFIG_NO_HZ_FULL */
3751static inline void sched_tick_start(int cpu) { }
3752static inline void sched_tick_stop(int cpu) { }
3753#endif
3754
3755#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
3756				defined(CONFIG_TRACE_PREEMPT_TOGGLE))
3757/*
3758 * If the value passed in is equal to the current preempt count
3759 * then we just disabled preemption. Start timing the latency.
3760 */
3761static inline void preempt_latency_start(int val)
3762{
3763	if (preempt_count() == val) {
3764		unsigned long ip = get_lock_parent_ip();
3765#ifdef CONFIG_DEBUG_PREEMPT
3766		current->preempt_disable_ip = ip;
3767#endif
3768		trace_preempt_off(CALLER_ADDR0, ip);
3769	}
3770}
3771
3772void preempt_count_add(int val)
3773{
3774#ifdef CONFIG_DEBUG_PREEMPT
3775	/*
3776	 * Underflow?
3777	 */
3778	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
3779		return;
3780#endif
3781	__preempt_count_add(val);
3782#ifdef CONFIG_DEBUG_PREEMPT
3783	/*
3784	 * Spinlock count overflowing soon?
3785	 */
3786	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
3787				PREEMPT_MASK - 10);
3788#endif
3789	preempt_latency_start(val);
3790}
3791EXPORT_SYMBOL(preempt_count_add);
3792NOKPROBE_SYMBOL(preempt_count_add);
3793
3794/*
3795 * If the value passed in equals to the current preempt count
3796 * then we just enabled preemption. Stop timing the latency.
3797 */
3798static inline void preempt_latency_stop(int val)
3799{
3800	if (preempt_count() == val)
3801		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
3802}
3803
3804void preempt_count_sub(int val)
3805{
3806#ifdef CONFIG_DEBUG_PREEMPT
3807	/*
3808	 * Underflow?
3809	 */
3810	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
3811		return;
3812	/*
3813	 * Is the spinlock portion underflowing?
3814	 */
3815	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
3816			!(preempt_count() & PREEMPT_MASK)))
3817		return;
3818#endif
3819
3820	preempt_latency_stop(val);
3821	__preempt_count_sub(val);
3822}
3823EXPORT_SYMBOL(preempt_count_sub);
3824NOKPROBE_SYMBOL(preempt_count_sub);
3825
3826#else
3827static inline void preempt_latency_start(int val) { }
3828static inline void preempt_latency_stop(int val) { }
3829#endif
3830
3831static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
3832{
3833#ifdef CONFIG_DEBUG_PREEMPT
3834	return p->preempt_disable_ip;
3835#else
3836	return 0;
3837#endif
3838}
3839
3840/*
3841 * Print scheduling while atomic bug:
3842 */
3843static noinline void __schedule_bug(struct task_struct *prev)
3844{
3845	/* Save this before calling printk(), since that will clobber it */
3846	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
3847
3848	if (oops_in_progress)
3849		return;
3850
3851	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3852		prev->comm, prev->pid, preempt_count());
3853
3854	debug_show_held_locks(prev);
3855	print_modules();
3856	if (irqs_disabled())
3857		print_irqtrace_events(prev);
3858	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
3859	    && in_atomic_preempt_off()) {
3860		pr_err("Preemption disabled at:");
3861		print_ip_sym(preempt_disable_ip);
3862		pr_cont("\n");
3863	}
3864	if (panic_on_warn)
3865		panic("scheduling while atomic\n");
3866
3867	dump_stack();
3868	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
3869}
3870
3871/*
3872 * Various schedule()-time debugging checks and statistics:
3873 */
3874static inline void schedule_debug(struct task_struct *prev, bool preempt)
3875{
3876#ifdef CONFIG_SCHED_STACK_END_CHECK
3877	if (task_stack_end_corrupted(prev))
3878		panic("corrupted stack end detected inside scheduler\n");
3879#endif
3880
3881#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
3882	if (!preempt && prev->state && prev->non_block_count) {
3883		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
3884			prev->comm, prev->pid, prev->non_block_count);
3885		dump_stack();
3886		add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
3887	}
3888#endif
3889
3890	if (unlikely(in_atomic_preempt_off())) {
3891		__schedule_bug(prev);
3892		preempt_count_set(PREEMPT_DISABLED);
3893	}
3894	rcu_sleep_check();
3895
3896	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
3897
3898	schedstat_inc(this_rq()->sched_count);
3899}
3900
3901/*
3902 * Pick up the highest-prio task:
3903 */
3904static inline struct task_struct *
3905pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
3906{
3907	const struct sched_class *class;
3908	struct task_struct *p;
3909
3910	/*
3911	 * Optimization: we know that if all tasks are in the fair class we can
3912	 * call that function directly, but only if the @prev task wasn't of a
3913	 * higher scheduling class, because otherwise those loose the
3914	 * opportunity to pull in more work from other CPUs.
3915	 */
3916	if (likely((prev->sched_class == &idle_sched_class ||
3917		    prev->sched_class == &fair_sched_class) &&
3918		   rq->nr_running == rq->cfs.h_nr_running)) {
3919
3920		p = fair_sched_class.pick_next_task(rq, prev, rf);
3921		if (unlikely(p == RETRY_TASK))
3922			goto restart;
3923
3924		/* Assumes fair_sched_class->next == idle_sched_class */
3925		if (unlikely(!p))
3926			p = idle_sched_class.pick_next_task(rq, prev, rf);
3927
3928		return p;
3929	}
3930
3931restart:
3932#ifdef CONFIG_SMP
3933	/*
3934	 * We must do the balancing pass before put_next_task(), such
3935	 * that when we release the rq->lock the task is in the same
3936	 * state as before we took rq->lock.
3937	 *
3938	 * We can terminate the balance pass as soon as we know there is
3939	 * a runnable task of @class priority or higher.
3940	 */
3941	for_class_range(class, prev->sched_class, &idle_sched_class) {
3942		if (class->balance(rq, prev, rf))
3943			break;
3944	}
3945#endif
3946
3947	put_prev_task(rq, prev);
3948
3949	for_each_class(class) {
3950		p = class->pick_next_task(rq, NULL, NULL);
3951		if (p)
 
 
3952			return p;
 
3953	}
3954
3955	/* The idle class should always have a runnable task: */
3956	BUG();
3957}
3958
3959/*
3960 * __schedule() is the main scheduler function.
3961 *
3962 * The main means of driving the scheduler and thus entering this function are:
3963 *
3964 *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
3965 *
3966 *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
3967 *      paths. For example, see arch/x86/entry_64.S.
3968 *
3969 *      To drive preemption between tasks, the scheduler sets the flag in timer
3970 *      interrupt handler scheduler_tick().
3971 *
3972 *   3. Wakeups don't really cause entry into schedule(). They add a
3973 *      task to the run-queue and that's it.
3974 *
3975 *      Now, if the new task added to the run-queue preempts the current
3976 *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
3977 *      called on the nearest possible occasion:
3978 *
3979 *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
3980 *
3981 *         - in syscall or exception context, at the next outmost
3982 *           preempt_enable(). (this might be as soon as the wake_up()'s
3983 *           spin_unlock()!)
3984 *
3985 *         - in IRQ context, return from interrupt-handler to
3986 *           preemptible context
3987 *
3988 *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
3989 *         then at the next:
3990 *
3991 *          - cond_resched() call
3992 *          - explicit schedule() call
3993 *          - return from syscall or exception to user-space
3994 *          - return from interrupt-handler to user-space
3995 *
3996 * WARNING: must be called with preemption disabled!
3997 */
3998static void __sched notrace __schedule(bool preempt)
3999{
4000	struct task_struct *prev, *next;
4001	unsigned long *switch_count;
4002	struct rq_flags rf;
4003	struct rq *rq;
4004	int cpu;
4005
4006	cpu = smp_processor_id();
4007	rq = cpu_rq(cpu);
4008	prev = rq->curr;
4009
4010	schedule_debug(prev, preempt);
4011
4012	if (sched_feat(HRTICK))
4013		hrtick_clear(rq);
4014
4015	local_irq_disable();
4016	rcu_note_context_switch(preempt);
4017
4018	/*
4019	 * Make sure that signal_pending_state()->signal_pending() below
4020	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
4021	 * done by the caller to avoid the race with signal_wake_up().
4022	 *
4023	 * The membarrier system call requires a full memory barrier
4024	 * after coming from user-space, before storing to rq->curr.
4025	 */
4026	rq_lock(rq, &rf);
4027	smp_mb__after_spinlock();
4028
4029	/* Promote REQ to ACT */
4030	rq->clock_update_flags <<= 1;
4031	update_rq_clock(rq);
4032
4033	switch_count = &prev->nivcsw;
4034	if (!preempt && prev->state) {
4035		if (signal_pending_state(prev->state, prev)) {
4036			prev->state = TASK_RUNNING;
4037		} else {
4038			deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK);
 
4039
4040			if (prev->in_iowait) {
4041				atomic_inc(&rq->nr_iowait);
4042				delayacct_blkio_start();
4043			}
 
 
 
 
 
 
 
 
 
 
 
 
 
4044		}
4045		switch_count = &prev->nvcsw;
4046	}
4047
4048	next = pick_next_task(rq, prev, &rf);
4049	clear_tsk_need_resched(prev);
4050	clear_preempt_need_resched();
4051
4052	if (likely(prev != next)) {
4053		rq->nr_switches++;
4054		/*
4055		 * RCU users of rcu_dereference(rq->curr) may not see
4056		 * changes to task_struct made by pick_next_task().
4057		 */
4058		RCU_INIT_POINTER(rq->curr, next);
4059		/*
4060		 * The membarrier system call requires each architecture
4061		 * to have a full memory barrier after updating
4062		 * rq->curr, before returning to user-space.
4063		 *
4064		 * Here are the schemes providing that barrier on the
4065		 * various architectures:
4066		 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
4067		 *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
4068		 * - finish_lock_switch() for weakly-ordered
4069		 *   architectures where spin_unlock is a full barrier,
4070		 * - switch_to() for arm64 (weakly-ordered, spin_unlock
4071		 *   is a RELEASE barrier),
4072		 */
4073		++*switch_count;
4074
4075		trace_sched_switch(preempt, prev, next);
4076
4077		/* Also unlocks the rq: */
4078		rq = context_switch(rq, prev, next, &rf);
4079	} else {
4080		rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
4081		rq_unlock_irq(rq, &rf);
4082	}
4083
4084	balance_callback(rq);
4085}
4086
4087void __noreturn do_task_dead(void)
4088{
4089	/* Causes final put_task_struct in finish_task_switch(): */
4090	set_special_state(TASK_DEAD);
4091
4092	/* Tell freezer to ignore us: */
4093	current->flags |= PF_NOFREEZE;
4094
4095	__schedule(false);
4096	BUG();
4097
4098	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
4099	for (;;)
4100		cpu_relax();
4101}
4102
4103static inline void sched_submit_work(struct task_struct *tsk)
4104{
4105	if (!tsk->state)
4106		return;
4107
4108	/*
4109	 * If a worker went to sleep, notify and ask workqueue whether
4110	 * it wants to wake up a task to maintain concurrency.
4111	 * As this function is called inside the schedule() context,
4112	 * we disable preemption to avoid it calling schedule() again
4113	 * in the possible wakeup of a kworker.
4114	 */
4115	if (tsk->flags & PF_WQ_WORKER) {
4116		preempt_disable();
4117		wq_worker_sleeping(tsk);
4118		preempt_enable_no_resched();
4119	}
4120
4121	if (tsk_is_pi_blocked(tsk))
4122		return;
4123
4124	/*
4125	 * If we are going to sleep and we have plugged IO queued,
4126	 * make sure to submit it to avoid deadlocks.
4127	 */
4128	if (blk_needs_flush_plug(tsk))
4129		blk_schedule_flush_plug(tsk);
4130}
4131
4132static void sched_update_worker(struct task_struct *tsk)
4133{
4134	if (tsk->flags & PF_WQ_WORKER)
4135		wq_worker_running(tsk);
4136}
4137
4138asmlinkage __visible void __sched schedule(void)
4139{
4140	struct task_struct *tsk = current;
4141
4142	sched_submit_work(tsk);
4143	do {
4144		preempt_disable();
4145		__schedule(false);
4146		sched_preempt_enable_no_resched();
4147	} while (need_resched());
4148	sched_update_worker(tsk);
4149}
4150EXPORT_SYMBOL(schedule);
4151
4152/*
4153 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
4154 * state (have scheduled out non-voluntarily) by making sure that all
4155 * tasks have either left the run queue or have gone into user space.
4156 * As idle tasks do not do either, they must not ever be preempted
4157 * (schedule out non-voluntarily).
4158 *
4159 * schedule_idle() is similar to schedule_preempt_disable() except that it
4160 * never enables preemption because it does not call sched_submit_work().
4161 */
4162void __sched schedule_idle(void)
4163{
4164	/*
4165	 * As this skips calling sched_submit_work(), which the idle task does
4166	 * regardless because that function is a nop when the task is in a
4167	 * TASK_RUNNING state, make sure this isn't used someplace that the
4168	 * current task can be in any other state. Note, idle is always in the
4169	 * TASK_RUNNING state.
4170	 */
4171	WARN_ON_ONCE(current->state);
4172	do {
4173		__schedule(false);
4174	} while (need_resched());
4175}
4176
4177#ifdef CONFIG_CONTEXT_TRACKING
4178asmlinkage __visible void __sched schedule_user(void)
4179{
4180	/*
4181	 * If we come here after a random call to set_need_resched(),
4182	 * or we have been woken up remotely but the IPI has not yet arrived,
4183	 * we haven't yet exited the RCU idle mode. Do it here manually until
4184	 * we find a better solution.
4185	 *
4186	 * NB: There are buggy callers of this function.  Ideally we
4187	 * should warn if prev_state != CONTEXT_USER, but that will trigger
4188	 * too frequently to make sense yet.
4189	 */
4190	enum ctx_state prev_state = exception_enter();
4191	schedule();
4192	exception_exit(prev_state);
4193}
4194#endif
4195
4196/**
4197 * schedule_preempt_disabled - called with preemption disabled
4198 *
4199 * Returns with preemption disabled. Note: preempt_count must be 1
4200 */
4201void __sched schedule_preempt_disabled(void)
4202{
4203	sched_preempt_enable_no_resched();
4204	schedule();
4205	preempt_disable();
4206}
4207
4208static void __sched notrace preempt_schedule_common(void)
4209{
4210	do {
4211		/*
4212		 * Because the function tracer can trace preempt_count_sub()
4213		 * and it also uses preempt_enable/disable_notrace(), if
4214		 * NEED_RESCHED is set, the preempt_enable_notrace() called
4215		 * by the function tracer will call this function again and
4216		 * cause infinite recursion.
4217		 *
4218		 * Preemption must be disabled here before the function
4219		 * tracer can trace. Break up preempt_disable() into two
4220		 * calls. One to disable preemption without fear of being
4221		 * traced. The other to still record the preemption latency,
4222		 * which can also be traced by the function tracer.
4223		 */
4224		preempt_disable_notrace();
4225		preempt_latency_start(1);
4226		__schedule(true);
4227		preempt_latency_stop(1);
4228		preempt_enable_no_resched_notrace();
4229
4230		/*
4231		 * Check again in case we missed a preemption opportunity
4232		 * between schedule and now.
4233		 */
4234	} while (need_resched());
4235}
4236
4237#ifdef CONFIG_PREEMPTION
4238/*
4239 * This is the entry point to schedule() from in-kernel preemption
4240 * off of preempt_enable.
 
4241 */
4242asmlinkage __visible void __sched notrace preempt_schedule(void)
4243{
4244	/*
4245	 * If there is a non-zero preempt_count or interrupts are disabled,
4246	 * we do not want to preempt the current task. Just return..
4247	 */
4248	if (likely(!preemptible()))
4249		return;
4250
4251	preempt_schedule_common();
4252}
4253NOKPROBE_SYMBOL(preempt_schedule);
4254EXPORT_SYMBOL(preempt_schedule);
4255
4256/**
4257 * preempt_schedule_notrace - preempt_schedule called by tracing
4258 *
4259 * The tracing infrastructure uses preempt_enable_notrace to prevent
4260 * recursion and tracing preempt enabling caused by the tracing
4261 * infrastructure itself. But as tracing can happen in areas coming
4262 * from userspace or just about to enter userspace, a preempt enable
4263 * can occur before user_exit() is called. This will cause the scheduler
4264 * to be called when the system is still in usermode.
4265 *
4266 * To prevent this, the preempt_enable_notrace will use this function
4267 * instead of preempt_schedule() to exit user context if needed before
4268 * calling the scheduler.
4269 */
4270asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
4271{
4272	enum ctx_state prev_ctx;
4273
4274	if (likely(!preemptible()))
4275		return;
4276
4277	do {
4278		/*
4279		 * Because the function tracer can trace preempt_count_sub()
4280		 * and it also uses preempt_enable/disable_notrace(), if
4281		 * NEED_RESCHED is set, the preempt_enable_notrace() called
4282		 * by the function tracer will call this function again and
4283		 * cause infinite recursion.
4284		 *
4285		 * Preemption must be disabled here before the function
4286		 * tracer can trace. Break up preempt_disable() into two
4287		 * calls. One to disable preemption without fear of being
4288		 * traced. The other to still record the preemption latency,
4289		 * which can also be traced by the function tracer.
4290		 */
4291		preempt_disable_notrace();
4292		preempt_latency_start(1);
4293		/*
4294		 * Needs preempt disabled in case user_exit() is traced
4295		 * and the tracer calls preempt_enable_notrace() causing
4296		 * an infinite recursion.
4297		 */
4298		prev_ctx = exception_enter();
4299		__schedule(true);
4300		exception_exit(prev_ctx);
4301
4302		preempt_latency_stop(1);
4303		preempt_enable_no_resched_notrace();
4304	} while (need_resched());
4305}
4306EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
4307
4308#endif /* CONFIG_PREEMPTION */
4309
4310/*
4311 * This is the entry point to schedule() from kernel preemption
4312 * off of irq context.
4313 * Note, that this is called and return with irqs disabled. This will
4314 * protect us against recursive calling from irq.
4315 */
4316asmlinkage __visible void __sched preempt_schedule_irq(void)
4317{
4318	enum ctx_state prev_state;
4319
4320	/* Catch callers which need to be fixed */
4321	BUG_ON(preempt_count() || !irqs_disabled());
4322
4323	prev_state = exception_enter();
4324
4325	do {
4326		preempt_disable();
4327		local_irq_enable();
4328		__schedule(true);
4329		local_irq_disable();
4330		sched_preempt_enable_no_resched();
4331	} while (need_resched());
4332
4333	exception_exit(prev_state);
4334}
4335
4336int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
4337			  void *key)
4338{
4339	return try_to_wake_up(curr->private, mode, wake_flags);
4340}
4341EXPORT_SYMBOL(default_wake_function);
4342
4343#ifdef CONFIG_RT_MUTEXES
4344
4345static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
4346{
4347	if (pi_task)
4348		prio = min(prio, pi_task->prio);
4349
4350	return prio;
4351}
4352
4353static inline int rt_effective_prio(struct task_struct *p, int prio)
4354{
4355	struct task_struct *pi_task = rt_mutex_get_top_task(p);
4356
4357	return __rt_effective_prio(pi_task, prio);
4358}
4359
4360/*
4361 * rt_mutex_setprio - set the current priority of a task
4362 * @p: task to boost
4363 * @pi_task: donor task
4364 *
4365 * This function changes the 'effective' priority of a task. It does
4366 * not touch ->normal_prio like __setscheduler().
4367 *
4368 * Used by the rt_mutex code to implement priority inheritance
4369 * logic. Call site only calls if the priority of the task changed.
4370 */
4371void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
4372{
4373	int prio, oldprio, queued, running, queue_flag =
4374		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
4375	const struct sched_class *prev_class;
4376	struct rq_flags rf;
4377	struct rq *rq;
4378
4379	/* XXX used to be waiter->prio, not waiter->task->prio */
4380	prio = __rt_effective_prio(pi_task, p->normal_prio);
4381
4382	/*
4383	 * If nothing changed; bail early.
4384	 */
4385	if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
4386		return;
4387
4388	rq = __task_rq_lock(p, &rf);
4389	update_rq_clock(rq);
4390	/*
4391	 * Set under pi_lock && rq->lock, such that the value can be used under
4392	 * either lock.
4393	 *
4394	 * Note that there is loads of tricky to make this pointer cache work
4395	 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
4396	 * ensure a task is de-boosted (pi_task is set to NULL) before the
4397	 * task is allowed to run again (and can exit). This ensures the pointer
4398	 * points to a blocked task -- which guaratees the task is present.
4399	 */
4400	p->pi_top_task = pi_task;
4401
4402	/*
4403	 * For FIFO/RR we only need to set prio, if that matches we're done.
4404	 */
4405	if (prio == p->prio && !dl_prio(prio))
4406		goto out_unlock;
4407
4408	/*
4409	 * Idle task boosting is a nono in general. There is one
4410	 * exception, when PREEMPT_RT and NOHZ is active:
4411	 *
4412	 * The idle task calls get_next_timer_interrupt() and holds
4413	 * the timer wheel base->lock on the CPU and another CPU wants
4414	 * to access the timer (probably to cancel it). We can safely
4415	 * ignore the boosting request, as the idle CPU runs this code
4416	 * with interrupts disabled and will complete the lock
4417	 * protected section without being interrupted. So there is no
4418	 * real need to boost.
4419	 */
4420	if (unlikely(p == rq->idle)) {
4421		WARN_ON(p != rq->curr);
4422		WARN_ON(p->pi_blocked_on);
4423		goto out_unlock;
4424	}
4425
4426	trace_sched_pi_setprio(p, pi_task);
4427	oldprio = p->prio;
4428
4429	if (oldprio == prio)
4430		queue_flag &= ~DEQUEUE_MOVE;
4431
4432	prev_class = p->sched_class;
4433	queued = task_on_rq_queued(p);
4434	running = task_current(rq, p);
4435	if (queued)
4436		dequeue_task(rq, p, queue_flag);
4437	if (running)
4438		put_prev_task(rq, p);
4439
4440	/*
4441	 * Boosting condition are:
4442	 * 1. -rt task is running and holds mutex A
4443	 *      --> -dl task blocks on mutex A
4444	 *
4445	 * 2. -dl task is running and holds mutex A
4446	 *      --> -dl task blocks on mutex A and could preempt the
4447	 *          running task
4448	 */
4449	if (dl_prio(prio)) {
4450		if (!dl_prio(p->normal_prio) ||
4451		    (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
4452			p->dl.dl_boosted = 1;
4453			queue_flag |= ENQUEUE_REPLENISH;
4454		} else
4455			p->dl.dl_boosted = 0;
4456		p->sched_class = &dl_sched_class;
4457	} else if (rt_prio(prio)) {
4458		if (dl_prio(oldprio))
4459			p->dl.dl_boosted = 0;
4460		if (oldprio < prio)
4461			queue_flag |= ENQUEUE_HEAD;
4462		p->sched_class = &rt_sched_class;
4463	} else {
4464		if (dl_prio(oldprio))
4465			p->dl.dl_boosted = 0;
4466		if (rt_prio(oldprio))
4467			p->rt.timeout = 0;
4468		p->sched_class = &fair_sched_class;
4469	}
4470
4471	p->prio = prio;
4472
4473	if (queued)
4474		enqueue_task(rq, p, queue_flag);
4475	if (running)
4476		set_next_task(rq, p);
4477
4478	check_class_changed(rq, p, prev_class, oldprio);
4479out_unlock:
4480	/* Avoid rq from going away on us: */
4481	preempt_disable();
4482	__task_rq_unlock(rq, &rf);
4483
4484	balance_callback(rq);
4485	preempt_enable();
4486}
4487#else
4488static inline int rt_effective_prio(struct task_struct *p, int prio)
4489{
4490	return prio;
4491}
4492#endif
4493
4494void set_user_nice(struct task_struct *p, long nice)
4495{
4496	bool queued, running;
4497	int old_prio, delta;
4498	struct rq_flags rf;
4499	struct rq *rq;
4500
4501	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
4502		return;
4503	/*
4504	 * We have to be careful, if called from sys_setpriority(),
4505	 * the task might be in the middle of scheduling on another CPU.
4506	 */
4507	rq = task_rq_lock(p, &rf);
4508	update_rq_clock(rq);
4509
4510	/*
4511	 * The RT priorities are set via sched_setscheduler(), but we still
4512	 * allow the 'normal' nice value to be set - but as expected
4513	 * it wont have any effect on scheduling until the task is
4514	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
4515	 */
4516	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
4517		p->static_prio = NICE_TO_PRIO(nice);
4518		goto out_unlock;
4519	}
4520	queued = task_on_rq_queued(p);
4521	running = task_current(rq, p);
4522	if (queued)
4523		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
4524	if (running)
4525		put_prev_task(rq, p);
4526
4527	p->static_prio = NICE_TO_PRIO(nice);
4528	set_load_weight(p, true);
4529	old_prio = p->prio;
4530	p->prio = effective_prio(p);
4531	delta = p->prio - old_prio;
4532
4533	if (queued) {
4534		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
4535		/*
4536		 * If the task increased its priority or is running and
4537		 * lowered its priority, then reschedule its CPU:
4538		 */
4539		if (delta < 0 || (delta > 0 && task_running(rq, p)))
4540			resched_curr(rq);
4541	}
4542	if (running)
4543		set_next_task(rq, p);
4544out_unlock:
4545	task_rq_unlock(rq, p, &rf);
4546}
4547EXPORT_SYMBOL(set_user_nice);
4548
4549/*
4550 * can_nice - check if a task can reduce its nice value
4551 * @p: task
4552 * @nice: nice value
4553 */
4554int can_nice(const struct task_struct *p, const int nice)
4555{
4556	/* Convert nice value [19,-20] to rlimit style value [1,40]: */
4557	int nice_rlim = nice_to_rlimit(nice);
4558
4559	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
4560		capable(CAP_SYS_NICE));
4561}
4562
4563#ifdef __ARCH_WANT_SYS_NICE
4564
4565/*
4566 * sys_nice - change the priority of the current process.
4567 * @increment: priority increment
4568 *
4569 * sys_setpriority is a more generic, but much slower function that
4570 * does similar things.
4571 */
4572SYSCALL_DEFINE1(nice, int, increment)
4573{
4574	long nice, retval;
4575
4576	/*
4577	 * Setpriority might change our priority at the same moment.
4578	 * We don't have to worry. Conceptually one call occurs first
4579	 * and we have a single winner.
4580	 */
4581	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
4582	nice = task_nice(current) + increment;
4583
4584	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
4585	if (increment < 0 && !can_nice(current, nice))
4586		return -EPERM;
4587
4588	retval = security_task_setnice(current, nice);
4589	if (retval)
4590		return retval;
4591
4592	set_user_nice(current, nice);
4593	return 0;
4594}
4595
4596#endif
4597
4598/**
4599 * task_prio - return the priority value of a given task.
4600 * @p: the task in question.
4601 *
4602 * Return: The priority value as seen by users in /proc.
4603 * RT tasks are offset by -200. Normal tasks are centered
4604 * around 0, value goes from -16 to +15.
4605 */
4606int task_prio(const struct task_struct *p)
4607{
4608	return p->prio - MAX_RT_PRIO;
4609}
4610
4611/**
4612 * idle_cpu - is a given CPU idle currently?
4613 * @cpu: the processor in question.
4614 *
4615 * Return: 1 if the CPU is currently idle. 0 otherwise.
4616 */
4617int idle_cpu(int cpu)
4618{
4619	struct rq *rq = cpu_rq(cpu);
4620
4621	if (rq->curr != rq->idle)
4622		return 0;
4623
4624	if (rq->nr_running)
4625		return 0;
4626
4627#ifdef CONFIG_SMP
4628	if (!llist_empty(&rq->wake_list))
4629		return 0;
4630#endif
4631
4632	return 1;
4633}
4634
4635/**
4636 * available_idle_cpu - is a given CPU idle for enqueuing work.
4637 * @cpu: the CPU in question.
4638 *
4639 * Return: 1 if the CPU is currently idle. 0 otherwise.
4640 */
4641int available_idle_cpu(int cpu)
4642{
4643	if (!idle_cpu(cpu))
4644		return 0;
4645
4646	if (vcpu_is_preempted(cpu))
4647		return 0;
4648
4649	return 1;
4650}
4651
4652/**
4653 * idle_task - return the idle task for a given CPU.
4654 * @cpu: the processor in question.
4655 *
4656 * Return: The idle task for the CPU @cpu.
4657 */
4658struct task_struct *idle_task(int cpu)
4659{
4660	return cpu_rq(cpu)->idle;
4661}
4662
4663/**
4664 * find_process_by_pid - find a process with a matching PID value.
4665 * @pid: the pid in question.
4666 *
4667 * The task of @pid, if found. %NULL otherwise.
4668 */
4669static struct task_struct *find_process_by_pid(pid_t pid)
4670{
4671	return pid ? find_task_by_vpid(pid) : current;
4672}
4673
4674/*
4675 * sched_setparam() passes in -1 for its policy, to let the functions
4676 * it calls know not to change it.
4677 */
4678#define SETPARAM_POLICY	-1
4679
4680static void __setscheduler_params(struct task_struct *p,
4681		const struct sched_attr *attr)
4682{
4683	int policy = attr->sched_policy;
4684
4685	if (policy == SETPARAM_POLICY)
4686		policy = p->policy;
4687
4688	p->policy = policy;
4689
4690	if (dl_policy(policy))
4691		__setparam_dl(p, attr);
4692	else if (fair_policy(policy))
4693		p->static_prio = NICE_TO_PRIO(attr->sched_nice);
4694
4695	/*
4696	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
4697	 * !rt_policy. Always setting this ensures that things like
4698	 * getparam()/getattr() don't report silly values for !rt tasks.
4699	 */
4700	p->rt_priority = attr->sched_priority;
4701	p->normal_prio = normal_prio(p);
4702	set_load_weight(p, true);
4703}
4704
4705/* Actually do priority change: must hold pi & rq lock. */
4706static void __setscheduler(struct rq *rq, struct task_struct *p,
4707			   const struct sched_attr *attr, bool keep_boost)
4708{
4709	/*
4710	 * If params can't change scheduling class changes aren't allowed
4711	 * either.
4712	 */
4713	if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)
4714		return;
4715
4716	__setscheduler_params(p, attr);
4717
4718	/*
4719	 * Keep a potential priority boosting if called from
4720	 * sched_setscheduler().
4721	 */
4722	p->prio = normal_prio(p);
4723	if (keep_boost)
4724		p->prio = rt_effective_prio(p, p->prio);
4725
4726	if (dl_prio(p->prio))
4727		p->sched_class = &dl_sched_class;
4728	else if (rt_prio(p->prio))
4729		p->sched_class = &rt_sched_class;
4730	else
4731		p->sched_class = &fair_sched_class;
4732}
4733
4734/*
4735 * Check the target process has a UID that matches the current process's:
4736 */
4737static bool check_same_owner(struct task_struct *p)
4738{
4739	const struct cred *cred = current_cred(), *pcred;
4740	bool match;
4741
4742	rcu_read_lock();
4743	pcred = __task_cred(p);
4744	match = (uid_eq(cred->euid, pcred->euid) ||
4745		 uid_eq(cred->euid, pcred->uid));
4746	rcu_read_unlock();
4747	return match;
4748}
4749
4750static int __sched_setscheduler(struct task_struct *p,
4751				const struct sched_attr *attr,
4752				bool user, bool pi)
4753{
4754	int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
4755		      MAX_RT_PRIO - 1 - attr->sched_priority;
4756	int retval, oldprio, oldpolicy = -1, queued, running;
4757	int new_effective_prio, policy = attr->sched_policy;
4758	const struct sched_class *prev_class;
4759	struct rq_flags rf;
4760	int reset_on_fork;
4761	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
4762	struct rq *rq;
4763
4764	/* The pi code expects interrupts enabled */
4765	BUG_ON(pi && in_interrupt());
4766recheck:
4767	/* Double check policy once rq lock held: */
4768	if (policy < 0) {
4769		reset_on_fork = p->sched_reset_on_fork;
4770		policy = oldpolicy = p->policy;
4771	} else {
4772		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
4773
4774		if (!valid_policy(policy))
4775			return -EINVAL;
4776	}
4777
4778	if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV))
4779		return -EINVAL;
4780
4781	/*
4782	 * Valid priorities for SCHED_FIFO and SCHED_RR are
4783	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
4784	 * SCHED_BATCH and SCHED_IDLE is 0.
4785	 */
4786	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
4787	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
4788		return -EINVAL;
4789	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
4790	    (rt_policy(policy) != (attr->sched_priority != 0)))
4791		return -EINVAL;
4792
4793	/*
4794	 * Allow unprivileged RT tasks to decrease priority:
4795	 */
4796	if (user && !capable(CAP_SYS_NICE)) {
4797		if (fair_policy(policy)) {
4798			if (attr->sched_nice < task_nice(p) &&
4799			    !can_nice(p, attr->sched_nice))
4800				return -EPERM;
4801		}
4802
4803		if (rt_policy(policy)) {
4804			unsigned long rlim_rtprio =
4805					task_rlimit(p, RLIMIT_RTPRIO);
4806
4807			/* Can't set/change the rt policy: */
4808			if (policy != p->policy && !rlim_rtprio)
4809				return -EPERM;
4810
4811			/* Can't increase priority: */
4812			if (attr->sched_priority > p->rt_priority &&
4813			    attr->sched_priority > rlim_rtprio)
4814				return -EPERM;
4815		}
4816
4817		 /*
4818		  * Can't set/change SCHED_DEADLINE policy at all for now
4819		  * (safest behavior); in the future we would like to allow
4820		  * unprivileged DL tasks to increase their relative deadline
4821		  * or reduce their runtime (both ways reducing utilization)
4822		  */
4823		if (dl_policy(policy))
4824			return -EPERM;
4825
4826		/*
4827		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
4828		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
4829		 */
4830		if (task_has_idle_policy(p) && !idle_policy(policy)) {
4831			if (!can_nice(p, task_nice(p)))
4832				return -EPERM;
4833		}
4834
4835		/* Can't change other user's priorities: */
4836		if (!check_same_owner(p))
4837			return -EPERM;
4838
4839		/* Normal users shall not reset the sched_reset_on_fork flag: */
4840		if (p->sched_reset_on_fork && !reset_on_fork)
4841			return -EPERM;
4842	}
4843
4844	if (user) {
4845		if (attr->sched_flags & SCHED_FLAG_SUGOV)
4846			return -EINVAL;
4847
4848		retval = security_task_setscheduler(p);
4849		if (retval)
4850			return retval;
4851	}
4852
4853	/* Update task specific "requested" clamps */
4854	if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) {
4855		retval = uclamp_validate(p, attr);
4856		if (retval)
4857			return retval;
4858	}
4859
4860	if (pi)
4861		cpuset_read_lock();
4862
4863	/*
4864	 * Make sure no PI-waiters arrive (or leave) while we are
4865	 * changing the priority of the task:
4866	 *
4867	 * To be able to change p->policy safely, the appropriate
4868	 * runqueue lock must be held.
4869	 */
4870	rq = task_rq_lock(p, &rf);
4871	update_rq_clock(rq);
4872
4873	/*
4874	 * Changing the policy of the stop threads its a very bad idea:
4875	 */
4876	if (p == rq->stop) {
4877		retval = -EINVAL;
4878		goto unlock;
4879	}
4880
4881	/*
4882	 * If not changing anything there's no need to proceed further,
4883	 * but store a possible modification of reset_on_fork.
4884	 */
4885	if (unlikely(policy == p->policy)) {
4886		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
4887			goto change;
4888		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
4889			goto change;
4890		if (dl_policy(policy) && dl_param_changed(p, attr))
4891			goto change;
4892		if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)
4893			goto change;
4894
4895		p->sched_reset_on_fork = reset_on_fork;
4896		retval = 0;
4897		goto unlock;
4898	}
4899change:
4900
4901	if (user) {
4902#ifdef CONFIG_RT_GROUP_SCHED
4903		/*
4904		 * Do not allow realtime tasks into groups that have no runtime
4905		 * assigned.
4906		 */
4907		if (rt_bandwidth_enabled() && rt_policy(policy) &&
4908				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
4909				!task_group_is_autogroup(task_group(p))) {
4910			retval = -EPERM;
4911			goto unlock;
4912		}
4913#endif
4914#ifdef CONFIG_SMP
4915		if (dl_bandwidth_enabled() && dl_policy(policy) &&
4916				!(attr->sched_flags & SCHED_FLAG_SUGOV)) {
4917			cpumask_t *span = rq->rd->span;
4918
4919			/*
4920			 * Don't allow tasks with an affinity mask smaller than
4921			 * the entire root_domain to become SCHED_DEADLINE. We
4922			 * will also fail if there's no bandwidth available.
4923			 */
4924			if (!cpumask_subset(span, p->cpus_ptr) ||
4925			    rq->rd->dl_bw.bw == 0) {
4926				retval = -EPERM;
4927				goto unlock;
4928			}
4929		}
4930#endif
4931	}
4932
4933	/* Re-check policy now with rq lock held: */
4934	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
4935		policy = oldpolicy = -1;
4936		task_rq_unlock(rq, p, &rf);
4937		if (pi)
4938			cpuset_read_unlock();
4939		goto recheck;
4940	}
4941
4942	/*
4943	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
4944	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
4945	 * is available.
4946	 */
4947	if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) {
4948		retval = -EBUSY;
4949		goto unlock;
4950	}
4951
4952	p->sched_reset_on_fork = reset_on_fork;
4953	oldprio = p->prio;
4954
4955	if (pi) {
4956		/*
4957		 * Take priority boosted tasks into account. If the new
4958		 * effective priority is unchanged, we just store the new
4959		 * normal parameters and do not touch the scheduler class and
4960		 * the runqueue. This will be done when the task deboost
4961		 * itself.
4962		 */
4963		new_effective_prio = rt_effective_prio(p, newprio);
4964		if (new_effective_prio == oldprio)
4965			queue_flags &= ~DEQUEUE_MOVE;
4966	}
4967
4968	queued = task_on_rq_queued(p);
4969	running = task_current(rq, p);
4970	if (queued)
4971		dequeue_task(rq, p, queue_flags);
4972	if (running)
4973		put_prev_task(rq, p);
4974
4975	prev_class = p->sched_class;
4976
4977	__setscheduler(rq, p, attr, pi);
4978	__setscheduler_uclamp(p, attr);
4979
4980	if (queued) {
4981		/*
4982		 * We enqueue to tail when the priority of a task is
4983		 * increased (user space view).
4984		 */
4985		if (oldprio < p->prio)
4986			queue_flags |= ENQUEUE_HEAD;
4987
4988		enqueue_task(rq, p, queue_flags);
4989	}
4990	if (running)
4991		set_next_task(rq, p);
4992
4993	check_class_changed(rq, p, prev_class, oldprio);
4994
4995	/* Avoid rq from going away on us: */
4996	preempt_disable();
4997	task_rq_unlock(rq, p, &rf);
4998
4999	if (pi) {
5000		cpuset_read_unlock();
5001		rt_mutex_adjust_pi(p);
5002	}
5003
5004	/* Run balance callbacks after we've adjusted the PI chain: */
5005	balance_callback(rq);
5006	preempt_enable();
5007
5008	return 0;
5009
5010unlock:
5011	task_rq_unlock(rq, p, &rf);
5012	if (pi)
5013		cpuset_read_unlock();
5014	return retval;
5015}
5016
5017static int _sched_setscheduler(struct task_struct *p, int policy,
5018			       const struct sched_param *param, bool check)
5019{
5020	struct sched_attr attr = {
5021		.sched_policy   = policy,
5022		.sched_priority = param->sched_priority,
5023		.sched_nice	= PRIO_TO_NICE(p->static_prio),
5024	};
5025
5026	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
5027	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
5028		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
5029		policy &= ~SCHED_RESET_ON_FORK;
5030		attr.sched_policy = policy;
5031	}
5032
5033	return __sched_setscheduler(p, &attr, check, true);
5034}
5035/**
5036 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
5037 * @p: the task in question.
5038 * @policy: new policy.
5039 * @param: structure containing the new RT priority.
5040 *
5041 * Return: 0 on success. An error code otherwise.
5042 *
5043 * NOTE that the task may be already dead.
5044 */
5045int sched_setscheduler(struct task_struct *p, int policy,
5046		       const struct sched_param *param)
5047{
5048	return _sched_setscheduler(p, policy, param, true);
5049}
5050EXPORT_SYMBOL_GPL(sched_setscheduler);
5051
5052int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
5053{
5054	return __sched_setscheduler(p, attr, true, true);
5055}
5056EXPORT_SYMBOL_GPL(sched_setattr);
5057
5058int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
5059{
5060	return __sched_setscheduler(p, attr, false, true);
5061}
5062
5063/**
5064 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
5065 * @p: the task in question.
5066 * @policy: new policy.
5067 * @param: structure containing the new RT priority.
5068 *
5069 * Just like sched_setscheduler, only don't bother checking if the
5070 * current context has permission.  For example, this is needed in
5071 * stop_machine(): we create temporary high priority worker threads,
5072 * but our caller might not have that capability.
5073 *
5074 * Return: 0 on success. An error code otherwise.
5075 */
5076int sched_setscheduler_nocheck(struct task_struct *p, int policy,
5077			       const struct sched_param *param)
5078{
5079	return _sched_setscheduler(p, policy, param, false);
5080}
5081EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
5082
5083static int
5084do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
5085{
5086	struct sched_param lparam;
5087	struct task_struct *p;
5088	int retval;
5089
5090	if (!param || pid < 0)
5091		return -EINVAL;
5092	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
5093		return -EFAULT;
5094
5095	rcu_read_lock();
5096	retval = -ESRCH;
5097	p = find_process_by_pid(pid);
5098	if (likely(p))
5099		get_task_struct(p);
5100	rcu_read_unlock();
5101
5102	if (likely(p)) {
5103		retval = sched_setscheduler(p, policy, &lparam);
5104		put_task_struct(p);
5105	}
5106
5107	return retval;
5108}
5109
5110/*
5111 * Mimics kernel/events/core.c perf_copy_attr().
5112 */
5113static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr)
5114{
5115	u32 size;
5116	int ret;
5117
 
 
 
5118	/* Zero the full structure, so that a short copy will be nice: */
5119	memset(attr, 0, sizeof(*attr));
5120
5121	ret = get_user(size, &uattr->size);
5122	if (ret)
5123		return ret;
5124
 
 
 
 
5125	/* ABI compatibility quirk: */
5126	if (!size)
5127		size = SCHED_ATTR_SIZE_VER0;
5128	if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE)
 
5129		goto err_size;
5130
5131	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
5132	if (ret) {
5133		if (ret == -E2BIG)
5134			goto err_size;
5135		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5136	}
5137
5138	if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) &&
5139	    size < SCHED_ATTR_SIZE_VER1)
5140		return -EINVAL;
5141
5142	/*
5143	 * XXX: Do we want to be lenient like existing syscalls; or do we want
5144	 * to be strict and return an error on out-of-bounds values?
5145	 */
5146	attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
5147
5148	return 0;
5149
5150err_size:
5151	put_user(sizeof(*attr), &uattr->size);
5152	return -E2BIG;
5153}
5154
5155/**
5156 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
5157 * @pid: the pid in question.
5158 * @policy: new policy.
5159 * @param: structure containing the new RT priority.
5160 *
5161 * Return: 0 on success. An error code otherwise.
5162 */
5163SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param)
5164{
5165	if (policy < 0)
5166		return -EINVAL;
5167
5168	return do_sched_setscheduler(pid, policy, param);
5169}
5170
5171/**
5172 * sys_sched_setparam - set/change the RT priority of a thread
5173 * @pid: the pid in question.
5174 * @param: structure containing the new RT priority.
5175 *
5176 * Return: 0 on success. An error code otherwise.
5177 */
5178SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
5179{
5180	return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
5181}
5182
5183/**
5184 * sys_sched_setattr - same as above, but with extended sched_attr
5185 * @pid: the pid in question.
5186 * @uattr: structure containing the extended parameters.
5187 * @flags: for future extension.
5188 */
5189SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
5190			       unsigned int, flags)
5191{
5192	struct sched_attr attr;
5193	struct task_struct *p;
5194	int retval;
5195
5196	if (!uattr || pid < 0 || flags)
5197		return -EINVAL;
5198
5199	retval = sched_copy_attr(uattr, &attr);
5200	if (retval)
5201		return retval;
5202
5203	if ((int)attr.sched_policy < 0)
5204		return -EINVAL;
5205	if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY)
5206		attr.sched_policy = SETPARAM_POLICY;
5207
5208	rcu_read_lock();
5209	retval = -ESRCH;
5210	p = find_process_by_pid(pid);
5211	if (likely(p))
5212		get_task_struct(p);
5213	rcu_read_unlock();
5214
5215	if (likely(p)) {
5216		retval = sched_setattr(p, &attr);
5217		put_task_struct(p);
5218	}
5219
5220	return retval;
5221}
5222
5223/**
5224 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
5225 * @pid: the pid in question.
5226 *
5227 * Return: On success, the policy of the thread. Otherwise, a negative error
5228 * code.
5229 */
5230SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
5231{
5232	struct task_struct *p;
5233	int retval;
5234
5235	if (pid < 0)
5236		return -EINVAL;
5237
5238	retval = -ESRCH;
5239	rcu_read_lock();
5240	p = find_process_by_pid(pid);
5241	if (p) {
5242		retval = security_task_getscheduler(p);
5243		if (!retval)
5244			retval = p->policy
5245				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
5246	}
5247	rcu_read_unlock();
5248	return retval;
5249}
5250
5251/**
5252 * sys_sched_getparam - get the RT priority of a thread
5253 * @pid: the pid in question.
5254 * @param: structure containing the RT priority.
5255 *
5256 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
5257 * code.
5258 */
5259SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
5260{
5261	struct sched_param lp = { .sched_priority = 0 };
5262	struct task_struct *p;
5263	int retval;
5264
5265	if (!param || pid < 0)
5266		return -EINVAL;
5267
5268	rcu_read_lock();
5269	p = find_process_by_pid(pid);
5270	retval = -ESRCH;
5271	if (!p)
5272		goto out_unlock;
5273
5274	retval = security_task_getscheduler(p);
5275	if (retval)
5276		goto out_unlock;
5277
5278	if (task_has_rt_policy(p))
5279		lp.sched_priority = p->rt_priority;
5280	rcu_read_unlock();
5281
5282	/*
5283	 * This one might sleep, we cannot do it with a spinlock held ...
5284	 */
5285	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
5286
5287	return retval;
5288
5289out_unlock:
5290	rcu_read_unlock();
5291	return retval;
5292}
5293
5294/*
5295 * Copy the kernel size attribute structure (which might be larger
5296 * than what user-space knows about) to user-space.
5297 *
5298 * Note that all cases are valid: user-space buffer can be larger or
5299 * smaller than the kernel-space buffer. The usual case is that both
5300 * have the same size.
5301 */
5302static int
5303sched_attr_copy_to_user(struct sched_attr __user *uattr,
5304			struct sched_attr *kattr,
5305			unsigned int usize)
5306{
5307	unsigned int ksize = sizeof(*kattr);
5308
5309	if (!access_ok(uattr, usize))
5310		return -EFAULT;
5311
5312	/*
5313	 * sched_getattr() ABI forwards and backwards compatibility:
5314	 *
5315	 * If usize == ksize then we just copy everything to user-space and all is good.
5316	 *
5317	 * If usize < ksize then we only copy as much as user-space has space for,
5318	 * this keeps ABI compatibility as well. We skip the rest.
5319	 *
5320	 * If usize > ksize then user-space is using a newer version of the ABI,
5321	 * which part the kernel doesn't know about. Just ignore it - tooling can
5322	 * detect the kernel's knowledge of attributes from the attr->size value
5323	 * which is set to ksize in this case.
5324	 */
5325	kattr->size = min(usize, ksize);
 
 
5326
5327	if (copy_to_user(uattr, kattr, kattr->size))
 
 
 
 
5328		return -EFAULT;
5329
5330	return 0;
5331}
5332
5333/**
5334 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
5335 * @pid: the pid in question.
5336 * @uattr: structure containing the extended parameters.
5337 * @usize: sizeof(attr) for fwd/bwd comp.
5338 * @flags: for future extension.
5339 */
5340SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
5341		unsigned int, usize, unsigned int, flags)
5342{
5343	struct sched_attr kattr = { };
 
 
5344	struct task_struct *p;
5345	int retval;
5346
5347	if (!uattr || pid < 0 || usize > PAGE_SIZE ||
5348	    usize < SCHED_ATTR_SIZE_VER0 || flags)
5349		return -EINVAL;
5350
5351	rcu_read_lock();
5352	p = find_process_by_pid(pid);
5353	retval = -ESRCH;
5354	if (!p)
5355		goto out_unlock;
5356
5357	retval = security_task_getscheduler(p);
5358	if (retval)
5359		goto out_unlock;
5360
5361	kattr.sched_policy = p->policy;
5362	if (p->sched_reset_on_fork)
5363		kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
5364	if (task_has_dl_policy(p))
5365		__getparam_dl(p, &kattr);
5366	else if (task_has_rt_policy(p))
5367		kattr.sched_priority = p->rt_priority;
5368	else
5369		kattr.sched_nice = task_nice(p);
5370
5371#ifdef CONFIG_UCLAMP_TASK
5372	kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value;
5373	kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value;
5374#endif
5375
5376	rcu_read_unlock();
5377
5378	return sched_attr_copy_to_user(uattr, &kattr, usize);
 
5379
5380out_unlock:
5381	rcu_read_unlock();
5382	return retval;
5383}
5384
5385long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5386{
5387	cpumask_var_t cpus_allowed, new_mask;
5388	struct task_struct *p;
5389	int retval;
5390
5391	rcu_read_lock();
5392
5393	p = find_process_by_pid(pid);
5394	if (!p) {
5395		rcu_read_unlock();
5396		return -ESRCH;
5397	}
5398
5399	/* Prevent p going away */
5400	get_task_struct(p);
5401	rcu_read_unlock();
5402
5403	if (p->flags & PF_NO_SETAFFINITY) {
5404		retval = -EINVAL;
5405		goto out_put_task;
5406	}
5407	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5408		retval = -ENOMEM;
5409		goto out_put_task;
5410	}
5411	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5412		retval = -ENOMEM;
5413		goto out_free_cpus_allowed;
5414	}
5415	retval = -EPERM;
5416	if (!check_same_owner(p)) {
5417		rcu_read_lock();
5418		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
5419			rcu_read_unlock();
5420			goto out_free_new_mask;
5421		}
5422		rcu_read_unlock();
5423	}
5424
5425	retval = security_task_setscheduler(p);
5426	if (retval)
5427		goto out_free_new_mask;
5428
5429
5430	cpuset_cpus_allowed(p, cpus_allowed);
5431	cpumask_and(new_mask, in_mask, cpus_allowed);
5432
5433	/*
5434	 * Since bandwidth control happens on root_domain basis,
5435	 * if admission test is enabled, we only admit -deadline
5436	 * tasks allowed to run on all the CPUs in the task's
5437	 * root_domain.
5438	 */
5439#ifdef CONFIG_SMP
5440	if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
5441		rcu_read_lock();
5442		if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
5443			retval = -EBUSY;
5444			rcu_read_unlock();
5445			goto out_free_new_mask;
5446		}
5447		rcu_read_unlock();
5448	}
5449#endif
5450again:
5451	retval = __set_cpus_allowed_ptr(p, new_mask, true);
5452
5453	if (!retval) {
5454		cpuset_cpus_allowed(p, cpus_allowed);
5455		if (!cpumask_subset(new_mask, cpus_allowed)) {
5456			/*
5457			 * We must have raced with a concurrent cpuset
5458			 * update. Just reset the cpus_allowed to the
5459			 * cpuset's cpus_allowed
5460			 */
5461			cpumask_copy(new_mask, cpus_allowed);
5462			goto again;
5463		}
5464	}
5465out_free_new_mask:
5466	free_cpumask_var(new_mask);
5467out_free_cpus_allowed:
5468	free_cpumask_var(cpus_allowed);
5469out_put_task:
5470	put_task_struct(p);
5471	return retval;
5472}
5473
5474static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5475			     struct cpumask *new_mask)
5476{
5477	if (len < cpumask_size())
5478		cpumask_clear(new_mask);
5479	else if (len > cpumask_size())
5480		len = cpumask_size();
5481
5482	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5483}
5484
5485/**
5486 * sys_sched_setaffinity - set the CPU affinity of a process
5487 * @pid: pid of the process
5488 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5489 * @user_mask_ptr: user-space pointer to the new CPU mask
5490 *
5491 * Return: 0 on success. An error code otherwise.
5492 */
5493SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
5494		unsigned long __user *, user_mask_ptr)
5495{
5496	cpumask_var_t new_mask;
5497	int retval;
5498
5499	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5500		return -ENOMEM;
5501
5502	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5503	if (retval == 0)
5504		retval = sched_setaffinity(pid, new_mask);
5505	free_cpumask_var(new_mask);
5506	return retval;
5507}
5508
5509long sched_getaffinity(pid_t pid, struct cpumask *mask)
5510{
5511	struct task_struct *p;
5512	unsigned long flags;
5513	int retval;
5514
5515	rcu_read_lock();
5516
5517	retval = -ESRCH;
5518	p = find_process_by_pid(pid);
5519	if (!p)
5520		goto out_unlock;
5521
5522	retval = security_task_getscheduler(p);
5523	if (retval)
5524		goto out_unlock;
5525
5526	raw_spin_lock_irqsave(&p->pi_lock, flags);
5527	cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
5528	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
5529
5530out_unlock:
5531	rcu_read_unlock();
5532
5533	return retval;
5534}
5535
5536/**
5537 * sys_sched_getaffinity - get the CPU affinity of a process
5538 * @pid: pid of the process
5539 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5540 * @user_mask_ptr: user-space pointer to hold the current CPU mask
5541 *
5542 * Return: size of CPU mask copied to user_mask_ptr on success. An
5543 * error code otherwise.
5544 */
5545SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
5546		unsigned long __user *, user_mask_ptr)
5547{
5548	int ret;
5549	cpumask_var_t mask;
5550
5551	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
5552		return -EINVAL;
5553	if (len & (sizeof(unsigned long)-1))
5554		return -EINVAL;
5555
5556	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5557		return -ENOMEM;
5558
5559	ret = sched_getaffinity(pid, mask);
5560	if (ret == 0) {
5561		unsigned int retlen = min(len, cpumask_size());
5562
5563		if (copy_to_user(user_mask_ptr, mask, retlen))
5564			ret = -EFAULT;
5565		else
5566			ret = retlen;
5567	}
5568	free_cpumask_var(mask);
5569
5570	return ret;
5571}
5572
5573/**
5574 * sys_sched_yield - yield the current processor to other threads.
5575 *
5576 * This function yields the current CPU to other tasks. If there are no
5577 * other threads running on this CPU then this function will return.
5578 *
5579 * Return: 0.
5580 */
5581static void do_sched_yield(void)
5582{
5583	struct rq_flags rf;
5584	struct rq *rq;
5585
5586	rq = this_rq_lock_irq(&rf);
 
 
5587
5588	schedstat_inc(rq->yld_count);
5589	current->sched_class->yield_task(rq);
5590
5591	/*
5592	 * Since we are going to call schedule() anyway, there's
5593	 * no need to preempt or enable interrupts:
5594	 */
5595	preempt_disable();
5596	rq_unlock(rq, &rf);
5597	sched_preempt_enable_no_resched();
5598
5599	schedule();
5600}
5601
5602SYSCALL_DEFINE0(sched_yield)
5603{
5604	do_sched_yield();
5605	return 0;
5606}
5607
5608#ifndef CONFIG_PREEMPTION
5609int __sched _cond_resched(void)
5610{
5611	if (should_resched(0)) {
5612		preempt_schedule_common();
5613		return 1;
5614	}
5615	rcu_all_qs();
5616	return 0;
5617}
5618EXPORT_SYMBOL(_cond_resched);
5619#endif
5620
5621/*
5622 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
5623 * call schedule, and on return reacquire the lock.
5624 *
5625 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level
5626 * operations here to prevent schedule() from being called twice (once via
5627 * spin_unlock(), once by hand).
5628 */
5629int __cond_resched_lock(spinlock_t *lock)
5630{
5631	int resched = should_resched(PREEMPT_LOCK_OFFSET);
5632	int ret = 0;
5633
5634	lockdep_assert_held(lock);
5635
5636	if (spin_needbreak(lock) || resched) {
5637		spin_unlock(lock);
5638		if (resched)
5639			preempt_schedule_common();
5640		else
5641			cpu_relax();
5642		ret = 1;
5643		spin_lock(lock);
5644	}
5645	return ret;
5646}
5647EXPORT_SYMBOL(__cond_resched_lock);
5648
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5649/**
5650 * yield - yield the current processor to other threads.
5651 *
5652 * Do not ever use this function, there's a 99% chance you're doing it wrong.
5653 *
5654 * The scheduler is at all times free to pick the calling task as the most
5655 * eligible task to run, if removing the yield() call from your code breaks
5656 * it, its already broken.
5657 *
5658 * Typical broken usage is:
5659 *
5660 * while (!event)
5661 *	yield();
5662 *
5663 * where one assumes that yield() will let 'the other' process run that will
5664 * make event true. If the current task is a SCHED_FIFO task that will never
5665 * happen. Never use yield() as a progress guarantee!!
5666 *
5667 * If you want to use yield() to wait for something, use wait_event().
5668 * If you want to use yield() to be 'nice' for others, use cond_resched().
5669 * If you still want to use yield(), do not!
5670 */
5671void __sched yield(void)
5672{
5673	set_current_state(TASK_RUNNING);
5674	do_sched_yield();
5675}
5676EXPORT_SYMBOL(yield);
5677
5678/**
5679 * yield_to - yield the current processor to another thread in
5680 * your thread group, or accelerate that thread toward the
5681 * processor it's on.
5682 * @p: target task
5683 * @preempt: whether task preemption is allowed or not
5684 *
5685 * It's the caller's job to ensure that the target task struct
5686 * can't go away on us before we can do any checks.
5687 *
5688 * Return:
5689 *	true (>0) if we indeed boosted the target task.
5690 *	false (0) if we failed to boost the target.
5691 *	-ESRCH if there's no task to yield to.
5692 */
5693int __sched yield_to(struct task_struct *p, bool preempt)
5694{
5695	struct task_struct *curr = current;
5696	struct rq *rq, *p_rq;
5697	unsigned long flags;
5698	int yielded = 0;
5699
5700	local_irq_save(flags);
5701	rq = this_rq();
5702
5703again:
5704	p_rq = task_rq(p);
5705	/*
5706	 * If we're the only runnable task on the rq and target rq also
5707	 * has only one task, there's absolutely no point in yielding.
5708	 */
5709	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
5710		yielded = -ESRCH;
5711		goto out_irq;
5712	}
5713
5714	double_rq_lock(rq, p_rq);
5715	if (task_rq(p) != p_rq) {
5716		double_rq_unlock(rq, p_rq);
5717		goto again;
5718	}
5719
5720	if (!curr->sched_class->yield_to_task)
5721		goto out_unlock;
5722
5723	if (curr->sched_class != p->sched_class)
5724		goto out_unlock;
5725
5726	if (task_running(p_rq, p) || p->state)
5727		goto out_unlock;
5728
5729	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
5730	if (yielded) {
5731		schedstat_inc(rq->yld_count);
5732		/*
5733		 * Make p's CPU reschedule; pick_next_entity takes care of
5734		 * fairness.
5735		 */
5736		if (preempt && rq != p_rq)
5737			resched_curr(p_rq);
5738	}
5739
5740out_unlock:
5741	double_rq_unlock(rq, p_rq);
5742out_irq:
5743	local_irq_restore(flags);
5744
5745	if (yielded > 0)
5746		schedule();
5747
5748	return yielded;
5749}
5750EXPORT_SYMBOL_GPL(yield_to);
5751
5752int io_schedule_prepare(void)
5753{
5754	int old_iowait = current->in_iowait;
5755
5756	current->in_iowait = 1;
5757	blk_schedule_flush_plug(current);
5758
5759	return old_iowait;
5760}
5761
5762void io_schedule_finish(int token)
5763{
5764	current->in_iowait = token;
5765}
5766
5767/*
5768 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
5769 * that process accounting knows that this is a task in IO wait state.
5770 */
5771long __sched io_schedule_timeout(long timeout)
5772{
5773	int token;
5774	long ret;
5775
5776	token = io_schedule_prepare();
5777	ret = schedule_timeout(timeout);
5778	io_schedule_finish(token);
5779
5780	return ret;
5781}
5782EXPORT_SYMBOL(io_schedule_timeout);
5783
5784void __sched io_schedule(void)
5785{
5786	int token;
5787
5788	token = io_schedule_prepare();
5789	schedule();
5790	io_schedule_finish(token);
5791}
5792EXPORT_SYMBOL(io_schedule);
5793
5794/**
5795 * sys_sched_get_priority_max - return maximum RT priority.
5796 * @policy: scheduling class.
5797 *
5798 * Return: On success, this syscall returns the maximum
5799 * rt_priority that can be used by a given scheduling class.
5800 * On failure, a negative error code is returned.
5801 */
5802SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
5803{
5804	int ret = -EINVAL;
5805
5806	switch (policy) {
5807	case SCHED_FIFO:
5808	case SCHED_RR:
5809		ret = MAX_USER_RT_PRIO-1;
5810		break;
5811	case SCHED_DEADLINE:
5812	case SCHED_NORMAL:
5813	case SCHED_BATCH:
5814	case SCHED_IDLE:
5815		ret = 0;
5816		break;
5817	}
5818	return ret;
5819}
5820
5821/**
5822 * sys_sched_get_priority_min - return minimum RT priority.
5823 * @policy: scheduling class.
5824 *
5825 * Return: On success, this syscall returns the minimum
5826 * rt_priority that can be used by a given scheduling class.
5827 * On failure, a negative error code is returned.
5828 */
5829SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
5830{
5831	int ret = -EINVAL;
5832
5833	switch (policy) {
5834	case SCHED_FIFO:
5835	case SCHED_RR:
5836		ret = 1;
5837		break;
5838	case SCHED_DEADLINE:
5839	case SCHED_NORMAL:
5840	case SCHED_BATCH:
5841	case SCHED_IDLE:
5842		ret = 0;
5843	}
5844	return ret;
5845}
5846
5847static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
5848{
5849	struct task_struct *p;
5850	unsigned int time_slice;
5851	struct rq_flags rf;
5852	struct rq *rq;
5853	int retval;
5854
5855	if (pid < 0)
5856		return -EINVAL;
5857
5858	retval = -ESRCH;
5859	rcu_read_lock();
5860	p = find_process_by_pid(pid);
5861	if (!p)
5862		goto out_unlock;
5863
5864	retval = security_task_getscheduler(p);
5865	if (retval)
5866		goto out_unlock;
5867
5868	rq = task_rq_lock(p, &rf);
5869	time_slice = 0;
5870	if (p->sched_class->get_rr_interval)
5871		time_slice = p->sched_class->get_rr_interval(rq, p);
5872	task_rq_unlock(rq, p, &rf);
5873
5874	rcu_read_unlock();
5875	jiffies_to_timespec64(time_slice, t);
5876	return 0;
5877
5878out_unlock:
5879	rcu_read_unlock();
5880	return retval;
5881}
5882
5883/**
5884 * sys_sched_rr_get_interval - return the default timeslice of a process.
5885 * @pid: pid of the process.
5886 * @interval: userspace pointer to the timeslice value.
5887 *
5888 * this syscall writes the default timeslice value of a given process
5889 * into the user-space timespec buffer. A value of '0' means infinity.
5890 *
5891 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
5892 * an error code.
5893 */
5894SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5895		struct __kernel_timespec __user *, interval)
5896{
5897	struct timespec64 t;
5898	int retval = sched_rr_get_interval(pid, &t);
5899
5900	if (retval == 0)
5901		retval = put_timespec64(&t, interval);
5902
5903	return retval;
5904}
5905
5906#ifdef CONFIG_COMPAT_32BIT_TIME
5907SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid,
5908		struct old_timespec32 __user *, interval)
 
5909{
5910	struct timespec64 t;
5911	int retval = sched_rr_get_interval(pid, &t);
5912
5913	if (retval == 0)
5914		retval = put_old_timespec32(&t, interval);
5915	return retval;
5916}
5917#endif
5918
5919void sched_show_task(struct task_struct *p)
5920{
5921	unsigned long free = 0;
5922	int ppid;
5923
5924	if (!try_get_task_stack(p))
5925		return;
5926
5927	printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p));
5928
5929	if (p->state == TASK_RUNNING)
5930		printk(KERN_CONT "  running task    ");
5931#ifdef CONFIG_DEBUG_STACK_USAGE
5932	free = stack_not_used(p);
5933#endif
5934	ppid = 0;
5935	rcu_read_lock();
5936	if (pid_alive(p))
5937		ppid = task_pid_nr(rcu_dereference(p->real_parent));
5938	rcu_read_unlock();
5939	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5940		task_pid_nr(p), ppid,
5941		(unsigned long)task_thread_info(p)->flags);
5942
5943	print_worker_info(KERN_INFO, p);
5944	show_stack(p, NULL);
5945	put_task_stack(p);
5946}
5947EXPORT_SYMBOL_GPL(sched_show_task);
5948
5949static inline bool
5950state_filter_match(unsigned long state_filter, struct task_struct *p)
5951{
5952	/* no filter, everything matches */
5953	if (!state_filter)
5954		return true;
5955
5956	/* filter, but doesn't match */
5957	if (!(p->state & state_filter))
5958		return false;
5959
5960	/*
5961	 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
5962	 * TASK_KILLABLE).
5963	 */
5964	if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE)
5965		return false;
5966
5967	return true;
5968}
5969
5970
5971void show_state_filter(unsigned long state_filter)
5972{
5973	struct task_struct *g, *p;
5974
5975#if BITS_PER_LONG == 32
5976	printk(KERN_INFO
5977		"  task                PC stack   pid father\n");
5978#else
5979	printk(KERN_INFO
5980		"  task                        PC stack   pid father\n");
5981#endif
5982	rcu_read_lock();
5983	for_each_process_thread(g, p) {
5984		/*
5985		 * reset the NMI-timeout, listing all files on a slow
5986		 * console might take a lot of time:
5987		 * Also, reset softlockup watchdogs on all CPUs, because
5988		 * another CPU might be blocked waiting for us to process
5989		 * an IPI.
5990		 */
5991		touch_nmi_watchdog();
5992		touch_all_softlockup_watchdogs();
5993		if (state_filter_match(state_filter, p))
5994			sched_show_task(p);
5995	}
5996
5997#ifdef CONFIG_SCHED_DEBUG
5998	if (!state_filter)
5999		sysrq_sched_debug_show();
6000#endif
6001	rcu_read_unlock();
6002	/*
6003	 * Only show locks if all tasks are dumped:
6004	 */
6005	if (!state_filter)
6006		debug_show_all_locks();
6007}
6008
6009/**
6010 * init_idle - set up an idle thread for a given CPU
6011 * @idle: task in question
6012 * @cpu: CPU the idle task belongs to
6013 *
6014 * NOTE: this function does not set the idle thread's NEED_RESCHED
6015 * flag, to make booting more robust.
6016 */
6017void init_idle(struct task_struct *idle, int cpu)
6018{
6019	struct rq *rq = cpu_rq(cpu);
6020	unsigned long flags;
6021
6022	__sched_fork(0, idle);
6023
6024	raw_spin_lock_irqsave(&idle->pi_lock, flags);
6025	raw_spin_lock(&rq->lock);
6026
 
6027	idle->state = TASK_RUNNING;
6028	idle->se.exec_start = sched_clock();
6029	idle->flags |= PF_IDLE;
6030
6031	kasan_unpoison_task_stack(idle);
6032
6033#ifdef CONFIG_SMP
6034	/*
6035	 * Its possible that init_idle() gets called multiple times on a task,
6036	 * in that case do_set_cpus_allowed() will not do the right thing.
6037	 *
6038	 * And since this is boot we can forgo the serialization.
6039	 */
6040	set_cpus_allowed_common(idle, cpumask_of(cpu));
6041#endif
6042	/*
6043	 * We're having a chicken and egg problem, even though we are
6044	 * holding rq->lock, the CPU isn't yet set to this CPU so the
6045	 * lockdep check in task_group() will fail.
6046	 *
6047	 * Similar case to sched_fork(). / Alternatively we could
6048	 * use task_rq_lock() here and obtain the other rq->lock.
6049	 *
6050	 * Silence PROVE_RCU
6051	 */
6052	rcu_read_lock();
6053	__set_task_cpu(idle, cpu);
6054	rcu_read_unlock();
6055
6056	rq->idle = idle;
6057	rcu_assign_pointer(rq->curr, idle);
6058	idle->on_rq = TASK_ON_RQ_QUEUED;
6059#ifdef CONFIG_SMP
6060	idle->on_cpu = 1;
6061#endif
6062	raw_spin_unlock(&rq->lock);
6063	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
6064
6065	/* Set the preempt count _outside_ the spinlocks! */
6066	init_idle_preempt_count(idle, cpu);
6067
6068	/*
6069	 * The idle tasks have their own, simple scheduling class:
6070	 */
6071	idle->sched_class = &idle_sched_class;
6072	ftrace_graph_init_idle_task(idle, cpu);
6073	vtime_init_idle(idle, cpu);
6074#ifdef CONFIG_SMP
6075	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
6076#endif
6077}
6078
6079#ifdef CONFIG_SMP
6080
6081int cpuset_cpumask_can_shrink(const struct cpumask *cur,
6082			      const struct cpumask *trial)
6083{
6084	int ret = 1;
6085
6086	if (!cpumask_weight(cur))
6087		return ret;
6088
6089	ret = dl_cpuset_cpumask_can_shrink(cur, trial);
6090
6091	return ret;
6092}
6093
6094int task_can_attach(struct task_struct *p,
6095		    const struct cpumask *cs_cpus_allowed)
6096{
6097	int ret = 0;
6098
6099	/*
6100	 * Kthreads which disallow setaffinity shouldn't be moved
6101	 * to a new cpuset; we don't want to change their CPU
6102	 * affinity and isolating such threads by their set of
6103	 * allowed nodes is unnecessary.  Thus, cpusets are not
6104	 * applicable for such threads.  This prevents checking for
6105	 * success of set_cpus_allowed_ptr() on all attached tasks
6106	 * before cpus_mask may be changed.
6107	 */
6108	if (p->flags & PF_NO_SETAFFINITY) {
6109		ret = -EINVAL;
6110		goto out;
6111	}
6112
6113	if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
6114					      cs_cpus_allowed))
6115		ret = dl_task_can_attach(p, cs_cpus_allowed);
6116
6117out:
6118	return ret;
6119}
6120
6121bool sched_smp_initialized __read_mostly;
6122
6123#ifdef CONFIG_NUMA_BALANCING
6124/* Migrate current task p to target_cpu */
6125int migrate_task_to(struct task_struct *p, int target_cpu)
6126{
6127	struct migration_arg arg = { p, target_cpu };
6128	int curr_cpu = task_cpu(p);
6129
6130	if (curr_cpu == target_cpu)
6131		return 0;
6132
6133	if (!cpumask_test_cpu(target_cpu, p->cpus_ptr))
6134		return -EINVAL;
6135
6136	/* TODO: This is not properly updating schedstats */
6137
6138	trace_sched_move_numa(p, curr_cpu, target_cpu);
6139	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
6140}
6141
6142/*
6143 * Requeue a task on a given node and accurately track the number of NUMA
6144 * tasks on the runqueues
6145 */
6146void sched_setnuma(struct task_struct *p, int nid)
6147{
6148	bool queued, running;
6149	struct rq_flags rf;
6150	struct rq *rq;
6151
6152	rq = task_rq_lock(p, &rf);
6153	queued = task_on_rq_queued(p);
6154	running = task_current(rq, p);
6155
6156	if (queued)
6157		dequeue_task(rq, p, DEQUEUE_SAVE);
6158	if (running)
6159		put_prev_task(rq, p);
6160
6161	p->numa_preferred_nid = nid;
6162
6163	if (queued)
6164		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
6165	if (running)
6166		set_next_task(rq, p);
6167	task_rq_unlock(rq, p, &rf);
6168}
6169#endif /* CONFIG_NUMA_BALANCING */
6170
6171#ifdef CONFIG_HOTPLUG_CPU
6172/*
6173 * Ensure that the idle task is using init_mm right before its CPU goes
6174 * offline.
6175 */
6176void idle_task_exit(void)
6177{
6178	struct mm_struct *mm = current->active_mm;
6179
6180	BUG_ON(cpu_online(smp_processor_id()));
6181
6182	if (mm != &init_mm) {
6183		switch_mm(mm, &init_mm, current);
6184		current->active_mm = &init_mm;
6185		finish_arch_post_lock_switch();
6186	}
6187	mmdrop(mm);
6188}
6189
6190/*
6191 * Since this CPU is going 'away' for a while, fold any nr_active delta
6192 * we might have. Assumes we're called after migrate_tasks() so that the
6193 * nr_active count is stable. We need to take the teardown thread which
6194 * is calling this into account, so we hand in adjust = 1 to the load
6195 * calculation.
6196 *
6197 * Also see the comment "Global load-average calculations".
6198 */
6199static void calc_load_migrate(struct rq *rq)
6200{
6201	long delta = calc_load_fold_active(rq, 1);
6202	if (delta)
6203		atomic_long_add(delta, &calc_load_tasks);
6204}
6205
6206static struct task_struct *__pick_migrate_task(struct rq *rq)
6207{
6208	const struct sched_class *class;
6209	struct task_struct *next;
6210
6211	for_each_class(class) {
6212		next = class->pick_next_task(rq, NULL, NULL);
6213		if (next) {
6214			next->sched_class->put_prev_task(rq, next);
6215			return next;
6216		}
6217	}
6218
6219	/* The idle class should always have a runnable task */
6220	BUG();
6221}
 
 
 
 
6222
6223/*
6224 * Migrate all tasks from the rq, sleeping tasks will be migrated by
6225 * try_to_wake_up()->select_task_rq().
6226 *
6227 * Called with rq->lock held even though we'er in stop_machine() and
6228 * there's no concurrency possible, we hold the required locks anyway
6229 * because of lock validation efforts.
6230 */
6231static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
6232{
6233	struct rq *rq = dead_rq;
6234	struct task_struct *next, *stop = rq->stop;
6235	struct rq_flags orf = *rf;
6236	int dest_cpu;
6237
6238	/*
6239	 * Fudge the rq selection such that the below task selection loop
6240	 * doesn't get stuck on the currently eligible stop task.
6241	 *
6242	 * We're currently inside stop_machine() and the rq is either stuck
6243	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
6244	 * either way we should never end up calling schedule() until we're
6245	 * done here.
6246	 */
6247	rq->stop = NULL;
6248
6249	/*
6250	 * put_prev_task() and pick_next_task() sched
6251	 * class method both need to have an up-to-date
6252	 * value of rq->clock[_task]
6253	 */
6254	update_rq_clock(rq);
6255
6256	for (;;) {
6257		/*
6258		 * There's this thread running, bail when that's the only
6259		 * remaining thread:
6260		 */
6261		if (rq->nr_running == 1)
6262			break;
6263
6264		next = __pick_migrate_task(rq);
 
 
 
 
 
6265
6266		/*
6267		 * Rules for changing task_struct::cpus_mask are holding
6268		 * both pi_lock and rq->lock, such that holding either
6269		 * stabilizes the mask.
6270		 *
6271		 * Drop rq->lock is not quite as disastrous as it usually is
6272		 * because !cpu_active at this point, which means load-balance
6273		 * will not interfere. Also, stop-machine.
6274		 */
6275		rq_unlock(rq, rf);
6276		raw_spin_lock(&next->pi_lock);
6277		rq_relock(rq, rf);
6278
6279		/*
6280		 * Since we're inside stop-machine, _nothing_ should have
6281		 * changed the task, WARN if weird stuff happened, because in
6282		 * that case the above rq->lock drop is a fail too.
6283		 */
6284		if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
6285			raw_spin_unlock(&next->pi_lock);
6286			continue;
6287		}
6288
6289		/* Find suitable destination for @next, with force if needed. */
6290		dest_cpu = select_fallback_rq(dead_rq->cpu, next);
6291		rq = __migrate_task(rq, rf, next, dest_cpu);
6292		if (rq != dead_rq) {
6293			rq_unlock(rq, rf);
6294			rq = dead_rq;
6295			*rf = orf;
6296			rq_relock(rq, rf);
6297		}
6298		raw_spin_unlock(&next->pi_lock);
6299	}
6300
6301	rq->stop = stop;
6302}
6303#endif /* CONFIG_HOTPLUG_CPU */
6304
6305void set_rq_online(struct rq *rq)
6306{
6307	if (!rq->online) {
6308		const struct sched_class *class;
6309
6310		cpumask_set_cpu(rq->cpu, rq->rd->online);
6311		rq->online = 1;
6312
6313		for_each_class(class) {
6314			if (class->rq_online)
6315				class->rq_online(rq);
6316		}
6317	}
6318}
6319
6320void set_rq_offline(struct rq *rq)
6321{
6322	if (rq->online) {
6323		const struct sched_class *class;
6324
6325		for_each_class(class) {
6326			if (class->rq_offline)
6327				class->rq_offline(rq);
6328		}
6329
6330		cpumask_clear_cpu(rq->cpu, rq->rd->online);
6331		rq->online = 0;
6332	}
6333}
6334
 
 
 
 
 
 
 
6335/*
6336 * used to mark begin/end of suspend/resume:
6337 */
6338static int num_cpus_frozen;
6339
6340/*
6341 * Update cpusets according to cpu_active mask.  If cpusets are
6342 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
6343 * around partition_sched_domains().
6344 *
6345 * If we come here as part of a suspend/resume, don't touch cpusets because we
6346 * want to restore it back to its original state upon resume anyway.
6347 */
6348static void cpuset_cpu_active(void)
6349{
6350	if (cpuhp_tasks_frozen) {
6351		/*
6352		 * num_cpus_frozen tracks how many CPUs are involved in suspend
6353		 * resume sequence. As long as this is not the last online
6354		 * operation in the resume sequence, just build a single sched
6355		 * domain, ignoring cpusets.
6356		 */
6357		partition_sched_domains(1, NULL, NULL);
6358		if (--num_cpus_frozen)
6359			return;
6360		/*
6361		 * This is the last CPU online operation. So fall through and
6362		 * restore the original sched domains by considering the
6363		 * cpuset configurations.
6364		 */
6365		cpuset_force_rebuild();
6366	}
6367	cpuset_update_active_cpus();
6368}
6369
6370static int cpuset_cpu_inactive(unsigned int cpu)
6371{
6372	if (!cpuhp_tasks_frozen) {
6373		if (dl_cpu_busy(cpu))
6374			return -EBUSY;
6375		cpuset_update_active_cpus();
6376	} else {
6377		num_cpus_frozen++;
6378		partition_sched_domains(1, NULL, NULL);
6379	}
6380	return 0;
6381}
6382
6383int sched_cpu_activate(unsigned int cpu)
6384{
6385	struct rq *rq = cpu_rq(cpu);
6386	struct rq_flags rf;
6387
6388#ifdef CONFIG_SCHED_SMT
6389	/*
6390	 * When going up, increment the number of cores with SMT present.
6391	 */
6392	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
6393		static_branch_inc_cpuslocked(&sched_smt_present);
6394#endif
6395	set_cpu_active(cpu, true);
6396
6397	if (sched_smp_initialized) {
6398		sched_domains_numa_masks_set(cpu);
6399		cpuset_cpu_active();
6400	}
6401
6402	/*
6403	 * Put the rq online, if not already. This happens:
6404	 *
6405	 * 1) In the early boot process, because we build the real domains
6406	 *    after all CPUs have been brought up.
6407	 *
6408	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
6409	 *    domains.
6410	 */
6411	rq_lock_irqsave(rq, &rf);
6412	if (rq->rd) {
6413		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6414		set_rq_online(rq);
6415	}
6416	rq_unlock_irqrestore(rq, &rf);
6417
 
 
6418	return 0;
6419}
6420
6421int sched_cpu_deactivate(unsigned int cpu)
6422{
6423	int ret;
6424
6425	set_cpu_active(cpu, false);
6426	/*
6427	 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
6428	 * users of this state to go away such that all new such users will
6429	 * observe it.
6430	 *
6431	 * Do sync before park smpboot threads to take care the rcu boost case.
6432	 */
6433	synchronize_rcu();
6434
6435#ifdef CONFIG_SCHED_SMT
6436	/*
6437	 * When going down, decrement the number of cores with SMT present.
6438	 */
6439	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
6440		static_branch_dec_cpuslocked(&sched_smt_present);
6441#endif
6442
6443	if (!sched_smp_initialized)
6444		return 0;
6445
6446	ret = cpuset_cpu_inactive(cpu);
6447	if (ret) {
6448		set_cpu_active(cpu, true);
6449		return ret;
6450	}
6451	sched_domains_numa_masks_clear(cpu);
6452	return 0;
6453}
6454
6455static void sched_rq_cpu_starting(unsigned int cpu)
6456{
6457	struct rq *rq = cpu_rq(cpu);
6458
6459	rq->calc_load_update = calc_load_update;
6460	update_max_interval();
6461}
6462
6463int sched_cpu_starting(unsigned int cpu)
6464{
 
6465	sched_rq_cpu_starting(cpu);
6466	sched_tick_start(cpu);
6467	return 0;
6468}
6469
6470#ifdef CONFIG_HOTPLUG_CPU
6471int sched_cpu_dying(unsigned int cpu)
6472{
6473	struct rq *rq = cpu_rq(cpu);
6474	struct rq_flags rf;
6475
6476	/* Handle pending wakeups and then migrate everything off */
6477	sched_ttwu_pending();
6478	sched_tick_stop(cpu);
6479
6480	rq_lock_irqsave(rq, &rf);
6481	if (rq->rd) {
6482		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6483		set_rq_offline(rq);
6484	}
6485	migrate_tasks(rq, &rf);
6486	BUG_ON(rq->nr_running != 1);
6487	rq_unlock_irqrestore(rq, &rf);
6488
6489	calc_load_migrate(rq);
6490	update_max_interval();
6491	nohz_balance_exit_idle(rq);
6492	hrtick_clear(rq);
6493	return 0;
6494}
6495#endif
6496
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6497void __init sched_init_smp(void)
6498{
6499	sched_init_numa();
6500
6501	/*
6502	 * There's no userspace yet to cause hotplug operations; hence all the
6503	 * CPU masks are stable and all blatant races in the below code cannot
6504	 * happen.
6505	 */
6506	mutex_lock(&sched_domains_mutex);
6507	sched_init_domains(cpu_active_mask);
6508	mutex_unlock(&sched_domains_mutex);
6509
6510	/* Move init over to a non-isolated CPU */
6511	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
6512		BUG();
6513	sched_init_granularity();
6514
6515	init_sched_rt_class();
6516	init_sched_dl_class();
6517
 
 
6518	sched_smp_initialized = true;
6519}
6520
6521static int __init migration_init(void)
6522{
6523	sched_cpu_starting(smp_processor_id());
6524	return 0;
6525}
6526early_initcall(migration_init);
6527
6528#else
6529void __init sched_init_smp(void)
6530{
6531	sched_init_granularity();
6532}
6533#endif /* CONFIG_SMP */
6534
6535int in_sched_functions(unsigned long addr)
6536{
6537	return in_lock_functions(addr) ||
6538		(addr >= (unsigned long)__sched_text_start
6539		&& addr < (unsigned long)__sched_text_end);
6540}
6541
6542#ifdef CONFIG_CGROUP_SCHED
6543/*
6544 * Default task group.
6545 * Every task in system belongs to this group at bootup.
6546 */
6547struct task_group root_task_group;
6548LIST_HEAD(task_groups);
6549
6550/* Cacheline aligned slab cache for task_group */
6551static struct kmem_cache *task_group_cache __read_mostly;
6552#endif
6553
6554DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
6555DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
6556
6557void __init sched_init(void)
6558{
6559	unsigned long ptr = 0;
6560	int i;
6561
 
6562	wait_bit_init();
6563
6564#ifdef CONFIG_FAIR_GROUP_SCHED
6565	ptr += 2 * nr_cpu_ids * sizeof(void **);
6566#endif
6567#ifdef CONFIG_RT_GROUP_SCHED
6568	ptr += 2 * nr_cpu_ids * sizeof(void **);
6569#endif
6570	if (ptr) {
6571		ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
6572
6573#ifdef CONFIG_FAIR_GROUP_SCHED
6574		root_task_group.se = (struct sched_entity **)ptr;
6575		ptr += nr_cpu_ids * sizeof(void **);
6576
6577		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6578		ptr += nr_cpu_ids * sizeof(void **);
6579
6580#endif /* CONFIG_FAIR_GROUP_SCHED */
6581#ifdef CONFIG_RT_GROUP_SCHED
6582		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
6583		ptr += nr_cpu_ids * sizeof(void **);
6584
6585		root_task_group.rt_rq = (struct rt_rq **)ptr;
6586		ptr += nr_cpu_ids * sizeof(void **);
6587
6588#endif /* CONFIG_RT_GROUP_SCHED */
6589	}
6590#ifdef CONFIG_CPUMASK_OFFSTACK
6591	for_each_possible_cpu(i) {
6592		per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
6593			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
6594		per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
6595			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
6596	}
6597#endif /* CONFIG_CPUMASK_OFFSTACK */
6598
6599	init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime());
6600	init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime());
6601
6602#ifdef CONFIG_SMP
6603	init_defrootdomain();
6604#endif
6605
6606#ifdef CONFIG_RT_GROUP_SCHED
6607	init_rt_bandwidth(&root_task_group.rt_bandwidth,
6608			global_rt_period(), global_rt_runtime());
6609#endif /* CONFIG_RT_GROUP_SCHED */
6610
6611#ifdef CONFIG_CGROUP_SCHED
6612	task_group_cache = KMEM_CACHE(task_group, 0);
6613
6614	list_add(&root_task_group.list, &task_groups);
6615	INIT_LIST_HEAD(&root_task_group.children);
6616	INIT_LIST_HEAD(&root_task_group.siblings);
6617	autogroup_init(&init_task);
6618#endif /* CONFIG_CGROUP_SCHED */
6619
6620	for_each_possible_cpu(i) {
6621		struct rq *rq;
6622
6623		rq = cpu_rq(i);
6624		raw_spin_lock_init(&rq->lock);
6625		rq->nr_running = 0;
6626		rq->calc_load_active = 0;
6627		rq->calc_load_update = jiffies + LOAD_FREQ;
6628		init_cfs_rq(&rq->cfs);
6629		init_rt_rq(&rq->rt);
6630		init_dl_rq(&rq->dl);
6631#ifdef CONFIG_FAIR_GROUP_SCHED
6632		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
6633		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
6634		rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
6635		/*
6636		 * How much CPU bandwidth does root_task_group get?
6637		 *
6638		 * In case of task-groups formed thr' the cgroup filesystem, it
6639		 * gets 100% of the CPU resources in the system. This overall
6640		 * system CPU resource is divided among the tasks of
6641		 * root_task_group and its child task-groups in a fair manner,
6642		 * based on each entity's (task or task-group's) weight
6643		 * (se->load.weight).
6644		 *
6645		 * In other words, if root_task_group has 10 tasks of weight
6646		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
6647		 * then A0's share of the CPU resource is:
6648		 *
6649		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
6650		 *
6651		 * We achieve this by letting root_task_group's tasks sit
6652		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
6653		 */
6654		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
6655		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
6656#endif /* CONFIG_FAIR_GROUP_SCHED */
6657
6658		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
6659#ifdef CONFIG_RT_GROUP_SCHED
6660		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
6661#endif
 
 
 
 
6662#ifdef CONFIG_SMP
6663		rq->sd = NULL;
6664		rq->rd = NULL;
6665		rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
6666		rq->balance_callback = NULL;
6667		rq->active_balance = 0;
6668		rq->next_balance = jiffies;
6669		rq->push_cpu = 0;
6670		rq->cpu = i;
6671		rq->online = 0;
6672		rq->idle_stamp = 0;
6673		rq->avg_idle = 2*sysctl_sched_migration_cost;
6674		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
6675
6676		INIT_LIST_HEAD(&rq->cfs_tasks);
6677
6678		rq_attach_root(rq, &def_root_domain);
6679#ifdef CONFIG_NO_HZ_COMMON
6680		rq->last_load_update_tick = jiffies;
6681		rq->last_blocked_load_update_tick = jiffies;
6682		atomic_set(&rq->nohz_flags, 0);
6683#endif
6684#endif /* CONFIG_SMP */
6685		hrtick_rq_init(rq);
6686		atomic_set(&rq->nr_iowait, 0);
6687	}
6688
6689	set_load_weight(&init_task, false);
6690
6691	/*
6692	 * The boot idle thread does lazy MMU switching as well:
6693	 */
6694	mmgrab(&init_mm);
6695	enter_lazy_tlb(&init_mm, current);
6696
6697	/*
6698	 * Make us the idle thread. Technically, schedule() should not be
6699	 * called from this thread, however somewhere below it might be,
6700	 * but because we are the idle thread, we just pick up running again
6701	 * when this runqueue becomes "idle".
6702	 */
6703	init_idle(current, smp_processor_id());
6704
6705	calc_load_update = jiffies + LOAD_FREQ;
6706
6707#ifdef CONFIG_SMP
6708	idle_thread_set_boot_cpu();
 
6709#endif
6710	init_sched_fair_class();
6711
6712	init_schedstats();
6713
6714	psi_init();
6715
6716	init_uclamp();
6717
6718	scheduler_running = 1;
6719}
6720
6721#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
6722static inline int preempt_count_equals(int preempt_offset)
6723{
6724	int nested = preempt_count() + rcu_preempt_depth();
6725
6726	return (nested == preempt_offset);
6727}
6728
6729void __might_sleep(const char *file, int line, int preempt_offset)
6730{
6731	/*
6732	 * Blocking primitives will set (and therefore destroy) current->state,
6733	 * since we will exit with TASK_RUNNING make sure we enter with it,
6734	 * otherwise we will destroy state.
6735	 */
6736	WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
6737			"do not call blocking ops when !TASK_RUNNING; "
6738			"state=%lx set at [<%p>] %pS\n",
6739			current->state,
6740			(void *)current->task_state_change,
6741			(void *)current->task_state_change);
6742
6743	___might_sleep(file, line, preempt_offset);
6744}
6745EXPORT_SYMBOL(__might_sleep);
6746
6747void ___might_sleep(const char *file, int line, int preempt_offset)
6748{
6749	/* Ratelimiting timestamp: */
6750	static unsigned long prev_jiffy;
6751
6752	unsigned long preempt_disable_ip;
6753
6754	/* WARN_ON_ONCE() by default, no rate limit required: */
6755	rcu_sleep_check();
6756
6757	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
6758	     !is_idle_task(current) && !current->non_block_count) ||
6759	    system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
6760	    oops_in_progress)
6761		return;
6762
6763	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
6764		return;
6765	prev_jiffy = jiffies;
6766
6767	/* Save this before calling printk(), since that will clobber it: */
6768	preempt_disable_ip = get_preempt_disable_ip(current);
6769
6770	printk(KERN_ERR
6771		"BUG: sleeping function called from invalid context at %s:%d\n",
6772			file, line);
6773	printk(KERN_ERR
6774		"in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
6775			in_atomic(), irqs_disabled(), current->non_block_count,
6776			current->pid, current->comm);
6777
6778	if (task_stack_end_corrupted(current))
6779		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
6780
6781	debug_show_held_locks(current);
6782	if (irqs_disabled())
6783		print_irqtrace_events(current);
6784	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
6785	    && !preempt_count_equals(preempt_offset)) {
6786		pr_err("Preemption disabled at:");
6787		print_ip_sym(preempt_disable_ip);
6788		pr_cont("\n");
6789	}
6790	dump_stack();
6791	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
6792}
6793EXPORT_SYMBOL(___might_sleep);
6794
6795void __cant_sleep(const char *file, int line, int preempt_offset)
6796{
6797	static unsigned long prev_jiffy;
6798
6799	if (irqs_disabled())
6800		return;
6801
6802	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
6803		return;
6804
6805	if (preempt_count() > preempt_offset)
6806		return;
6807
6808	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
6809		return;
6810	prev_jiffy = jiffies;
6811
6812	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
6813	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
6814			in_atomic(), irqs_disabled(),
6815			current->pid, current->comm);
6816
6817	debug_show_held_locks(current);
6818	dump_stack();
6819	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
6820}
6821EXPORT_SYMBOL_GPL(__cant_sleep);
6822#endif
6823
6824#ifdef CONFIG_MAGIC_SYSRQ
6825void normalize_rt_tasks(void)
6826{
6827	struct task_struct *g, *p;
6828	struct sched_attr attr = {
6829		.sched_policy = SCHED_NORMAL,
6830	};
6831
6832	read_lock(&tasklist_lock);
6833	for_each_process_thread(g, p) {
6834		/*
6835		 * Only normalize user tasks:
6836		 */
6837		if (p->flags & PF_KTHREAD)
6838			continue;
6839
6840		p->se.exec_start = 0;
6841		schedstat_set(p->se.statistics.wait_start,  0);
6842		schedstat_set(p->se.statistics.sleep_start, 0);
6843		schedstat_set(p->se.statistics.block_start, 0);
6844
6845		if (!dl_task(p) && !rt_task(p)) {
6846			/*
6847			 * Renice negative nice level userspace
6848			 * tasks back to 0:
6849			 */
6850			if (task_nice(p) < 0)
6851				set_user_nice(p, 0);
6852			continue;
6853		}
6854
6855		__sched_setscheduler(p, &attr, false, false);
6856	}
6857	read_unlock(&tasklist_lock);
6858}
6859
6860#endif /* CONFIG_MAGIC_SYSRQ */
6861
6862#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
6863/*
6864 * These functions are only useful for the IA64 MCA handling, or kdb.
6865 *
6866 * They can only be called when the whole system has been
6867 * stopped - every CPU needs to be quiescent, and no scheduling
6868 * activity can take place. Using them for anything else would
6869 * be a serious bug, and as a result, they aren't even visible
6870 * under any other configuration.
6871 */
6872
6873/**
6874 * curr_task - return the current task for a given CPU.
6875 * @cpu: the processor in question.
6876 *
6877 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
6878 *
6879 * Return: The current task for @cpu.
6880 */
6881struct task_struct *curr_task(int cpu)
6882{
6883	return cpu_curr(cpu);
6884}
6885
6886#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
6887
6888#ifdef CONFIG_IA64
6889/**
6890 * ia64_set_curr_task - set the current task for a given CPU.
6891 * @cpu: the processor in question.
6892 * @p: the task pointer to set.
6893 *
6894 * Description: This function must only be used when non-maskable interrupts
6895 * are serviced on a separate stack. It allows the architecture to switch the
6896 * notion of the current task on a CPU in a non-blocking manner. This function
6897 * must be called with all CPU's synchronized, and interrupts disabled, the
6898 * and caller must save the original value of the current task (see
6899 * curr_task() above) and restore that value before reenabling interrupts and
6900 * re-starting the system.
6901 *
6902 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
6903 */
6904void ia64_set_curr_task(int cpu, struct task_struct *p)
6905{
6906	cpu_curr(cpu) = p;
6907}
6908
6909#endif
6910
6911#ifdef CONFIG_CGROUP_SCHED
6912/* task_group_lock serializes the addition/removal of task groups */
6913static DEFINE_SPINLOCK(task_group_lock);
6914
6915static inline void alloc_uclamp_sched_group(struct task_group *tg,
6916					    struct task_group *parent)
6917{
6918#ifdef CONFIG_UCLAMP_TASK_GROUP
6919	enum uclamp_id clamp_id;
6920
6921	for_each_clamp_id(clamp_id) {
6922		uclamp_se_set(&tg->uclamp_req[clamp_id],
6923			      uclamp_none(clamp_id), false);
6924		tg->uclamp[clamp_id] = parent->uclamp[clamp_id];
6925	}
6926#endif
6927}
6928
6929static void sched_free_group(struct task_group *tg)
6930{
6931	free_fair_sched_group(tg);
6932	free_rt_sched_group(tg);
6933	autogroup_free(tg);
6934	kmem_cache_free(task_group_cache, tg);
6935}
6936
6937/* allocate runqueue etc for a new task group */
6938struct task_group *sched_create_group(struct task_group *parent)
6939{
6940	struct task_group *tg;
6941
6942	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
6943	if (!tg)
6944		return ERR_PTR(-ENOMEM);
6945
6946	if (!alloc_fair_sched_group(tg, parent))
6947		goto err;
6948
6949	if (!alloc_rt_sched_group(tg, parent))
6950		goto err;
6951
6952	alloc_uclamp_sched_group(tg, parent);
6953
6954	return tg;
6955
6956err:
6957	sched_free_group(tg);
6958	return ERR_PTR(-ENOMEM);
6959}
6960
6961void sched_online_group(struct task_group *tg, struct task_group *parent)
6962{
6963	unsigned long flags;
6964
6965	spin_lock_irqsave(&task_group_lock, flags);
6966	list_add_rcu(&tg->list, &task_groups);
6967
6968	/* Root should already exist: */
6969	WARN_ON(!parent);
6970
6971	tg->parent = parent;
6972	INIT_LIST_HEAD(&tg->children);
6973	list_add_rcu(&tg->siblings, &parent->children);
6974	spin_unlock_irqrestore(&task_group_lock, flags);
6975
6976	online_fair_sched_group(tg);
6977}
6978
6979/* rcu callback to free various structures associated with a task group */
6980static void sched_free_group_rcu(struct rcu_head *rhp)
6981{
6982	/* Now it should be safe to free those cfs_rqs: */
6983	sched_free_group(container_of(rhp, struct task_group, rcu));
6984}
6985
6986void sched_destroy_group(struct task_group *tg)
6987{
6988	/* Wait for possible concurrent references to cfs_rqs complete: */
6989	call_rcu(&tg->rcu, sched_free_group_rcu);
6990}
6991
6992void sched_offline_group(struct task_group *tg)
6993{
6994	unsigned long flags;
6995
6996	/* End participation in shares distribution: */
6997	unregister_fair_sched_group(tg);
6998
6999	spin_lock_irqsave(&task_group_lock, flags);
7000	list_del_rcu(&tg->list);
7001	list_del_rcu(&tg->siblings);
7002	spin_unlock_irqrestore(&task_group_lock, flags);
7003}
7004
7005static void sched_change_group(struct task_struct *tsk, int type)
7006{
7007	struct task_group *tg;
7008
7009	/*
7010	 * All callers are synchronized by task_rq_lock(); we do not use RCU
7011	 * which is pointless here. Thus, we pass "true" to task_css_check()
7012	 * to prevent lockdep warnings.
7013	 */
7014	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
7015			  struct task_group, css);
7016	tg = autogroup_task_group(tsk, tg);
7017	tsk->sched_task_group = tg;
7018
7019#ifdef CONFIG_FAIR_GROUP_SCHED
7020	if (tsk->sched_class->task_change_group)
7021		tsk->sched_class->task_change_group(tsk, type);
7022	else
7023#endif
7024		set_task_rq(tsk, task_cpu(tsk));
7025}
7026
7027/*
7028 * Change task's runqueue when it moves between groups.
7029 *
7030 * The caller of this function should have put the task in its new group by
7031 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
7032 * its new group.
7033 */
7034void sched_move_task(struct task_struct *tsk)
7035{
7036	int queued, running, queue_flags =
7037		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
7038	struct rq_flags rf;
7039	struct rq *rq;
7040
7041	rq = task_rq_lock(tsk, &rf);
7042	update_rq_clock(rq);
7043
7044	running = task_current(rq, tsk);
7045	queued = task_on_rq_queued(tsk);
7046
7047	if (queued)
7048		dequeue_task(rq, tsk, queue_flags);
7049	if (running)
7050		put_prev_task(rq, tsk);
7051
7052	sched_change_group(tsk, TASK_MOVE_GROUP);
7053
7054	if (queued)
7055		enqueue_task(rq, tsk, queue_flags);
7056	if (running)
7057		set_next_task(rq, tsk);
7058
7059	task_rq_unlock(rq, tsk, &rf);
7060}
7061
7062static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
7063{
7064	return css ? container_of(css, struct task_group, css) : NULL;
7065}
7066
7067static struct cgroup_subsys_state *
7068cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
7069{
7070	struct task_group *parent = css_tg(parent_css);
7071	struct task_group *tg;
7072
7073	if (!parent) {
7074		/* This is early initialization for the top cgroup */
7075		return &root_task_group.css;
7076	}
7077
7078	tg = sched_create_group(parent);
7079	if (IS_ERR(tg))
7080		return ERR_PTR(-ENOMEM);
7081
7082	return &tg->css;
7083}
7084
7085/* Expose task group only after completing cgroup initialization */
7086static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
7087{
7088	struct task_group *tg = css_tg(css);
7089	struct task_group *parent = css_tg(css->parent);
7090
7091	if (parent)
7092		sched_online_group(tg, parent);
7093	return 0;
7094}
7095
7096static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
7097{
7098	struct task_group *tg = css_tg(css);
7099
7100	sched_offline_group(tg);
7101}
7102
7103static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
7104{
7105	struct task_group *tg = css_tg(css);
7106
7107	/*
7108	 * Relies on the RCU grace period between css_released() and this.
7109	 */
7110	sched_free_group(tg);
7111}
7112
7113/*
7114 * This is called before wake_up_new_task(), therefore we really only
7115 * have to set its group bits, all the other stuff does not apply.
7116 */
7117static void cpu_cgroup_fork(struct task_struct *task)
7118{
7119	struct rq_flags rf;
7120	struct rq *rq;
7121
7122	rq = task_rq_lock(task, &rf);
7123
7124	update_rq_clock(rq);
7125	sched_change_group(task, TASK_SET_GROUP);
7126
7127	task_rq_unlock(rq, task, &rf);
7128}
7129
7130static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
7131{
7132	struct task_struct *task;
7133	struct cgroup_subsys_state *css;
7134	int ret = 0;
7135
7136	cgroup_taskset_for_each(task, css, tset) {
7137#ifdef CONFIG_RT_GROUP_SCHED
7138		if (!sched_rt_can_attach(css_tg(css), task))
7139			return -EINVAL;
 
 
 
 
7140#endif
7141		/*
7142		 * Serialize against wake_up_new_task() such that if its
7143		 * running, we're sure to observe its full state.
7144		 */
7145		raw_spin_lock_irq(&task->pi_lock);
7146		/*
7147		 * Avoid calling sched_move_task() before wake_up_new_task()
7148		 * has happened. This would lead to problems with PELT, due to
7149		 * move wanting to detach+attach while we're not attached yet.
7150		 */
7151		if (task->state == TASK_NEW)
7152			ret = -EINVAL;
7153		raw_spin_unlock_irq(&task->pi_lock);
7154
7155		if (ret)
7156			break;
7157	}
7158	return ret;
7159}
7160
7161static void cpu_cgroup_attach(struct cgroup_taskset *tset)
7162{
7163	struct task_struct *task;
7164	struct cgroup_subsys_state *css;
7165
7166	cgroup_taskset_for_each(task, css, tset)
7167		sched_move_task(task);
7168}
7169
7170#ifdef CONFIG_UCLAMP_TASK_GROUP
7171static void cpu_util_update_eff(struct cgroup_subsys_state *css)
7172{
7173	struct cgroup_subsys_state *top_css = css;
7174	struct uclamp_se *uc_parent = NULL;
7175	struct uclamp_se *uc_se = NULL;
7176	unsigned int eff[UCLAMP_CNT];
7177	enum uclamp_id clamp_id;
7178	unsigned int clamps;
7179
7180	css_for_each_descendant_pre(css, top_css) {
7181		uc_parent = css_tg(css)->parent
7182			? css_tg(css)->parent->uclamp : NULL;
7183
7184		for_each_clamp_id(clamp_id) {
7185			/* Assume effective clamps matches requested clamps */
7186			eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value;
7187			/* Cap effective clamps with parent's effective clamps */
7188			if (uc_parent &&
7189			    eff[clamp_id] > uc_parent[clamp_id].value) {
7190				eff[clamp_id] = uc_parent[clamp_id].value;
7191			}
7192		}
7193		/* Ensure protection is always capped by limit */
7194		eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]);
7195
7196		/* Propagate most restrictive effective clamps */
7197		clamps = 0x0;
7198		uc_se = css_tg(css)->uclamp;
7199		for_each_clamp_id(clamp_id) {
7200			if (eff[clamp_id] == uc_se[clamp_id].value)
7201				continue;
7202			uc_se[clamp_id].value = eff[clamp_id];
7203			uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]);
7204			clamps |= (0x1 << clamp_id);
7205		}
7206		if (!clamps) {
7207			css = css_rightmost_descendant(css);
7208			continue;
7209		}
7210
7211		/* Immediately update descendants RUNNABLE tasks */
7212		uclamp_update_active_tasks(css, clamps);
7213	}
7214}
7215
7216/*
7217 * Integer 10^N with a given N exponent by casting to integer the literal "1eN"
7218 * C expression. Since there is no way to convert a macro argument (N) into a
7219 * character constant, use two levels of macros.
7220 */
7221#define _POW10(exp) ((unsigned int)1e##exp)
7222#define POW10(exp) _POW10(exp)
7223
7224struct uclamp_request {
7225#define UCLAMP_PERCENT_SHIFT	2
7226#define UCLAMP_PERCENT_SCALE	(100 * POW10(UCLAMP_PERCENT_SHIFT))
7227	s64 percent;
7228	u64 util;
7229	int ret;
7230};
7231
7232static inline struct uclamp_request
7233capacity_from_percent(char *buf)
7234{
7235	struct uclamp_request req = {
7236		.percent = UCLAMP_PERCENT_SCALE,
7237		.util = SCHED_CAPACITY_SCALE,
7238		.ret = 0,
7239	};
7240
7241	buf = strim(buf);
7242	if (strcmp(buf, "max")) {
7243		req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT,
7244					     &req.percent);
7245		if (req.ret)
7246			return req;
7247		if (req.percent > UCLAMP_PERCENT_SCALE) {
7248			req.ret = -ERANGE;
7249			return req;
7250		}
7251
7252		req.util = req.percent << SCHED_CAPACITY_SHIFT;
7253		req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE);
7254	}
7255
7256	return req;
7257}
7258
7259static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf,
7260				size_t nbytes, loff_t off,
7261				enum uclamp_id clamp_id)
7262{
7263	struct uclamp_request req;
7264	struct task_group *tg;
7265
7266	req = capacity_from_percent(buf);
7267	if (req.ret)
7268		return req.ret;
7269
7270	mutex_lock(&uclamp_mutex);
7271	rcu_read_lock();
7272
7273	tg = css_tg(of_css(of));
7274	if (tg->uclamp_req[clamp_id].value != req.util)
7275		uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false);
7276
7277	/*
7278	 * Because of not recoverable conversion rounding we keep track of the
7279	 * exact requested value
7280	 */
7281	tg->uclamp_pct[clamp_id] = req.percent;
7282
7283	/* Update effective clamps to track the most restrictive value */
7284	cpu_util_update_eff(of_css(of));
7285
7286	rcu_read_unlock();
7287	mutex_unlock(&uclamp_mutex);
7288
7289	return nbytes;
7290}
7291
7292static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of,
7293				    char *buf, size_t nbytes,
7294				    loff_t off)
7295{
7296	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN);
7297}
7298
7299static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of,
7300				    char *buf, size_t nbytes,
7301				    loff_t off)
7302{
7303	return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX);
7304}
7305
7306static inline void cpu_uclamp_print(struct seq_file *sf,
7307				    enum uclamp_id clamp_id)
7308{
7309	struct task_group *tg;
7310	u64 util_clamp;
7311	u64 percent;
7312	u32 rem;
7313
7314	rcu_read_lock();
7315	tg = css_tg(seq_css(sf));
7316	util_clamp = tg->uclamp_req[clamp_id].value;
7317	rcu_read_unlock();
7318
7319	if (util_clamp == SCHED_CAPACITY_SCALE) {
7320		seq_puts(sf, "max\n");
7321		return;
7322	}
7323
7324	percent = tg->uclamp_pct[clamp_id];
7325	percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem);
7326	seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem);
7327}
7328
7329static int cpu_uclamp_min_show(struct seq_file *sf, void *v)
7330{
7331	cpu_uclamp_print(sf, UCLAMP_MIN);
7332	return 0;
7333}
7334
7335static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
7336{
7337	cpu_uclamp_print(sf, UCLAMP_MAX);
7338	return 0;
7339}
7340#endif /* CONFIG_UCLAMP_TASK_GROUP */
7341
7342#ifdef CONFIG_FAIR_GROUP_SCHED
7343static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
7344				struct cftype *cftype, u64 shareval)
7345{
7346	if (shareval > scale_load_down(ULONG_MAX))
7347		shareval = MAX_SHARES;
7348	return sched_group_set_shares(css_tg(css), scale_load(shareval));
7349}
7350
7351static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
7352			       struct cftype *cft)
7353{
7354	struct task_group *tg = css_tg(css);
7355
7356	return (u64) scale_load_down(tg->shares);
7357}
7358
7359#ifdef CONFIG_CFS_BANDWIDTH
7360static DEFINE_MUTEX(cfs_constraints_mutex);
7361
7362const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
7363static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
7364
7365static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
7366
7367static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
7368{
7369	int i, ret = 0, runtime_enabled, runtime_was_enabled;
7370	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7371
7372	if (tg == &root_task_group)
7373		return -EINVAL;
7374
7375	/*
7376	 * Ensure we have at some amount of bandwidth every period.  This is
7377	 * to prevent reaching a state of large arrears when throttled via
7378	 * entity_tick() resulting in prolonged exit starvation.
7379	 */
7380	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
7381		return -EINVAL;
7382
7383	/*
7384	 * Likewise, bound things on the otherside by preventing insane quota
7385	 * periods.  This also allows us to normalize in computing quota
7386	 * feasibility.
7387	 */
7388	if (period > max_cfs_quota_period)
7389		return -EINVAL;
7390
7391	/*
7392	 * Prevent race between setting of cfs_rq->runtime_enabled and
7393	 * unthrottle_offline_cfs_rqs().
7394	 */
7395	get_online_cpus();
7396	mutex_lock(&cfs_constraints_mutex);
7397	ret = __cfs_schedulable(tg, period, quota);
7398	if (ret)
7399		goto out_unlock;
7400
7401	runtime_enabled = quota != RUNTIME_INF;
7402	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
7403	/*
7404	 * If we need to toggle cfs_bandwidth_used, off->on must occur
7405	 * before making related changes, and on->off must occur afterwards
7406	 */
7407	if (runtime_enabled && !runtime_was_enabled)
7408		cfs_bandwidth_usage_inc();
7409	raw_spin_lock_irq(&cfs_b->lock);
7410	cfs_b->period = ns_to_ktime(period);
7411	cfs_b->quota = quota;
7412
7413	__refill_cfs_bandwidth_runtime(cfs_b);
7414
7415	/* Restart the period timer (if active) to handle new period expiry: */
7416	if (runtime_enabled)
7417		start_cfs_bandwidth(cfs_b);
7418
7419	raw_spin_unlock_irq(&cfs_b->lock);
7420
7421	for_each_online_cpu(i) {
7422		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
7423		struct rq *rq = cfs_rq->rq;
7424		struct rq_flags rf;
7425
7426		rq_lock_irq(rq, &rf);
7427		cfs_rq->runtime_enabled = runtime_enabled;
7428		cfs_rq->runtime_remaining = 0;
7429
7430		if (cfs_rq->throttled)
7431			unthrottle_cfs_rq(cfs_rq);
7432		rq_unlock_irq(rq, &rf);
7433	}
7434	if (runtime_was_enabled && !runtime_enabled)
7435		cfs_bandwidth_usage_dec();
7436out_unlock:
7437	mutex_unlock(&cfs_constraints_mutex);
7438	put_online_cpus();
7439
7440	return ret;
7441}
7442
7443static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
7444{
7445	u64 quota, period;
7446
7447	period = ktime_to_ns(tg->cfs_bandwidth.period);
7448	if (cfs_quota_us < 0)
7449		quota = RUNTIME_INF;
7450	else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC)
7451		quota = (u64)cfs_quota_us * NSEC_PER_USEC;
7452	else
7453		return -EINVAL;
7454
7455	return tg_set_cfs_bandwidth(tg, period, quota);
7456}
7457
7458static long tg_get_cfs_quota(struct task_group *tg)
7459{
7460	u64 quota_us;
7461
7462	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
7463		return -1;
7464
7465	quota_us = tg->cfs_bandwidth.quota;
7466	do_div(quota_us, NSEC_PER_USEC);
7467
7468	return quota_us;
7469}
7470
7471static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
7472{
7473	u64 quota, period;
7474
7475	if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC)
7476		return -EINVAL;
7477
7478	period = (u64)cfs_period_us * NSEC_PER_USEC;
7479	quota = tg->cfs_bandwidth.quota;
7480
7481	return tg_set_cfs_bandwidth(tg, period, quota);
7482}
7483
7484static long tg_get_cfs_period(struct task_group *tg)
7485{
7486	u64 cfs_period_us;
7487
7488	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
7489	do_div(cfs_period_us, NSEC_PER_USEC);
7490
7491	return cfs_period_us;
7492}
7493
7494static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
7495				  struct cftype *cft)
7496{
7497	return tg_get_cfs_quota(css_tg(css));
7498}
7499
7500static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
7501				   struct cftype *cftype, s64 cfs_quota_us)
7502{
7503	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
7504}
7505
7506static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
7507				   struct cftype *cft)
7508{
7509	return tg_get_cfs_period(css_tg(css));
7510}
7511
7512static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
7513				    struct cftype *cftype, u64 cfs_period_us)
7514{
7515	return tg_set_cfs_period(css_tg(css), cfs_period_us);
7516}
7517
7518struct cfs_schedulable_data {
7519	struct task_group *tg;
7520	u64 period, quota;
7521};
7522
7523/*
7524 * normalize group quota/period to be quota/max_period
7525 * note: units are usecs
7526 */
7527static u64 normalize_cfs_quota(struct task_group *tg,
7528			       struct cfs_schedulable_data *d)
7529{
7530	u64 quota, period;
7531
7532	if (tg == d->tg) {
7533		period = d->period;
7534		quota = d->quota;
7535	} else {
7536		period = tg_get_cfs_period(tg);
7537		quota = tg_get_cfs_quota(tg);
7538	}
7539
7540	/* note: these should typically be equivalent */
7541	if (quota == RUNTIME_INF || quota == -1)
7542		return RUNTIME_INF;
7543
7544	return to_ratio(period, quota);
7545}
7546
7547static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
7548{
7549	struct cfs_schedulable_data *d = data;
7550	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7551	s64 quota = 0, parent_quota = -1;
7552
7553	if (!tg->parent) {
7554		quota = RUNTIME_INF;
7555	} else {
7556		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
7557
7558		quota = normalize_cfs_quota(tg, d);
7559		parent_quota = parent_b->hierarchical_quota;
7560
7561		/*
7562		 * Ensure max(child_quota) <= parent_quota.  On cgroup2,
7563		 * always take the min.  On cgroup1, only inherit when no
7564		 * limit is set:
7565		 */
7566		if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) {
7567			quota = min(quota, parent_quota);
7568		} else {
7569			if (quota == RUNTIME_INF)
7570				quota = parent_quota;
7571			else if (parent_quota != RUNTIME_INF && quota > parent_quota)
7572				return -EINVAL;
7573		}
7574	}
7575	cfs_b->hierarchical_quota = quota;
7576
7577	return 0;
7578}
7579
7580static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
7581{
7582	int ret;
7583	struct cfs_schedulable_data data = {
7584		.tg = tg,
7585		.period = period,
7586		.quota = quota,
7587	};
7588
7589	if (quota != RUNTIME_INF) {
7590		do_div(data.period, NSEC_PER_USEC);
7591		do_div(data.quota, NSEC_PER_USEC);
7592	}
7593
7594	rcu_read_lock();
7595	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
7596	rcu_read_unlock();
7597
7598	return ret;
7599}
7600
7601static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
7602{
7603	struct task_group *tg = css_tg(seq_css(sf));
7604	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7605
7606	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
7607	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
7608	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
7609
7610	if (schedstat_enabled() && tg != &root_task_group) {
7611		u64 ws = 0;
7612		int i;
7613
7614		for_each_possible_cpu(i)
7615			ws += schedstat_val(tg->se[i]->statistics.wait_sum);
7616
7617		seq_printf(sf, "wait_sum %llu\n", ws);
7618	}
7619
7620	return 0;
7621}
7622#endif /* CONFIG_CFS_BANDWIDTH */
7623#endif /* CONFIG_FAIR_GROUP_SCHED */
7624
7625#ifdef CONFIG_RT_GROUP_SCHED
7626static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
7627				struct cftype *cft, s64 val)
7628{
7629	return sched_group_set_rt_runtime(css_tg(css), val);
7630}
7631
7632static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
7633			       struct cftype *cft)
7634{
7635	return sched_group_rt_runtime(css_tg(css));
7636}
7637
7638static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
7639				    struct cftype *cftype, u64 rt_period_us)
7640{
7641	return sched_group_set_rt_period(css_tg(css), rt_period_us);
7642}
7643
7644static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
7645				   struct cftype *cft)
7646{
7647	return sched_group_rt_period(css_tg(css));
7648}
7649#endif /* CONFIG_RT_GROUP_SCHED */
7650
7651static struct cftype cpu_legacy_files[] = {
7652#ifdef CONFIG_FAIR_GROUP_SCHED
7653	{
7654		.name = "shares",
7655		.read_u64 = cpu_shares_read_u64,
7656		.write_u64 = cpu_shares_write_u64,
7657	},
7658#endif
7659#ifdef CONFIG_CFS_BANDWIDTH
7660	{
7661		.name = "cfs_quota_us",
7662		.read_s64 = cpu_cfs_quota_read_s64,
7663		.write_s64 = cpu_cfs_quota_write_s64,
7664	},
7665	{
7666		.name = "cfs_period_us",
7667		.read_u64 = cpu_cfs_period_read_u64,
7668		.write_u64 = cpu_cfs_period_write_u64,
7669	},
7670	{
7671		.name = "stat",
7672		.seq_show = cpu_cfs_stat_show,
7673	},
7674#endif
7675#ifdef CONFIG_RT_GROUP_SCHED
7676	{
7677		.name = "rt_runtime_us",
7678		.read_s64 = cpu_rt_runtime_read,
7679		.write_s64 = cpu_rt_runtime_write,
7680	},
7681	{
7682		.name = "rt_period_us",
7683		.read_u64 = cpu_rt_period_read_uint,
7684		.write_u64 = cpu_rt_period_write_uint,
7685	},
7686#endif
7687#ifdef CONFIG_UCLAMP_TASK_GROUP
7688	{
7689		.name = "uclamp.min",
7690		.flags = CFTYPE_NOT_ON_ROOT,
7691		.seq_show = cpu_uclamp_min_show,
7692		.write = cpu_uclamp_min_write,
7693	},
7694	{
7695		.name = "uclamp.max",
7696		.flags = CFTYPE_NOT_ON_ROOT,
7697		.seq_show = cpu_uclamp_max_show,
7698		.write = cpu_uclamp_max_write,
7699	},
7700#endif
7701	{ }	/* Terminate */
7702};
7703
7704static int cpu_extra_stat_show(struct seq_file *sf,
7705			       struct cgroup_subsys_state *css)
7706{
7707#ifdef CONFIG_CFS_BANDWIDTH
7708	{
7709		struct task_group *tg = css_tg(css);
7710		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7711		u64 throttled_usec;
7712
7713		throttled_usec = cfs_b->throttled_time;
7714		do_div(throttled_usec, NSEC_PER_USEC);
7715
7716		seq_printf(sf, "nr_periods %d\n"
7717			   "nr_throttled %d\n"
7718			   "throttled_usec %llu\n",
7719			   cfs_b->nr_periods, cfs_b->nr_throttled,
7720			   throttled_usec);
7721	}
7722#endif
7723	return 0;
7724}
7725
7726#ifdef CONFIG_FAIR_GROUP_SCHED
7727static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
7728			       struct cftype *cft)
7729{
7730	struct task_group *tg = css_tg(css);
7731	u64 weight = scale_load_down(tg->shares);
7732
7733	return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024);
7734}
7735
7736static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
7737				struct cftype *cft, u64 weight)
7738{
7739	/*
7740	 * cgroup weight knobs should use the common MIN, DFL and MAX
7741	 * values which are 1, 100 and 10000 respectively.  While it loses
7742	 * a bit of range on both ends, it maps pretty well onto the shares
7743	 * value used by scheduler and the round-trip conversions preserve
7744	 * the original value over the entire range.
7745	 */
7746	if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX)
7747		return -ERANGE;
7748
7749	weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL);
7750
7751	return sched_group_set_shares(css_tg(css), scale_load(weight));
7752}
7753
7754static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
7755				    struct cftype *cft)
7756{
7757	unsigned long weight = scale_load_down(css_tg(css)->shares);
7758	int last_delta = INT_MAX;
7759	int prio, delta;
7760
7761	/* find the closest nice value to the current weight */
7762	for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) {
7763		delta = abs(sched_prio_to_weight[prio] - weight);
7764		if (delta >= last_delta)
7765			break;
7766		last_delta = delta;
7767	}
7768
7769	return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO);
7770}
7771
7772static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
7773				     struct cftype *cft, s64 nice)
7774{
7775	unsigned long weight;
7776	int idx;
7777
7778	if (nice < MIN_NICE || nice > MAX_NICE)
7779		return -ERANGE;
7780
7781	idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO;
7782	idx = array_index_nospec(idx, 40);
7783	weight = sched_prio_to_weight[idx];
7784
7785	return sched_group_set_shares(css_tg(css), scale_load(weight));
7786}
7787#endif
7788
7789static void __maybe_unused cpu_period_quota_print(struct seq_file *sf,
7790						  long period, long quota)
7791{
7792	if (quota < 0)
7793		seq_puts(sf, "max");
7794	else
7795		seq_printf(sf, "%ld", quota);
7796
7797	seq_printf(sf, " %ld\n", period);
7798}
7799
7800/* caller should put the current value in *@periodp before calling */
7801static int __maybe_unused cpu_period_quota_parse(char *buf,
7802						 u64 *periodp, u64 *quotap)
7803{
7804	char tok[21];	/* U64_MAX */
7805
7806	if (sscanf(buf, "%20s %llu", tok, periodp) < 1)
7807		return -EINVAL;
7808
7809	*periodp *= NSEC_PER_USEC;
7810
7811	if (sscanf(tok, "%llu", quotap))
7812		*quotap *= NSEC_PER_USEC;
7813	else if (!strcmp(tok, "max"))
7814		*quotap = RUNTIME_INF;
7815	else
7816		return -EINVAL;
7817
7818	return 0;
7819}
7820
7821#ifdef CONFIG_CFS_BANDWIDTH
7822static int cpu_max_show(struct seq_file *sf, void *v)
7823{
7824	struct task_group *tg = css_tg(seq_css(sf));
7825
7826	cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg));
7827	return 0;
7828}
7829
7830static ssize_t cpu_max_write(struct kernfs_open_file *of,
7831			     char *buf, size_t nbytes, loff_t off)
7832{
7833	struct task_group *tg = css_tg(of_css(of));
7834	u64 period = tg_get_cfs_period(tg);
7835	u64 quota;
7836	int ret;
7837
7838	ret = cpu_period_quota_parse(buf, &period, &quota);
7839	if (!ret)
7840		ret = tg_set_cfs_bandwidth(tg, period, quota);
7841	return ret ?: nbytes;
7842}
7843#endif
7844
7845static struct cftype cpu_files[] = {
7846#ifdef CONFIG_FAIR_GROUP_SCHED
7847	{
7848		.name = "weight",
7849		.flags = CFTYPE_NOT_ON_ROOT,
7850		.read_u64 = cpu_weight_read_u64,
7851		.write_u64 = cpu_weight_write_u64,
7852	},
7853	{
7854		.name = "weight.nice",
7855		.flags = CFTYPE_NOT_ON_ROOT,
7856		.read_s64 = cpu_weight_nice_read_s64,
7857		.write_s64 = cpu_weight_nice_write_s64,
7858	},
7859#endif
7860#ifdef CONFIG_CFS_BANDWIDTH
7861	{
7862		.name = "max",
7863		.flags = CFTYPE_NOT_ON_ROOT,
7864		.seq_show = cpu_max_show,
7865		.write = cpu_max_write,
7866	},
7867#endif
7868#ifdef CONFIG_UCLAMP_TASK_GROUP
7869	{
7870		.name = "uclamp.min",
7871		.flags = CFTYPE_NOT_ON_ROOT,
7872		.seq_show = cpu_uclamp_min_show,
7873		.write = cpu_uclamp_min_write,
7874	},
7875	{
7876		.name = "uclamp.max",
7877		.flags = CFTYPE_NOT_ON_ROOT,
7878		.seq_show = cpu_uclamp_max_show,
7879		.write = cpu_uclamp_max_write,
7880	},
7881#endif
7882	{ }	/* terminate */
7883};
7884
7885struct cgroup_subsys cpu_cgrp_subsys = {
7886	.css_alloc	= cpu_cgroup_css_alloc,
7887	.css_online	= cpu_cgroup_css_online,
7888	.css_released	= cpu_cgroup_css_released,
7889	.css_free	= cpu_cgroup_css_free,
7890	.css_extra_stat_show = cpu_extra_stat_show,
7891	.fork		= cpu_cgroup_fork,
7892	.can_attach	= cpu_cgroup_can_attach,
7893	.attach		= cpu_cgroup_attach,
7894	.legacy_cftypes	= cpu_legacy_files,
7895	.dfl_cftypes	= cpu_files,
7896	.early_init	= true,
7897	.threaded	= true,
7898};
7899
7900#endif	/* CONFIG_CGROUP_SCHED */
7901
7902void dump_cpu_task(int cpu)
7903{
7904	pr_info("Task dump for CPU %d:\n", cpu);
7905	sched_show_task(cpu_curr(cpu));
7906}
7907
7908/*
7909 * Nice levels are multiplicative, with a gentle 10% change for every
7910 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
7911 * nice 1, it will get ~10% less CPU time than another CPU-bound task
7912 * that remained on nice 0.
7913 *
7914 * The "10% effect" is relative and cumulative: from _any_ nice level,
7915 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
7916 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
7917 * If a task goes up by ~10% and another task goes down by ~10% then
7918 * the relative distance between them is ~25%.)
7919 */
7920const int sched_prio_to_weight[40] = {
7921 /* -20 */     88761,     71755,     56483,     46273,     36291,
7922 /* -15 */     29154,     23254,     18705,     14949,     11916,
7923 /* -10 */      9548,      7620,      6100,      4904,      3906,
7924 /*  -5 */      3121,      2501,      1991,      1586,      1277,
7925 /*   0 */      1024,       820,       655,       526,       423,
7926 /*   5 */       335,       272,       215,       172,       137,
7927 /*  10 */       110,        87,        70,        56,        45,
7928 /*  15 */        36,        29,        23,        18,        15,
7929};
7930
7931/*
7932 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated.
7933 *
7934 * In cases where the weight does not change often, we can use the
7935 * precalculated inverse to speed up arithmetics by turning divisions
7936 * into multiplications:
7937 */
7938const u32 sched_prio_to_wmult[40] = {
7939 /* -20 */     48388,     59856,     76040,     92818,    118348,
7940 /* -15 */    147320,    184698,    229616,    287308,    360437,
7941 /* -10 */    449829,    563644,    704093,    875809,   1099582,
7942 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
7943 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
7944 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
7945 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
7946 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
7947};
7948
7949#undef CREATE_TRACE_POINTS