1/*
   2 *  kernel/sched/core.c
   3 *
   4 *  Kernel scheduler and related syscalls
   5 *
   6 *  Copyright (C) 1991-2002  Linus Torvalds
   7 *
   8 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
   9 *		make semaphores SMP safe
  10 *  1998-11-19	Implemented schedule_timeout() and related stuff
  11 *		by Andrea Arcangeli
  12 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
  13 *		hybrid priority-list and round-robin design with
  14 *		an array-switch method of distributing timeslices
  15 *		and per-CPU runqueues.  Cleanups and useful suggestions
  16 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
  17 *  2003-09-03	Interactivity tuning by Con Kolivas.
  18 *  2004-04-02	Scheduler domains code by Nick Piggin
  19 *  2007-04-15  Work begun on replacing all interactivity tuning with a
  20 *              fair scheduling design by Con Kolivas.
  21 *  2007-05-05  Load balancing (smp-nice) and other improvements
  22 *              by Peter Williams
  23 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
  24 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
  25 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26 *              Thomas Gleixner, Mike Kravetz
  27 */
  28
  29#include <linux/mm.h>
  30#include <linux/module.h>
  31#include <linux/nmi.h>
  32#include <linux/init.h>
  33#include <linux/uaccess.h>
  34#include <linux/highmem.h>
  35#include <asm/mmu_context.h>
  36#include <linux/interrupt.h>
  37#include <linux/capability.h>
  38#include <linux/completion.h>
  39#include <linux/kernel_stat.h>
  40#include <linux/debug_locks.h>
  41#include <linux/perf_event.h>
  42#include <linux/security.h>
  43#include <linux/notifier.h>
  44#include <linux/profile.h>
  45#include <linux/freezer.h>
  46#include <linux/vmalloc.h>
  47#include <linux/blkdev.h>
  48#include <linux/delay.h>
  49#include <linux/pid_namespace.h>
  50#include <linux/smp.h>
  51#include <linux/threads.h>
  52#include <linux/timer.h>
  53#include <linux/rcupdate.h>
  54#include <linux/cpu.h>
  55#include <linux/cpuset.h>
  56#include <linux/percpu.h>
  57#include <linux/proc_fs.h>
  58#include <linux/seq_file.h>
  59#include <linux/sysctl.h>
  60#include <linux/syscalls.h>
  61#include <linux/times.h>
  62#include <linux/tsacct_kern.h>
  63#include <linux/kprobes.h>
  64#include <linux/delayacct.h>
  65#include <linux/unistd.h>
  66#include <linux/pagemap.h>
  67#include <linux/hrtimer.h>
  68#include <linux/tick.h>
  69#include <linux/debugfs.h>
  70#include <linux/ctype.h>
  71#include <linux/ftrace.h>
  72#include <linux/slab.h>
  73#include <linux/init_task.h>
  74#include <linux/binfmts.h>
  75#include <linux/context_tracking.h>
  76#include <linux/compiler.h>
  77
  78#include <asm/switch_to.h>
  79#include <asm/tlb.h>
  80#include <asm/irq_regs.h>
  81#include <asm/mutex.h>
  82#ifdef CONFIG_PARAVIRT
  83#include <asm/paravirt.h>
  84#endif
  85
  86#include "sched.h"
  87#include "../workqueue_internal.h"
  88#include "../smpboot.h"
  89
  90#define CREATE_TRACE_POINTS
  91#include <trace/events/sched.h>
 
  92
  93void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
  94{
  95	unsigned long delta;
  96	ktime_t soft, hard, now;
  97
  98	for (;;) {
  99		if (hrtimer_active(period_timer))
 100			break;
 101
 102		now = hrtimer_cb_get_time(period_timer);
 103		hrtimer_forward(period_timer, now, period);
 104
 105		soft = hrtimer_get_softexpires(period_timer);
 106		hard = hrtimer_get_expires(period_timer);
 107		delta = ktime_to_ns(ktime_sub(hard, soft));
 108		__hrtimer_start_range_ns(period_timer, soft, delta,
 109					 HRTIMER_MODE_ABS_PINNED, 0);
 110	}
 111}
 112
 113DEFINE_MUTEX(sched_domains_mutex);
 114DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 115
 116static void update_rq_clock_task(struct rq *rq, s64 delta);
 
 
 117
 118void update_rq_clock(struct rq *rq)
 119{
 120	s64 delta;
 121
 122	if (rq->skip_clock_update > 0)
 123		return;
 
 
 
 
 
 
 
 
 
 
 
 124
 125	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
 126	rq->clock += delta;
 127	update_rq_clock_task(rq, delta);
 128}
 129
 
 130/*
 131 * Debugging: various feature bits
 
 
 
 
 132 */
 133
 134#define SCHED_FEAT(name, enabled)	\
 135	(1UL << __SCHED_FEAT_##name) * enabled |
 136
 137const_debug unsigned int sysctl_sched_features =
 138#include "features.h"
 139	0;
 140
 141#undef SCHED_FEAT
 142
 143#ifdef CONFIG_SCHED_DEBUG
 144#define SCHED_FEAT(name, enabled)	\
 145	#name ,
 146
 147static const char * const sched_feat_names[] = {
 148#include "features.h"
 149};
 150
 151#undef SCHED_FEAT
 152
 153static int sched_feat_show(struct seq_file *m, void *v)
 154{
 155	int i;
 156
 157	for (i = 0; i < __SCHED_FEAT_NR; i++) {
 158		if (!(sysctl_sched_features & (1UL << i)))
 159			seq_puts(m, "NO_");
 160		seq_printf(m, "%s ", sched_feat_names[i]);
 161	}
 162	seq_puts(m, "\n");
 163
 164	return 0;
 165}
 166
 167#ifdef HAVE_JUMP_LABEL
 168
 169#define jump_label_key__true  STATIC_KEY_INIT_TRUE
 170#define jump_label_key__false STATIC_KEY_INIT_FALSE
 171
 172#define SCHED_FEAT(name, enabled)	\
 173	jump_label_key__##enabled ,
 174
 175struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
 176#include "features.h"
 177};
 178
 179#undef SCHED_FEAT
 180
 181static void sched_feat_disable(int i)
 182{
 183	if (static_key_enabled(&sched_feat_keys[i]))
 184		static_key_slow_dec(&sched_feat_keys[i]);
 185}
 186
 187static void sched_feat_enable(int i)
 188{
 189	if (!static_key_enabled(&sched_feat_keys[i]))
 190		static_key_slow_inc(&sched_feat_keys[i]);
 191}
 192#else
 193static void sched_feat_disable(int i) { };
 194static void sched_feat_enable(int i) { };
 195#endif /* HAVE_JUMP_LABEL */
 196
 197static int sched_feat_set(char *cmp)
 198{
 199	int i;
 200	int neg = 0;
 201
 202	if (strncmp(cmp, "NO_", 3) == 0) {
 203		neg = 1;
 204		cmp += 3;
 205	}
 206
 207	for (i = 0; i < __SCHED_FEAT_NR; i++) {
 208		if (strcmp(cmp, sched_feat_names[i]) == 0) {
 209			if (neg) {
 210				sysctl_sched_features &= ~(1UL << i);
 211				sched_feat_disable(i);
 212			} else {
 213				sysctl_sched_features |= (1UL << i);
 214				sched_feat_enable(i);
 215			}
 216			break;
 217		}
 218	}
 219
 220	return i;
 221}
 222
 223static ssize_t
 224sched_feat_write(struct file *filp, const char __user *ubuf,
 225		size_t cnt, loff_t *ppos)
 226{
 227	char buf[64];
 228	char *cmp;
 229	int i;
 230
 231	if (cnt > 63)
 232		cnt = 63;
 233
 234	if (copy_from_user(&buf, ubuf, cnt))
 235		return -EFAULT;
 236
 237	buf[cnt] = 0;
 238	cmp = strstrip(buf);
 239
 240	i = sched_feat_set(cmp);
 241	if (i == __SCHED_FEAT_NR)
 242		return -EINVAL;
 243
 244	*ppos += cnt;
 245
 246	return cnt;
 247}
 248
 249static int sched_feat_open(struct inode *inode, struct file *filp)
 250{
 251	return single_open(filp, sched_feat_show, NULL);
 252}
 253
 254static const struct file_operations sched_feat_fops = {
 255	.open		= sched_feat_open,
 256	.write		= sched_feat_write,
 257	.read		= seq_read,
 258	.llseek		= seq_lseek,
 259	.release	= single_release,
 260};
 261
 262static __init int sched_init_debug(void)
 263{
 264	debugfs_create_file("sched_features", 0644, NULL, NULL,
 265			&sched_feat_fops);
 266
 267	return 0;
 268}
 269late_initcall(sched_init_debug);
 270#endif /* CONFIG_SCHED_DEBUG */
 271
 272/*
 273 * Number of tasks to iterate in a single balance run.
 274 * Limited because this is done with IRQs disabled.
 275 */
 276const_debug unsigned int sysctl_sched_nr_migrate = 32;
 277
 278/*
 279 * period over which we average the RT time consumption, measured
 280 * in ms.
 281 *
 282 * default: 1s
 283 */
 284const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
 285
 286/*
 287 * period over which we measure -rt task cpu usage in us.
 288 * default: 1s
 289 */
 290unsigned int sysctl_sched_rt_period = 1000000;
 291
 292__read_mostly int scheduler_running;
 293
 294/*
 295 * part of the period that we allow rt tasks to run in us.
 296 * default: 0.95s
 297 */
 298int sysctl_sched_rt_runtime = 950000;
 299
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 300/*
 301 * __task_rq_lock - lock the rq @p resides on.
 302 */
 303static inline struct rq *__task_rq_lock(struct task_struct *p)
 304	__acquires(rq->lock)
 305{
 306	struct rq *rq;
 307
 308	lockdep_assert_held(&p->pi_lock);
 309
 310	for (;;) {
 311		rq = task_rq(p);
 312		raw_spin_lock(&rq->lock);
 313		if (likely(rq == task_rq(p)))
 
 314			return rq;
 
 315		raw_spin_unlock(&rq->lock);
 
 
 
 316	}
 317}
 318
 319/*
 320 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
 321 */
 322static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
 323	__acquires(p->pi_lock)
 324	__acquires(rq->lock)
 325{
 326	struct rq *rq;
 327
 328	for (;;) {
 329		raw_spin_lock_irqsave(&p->pi_lock, *flags);
 330		rq = task_rq(p);
 331		raw_spin_lock(&rq->lock);
 332		if (likely(rq == task_rq(p)))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 333			return rq;
 
 334		raw_spin_unlock(&rq->lock);
 335		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
 
 
 
 336	}
 337}
 338
 339static void __task_rq_unlock(struct rq *rq)
 340	__releases(rq->lock)
 341{
 342	raw_spin_unlock(&rq->lock);
 343}
 344
 345static inline void
 346task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
 347	__releases(rq->lock)
 348	__releases(p->pi_lock)
 349{
 350	raw_spin_unlock(&rq->lock);
 351	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
 352}
 353
 354/*
 355 * this_rq_lock - lock this runqueue and disable interrupts.
 
 356 */
 357static struct rq *this_rq_lock(void)
 358	__acquires(rq->lock)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 359{
 360	struct rq *rq;
 361
 362	local_irq_disable();
 363	rq = this_rq();
 364	raw_spin_lock(&rq->lock);
 365
 366	return rq;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 367}
 368
 369#ifdef CONFIG_SCHED_HRTICK
 370/*
 371 * Use HR-timers to deliver accurate preemption points.
 372 */
 373
 374static void hrtick_clear(struct rq *rq)
 375{
 376	if (hrtimer_active(&rq->hrtick_timer))
 377		hrtimer_cancel(&rq->hrtick_timer);
 378}
 379
 380/*
 381 * High-resolution timer tick.
 382 * Runs from hardirq context with interrupts disabled.
 383 */
 384static enum hrtimer_restart hrtick(struct hrtimer *timer)
 385{
 386	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
 
 387
 388	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
 389
 390	raw_spin_lock(&rq->lock);
 391	update_rq_clock(rq);
 392	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
 393	raw_spin_unlock(&rq->lock);
 394
 395	return HRTIMER_NORESTART;
 396}
 397
 398#ifdef CONFIG_SMP
 399
 400static int __hrtick_restart(struct rq *rq)
 401{
 402	struct hrtimer *timer = &rq->hrtick_timer;
 403	ktime_t time = hrtimer_get_softexpires(timer);
 404
 405	return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
 406}
 407
 408/*
 409 * called from hardirq (IPI) context
 410 */
 411static void __hrtick_start(void *arg)
 412{
 413	struct rq *rq = arg;
 
 414
 415	raw_spin_lock(&rq->lock);
 416	__hrtick_restart(rq);
 417	rq->hrtick_csd_pending = 0;
 418	raw_spin_unlock(&rq->lock);
 419}
 420
 421/*
 422 * Called to set the hrtick timer state.
 423 *
 424 * called with rq->lock held and irqs disabled
 425 */
 426void hrtick_start(struct rq *rq, u64 delay)
 427{
 428	struct hrtimer *timer = &rq->hrtick_timer;
 429	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
 
 
 
 
 
 
 
 
 430
 431	hrtimer_set_expires(timer, time);
 432
 433	if (rq == this_rq()) {
 434		__hrtick_restart(rq);
 435	} else if (!rq->hrtick_csd_pending) {
 436		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
 437		rq->hrtick_csd_pending = 1;
 438	}
 439}
 440
 441static int
 442hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 
 
 
 
 443{
 444	int cpu = (int)(long)hcpu;
 
 
 
 
 
 
 
 445
 446	switch (action) {
 447	case CPU_UP_CANCELED:
 448	case CPU_UP_CANCELED_FROZEN:
 449	case CPU_DOWN_PREPARE:
 450	case CPU_DOWN_PREPARE_FROZEN:
 451	case CPU_DEAD:
 452	case CPU_DEAD_FROZEN:
 453		hrtick_clear(cpu_rq(cpu));
 454		return NOTIFY_OK;
 455	}
 456
 457	return NOTIFY_DONE;
 
 
 
 
 
 
 
 
 
 
 458}
 459
 460static __init void init_hrtick(void)
 461{
 462	hotcpu_notifier(hotplug_hrtick, 0);
 463}
 464#else
 
 465/*
 466 * Called to set the hrtick timer state.
 467 *
 468 * called with rq->lock held and irqs disabled
 469 */
 470void hrtick_start(struct rq *rq, u64 delay)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 471{
 472	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
 473			HRTIMER_MODE_REL_PINNED, 0);
 474}
 475
 476static inline void init_hrtick(void)
 
 
 
 
 
 
 477{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 478}
 479#endif /* CONFIG_SMP */
 480
 481static void init_rq_hrtick(struct rq *rq)
 
 482{
 483#ifdef CONFIG_SMP
 484	rq->hrtick_csd_pending = 0;
 
 485
 486	rq->hrtick_csd.flags = 0;
 487	rq->hrtick_csd.func = __hrtick_start;
 488	rq->hrtick_csd.info = rq;
 
 
 
 489#endif
 490
 491	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 492	rq->hrtick_timer.function = hrtick;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 493}
 494#else	/* CONFIG_SCHED_HRTICK */
 495static inline void hrtick_clear(struct rq *rq)
 
 
 
 
 
 
 
 
 
 
 
 
 496{
 
 
 497}
 498
 499static inline void init_rq_hrtick(struct rq *rq)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 500{
 
 
 501}
 502
 503static inline void init_hrtick(void)
 504{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 505}
 506#endif	/* CONFIG_SCHED_HRTICK */
 507
 508/*
 509 * resched_task - mark a task 'to be rescheduled now'.
 510 *
 511 * On UP this means the setting of the need_resched flag, on SMP it
 512 * might also involve a cross-CPU call to trigger the scheduler on
 513 * the target CPU.
 514 */
 515void resched_task(struct task_struct *p)
 516{
 
 517	int cpu;
 518
 519	lockdep_assert_held(&task_rq(p)->lock);
 520
 521	if (test_tsk_need_resched(p))
 522		return;
 523
 524	set_tsk_need_resched(p);
 525
 526	cpu = task_cpu(p);
 527	if (cpu == smp_processor_id()) {
 
 528		set_preempt_need_resched();
 529		return;
 530	}
 531
 532	/* NEED_RESCHED must be visible before we test polling */
 533	smp_mb();
 534	if (!tsk_is_polling(p))
 535		smp_send_reschedule(cpu);
 
 
 536}
 537
 538void resched_cpu(int cpu)
 539{
 540	struct rq *rq = cpu_rq(cpu);
 541	unsigned long flags;
 542
 543	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
 544		return;
 545	resched_task(cpu_curr(cpu));
 546	raw_spin_unlock_irqrestore(&rq->lock, flags);
 547}
 548
 549#ifdef CONFIG_SMP
 550#ifdef CONFIG_NO_HZ_COMMON
 551/*
 552 * In the semi idle case, use the nearest busy cpu for migrating timers
 553 * from an idle cpu.  This is good for power-savings.
 554 *
 555 * We don't do similar optimization for completely idle system, as
 556 * selecting an idle cpu will add more delays to the timers than intended
 557 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
 558 */
 559int get_nohz_timer_target(int pinned)
 560{
 561	int cpu = smp_processor_id();
 562	int i;
 563	struct sched_domain *sd;
 564
 565	if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu))
 566		return cpu;
 
 
 
 567
 568	rcu_read_lock();
 569	for_each_domain(cpu, sd) {
 570		for_each_cpu(i, sched_domain_span(sd)) {
 
 
 
 
 571			if (!idle_cpu(i)) {
 572				cpu = i;
 573				goto unlock;
 574			}
 575		}
 576	}
 
 
 
 
 577unlock:
 578	rcu_read_unlock();
 579	return cpu;
 580}
 
 581/*
 582 * When add_timer_on() enqueues a timer into the timer wheel of an
 583 * idle CPU then this timer might expire before the next timer event
 584 * which is scheduled to wake up that CPU. In case of a completely
 585 * idle system the next event might even be infinite time into the
 586 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 587 * leaves the inner idle loop so the newly added timer is taken into
 588 * account when the CPU goes back to idle and evaluates the timer
 589 * wheel for the next timer event.
 590 */
 591static void wake_up_idle_cpu(int cpu)
 592{
 593	struct rq *rq = cpu_rq(cpu);
 594
 595	if (cpu == smp_processor_id())
 596		return;
 597
 598	/*
 599	 * This is safe, as this function is called with the timer
 600	 * wheel base lock of (cpu) held. When the CPU is on the way
 601	 * to idle and has not yet set rq->curr to idle then it will
 602	 * be serialized on the timer wheel base lock and take the new
 603	 * timer into account automatically.
 604	 */
 605	if (rq->curr != rq->idle)
 606		return;
 607
 608	/*
 609	 * We can set TIF_RESCHED on the idle task of the other CPU
 610	 * lockless. The worst case is that the other CPU runs the
 611	 * idle task through an additional NOOP schedule()
 612	 */
 613	set_tsk_need_resched(rq->idle);
 614
 615	/* NEED_RESCHED must be visible before we test polling */
 616	smp_mb();
 617	if (!tsk_is_polling(rq->idle))
 618		smp_send_reschedule(cpu);
 
 
 619}
 620
 621static bool wake_up_full_nohz_cpu(int cpu)
 622{
 
 
 
 
 
 
 
 
 623	if (tick_nohz_full_cpu(cpu)) {
 624		if (cpu != smp_processor_id() ||
 625		    tick_nohz_tick_stopped())
 626			smp_send_reschedule(cpu);
 627		return true;
 628	}
 629
 630	return false;
 631}
 632
 
 
 
 
 
 633void wake_up_nohz_cpu(int cpu)
 634{
 635	if (!wake_up_full_nohz_cpu(cpu))
 636		wake_up_idle_cpu(cpu);
 637}
 638
 639static inline bool got_nohz_idle_kick(void)
 640{
 641	int cpu = smp_processor_id();
 642
 643	if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
 644		return false;
 645
 646	if (idle_cpu(cpu) && !need_resched())
 647		return true;
 648
 649	/*
 650	 * We can't run Idle Load Balance on this CPU for this time so we
 651	 * cancel it and clear NOHZ_BALANCE_KICK
 652	 */
 653	clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
 654	return false;
 655}
 656
 657#else /* CONFIG_NO_HZ_COMMON */
 658
 659static inline bool got_nohz_idle_kick(void)
 660{
 661	return false;
 662}
 663
 664#endif /* CONFIG_NO_HZ_COMMON */
 665
 666#ifdef CONFIG_NO_HZ_FULL
 667bool sched_can_stop_tick(void)
 668{
 669       struct rq *rq;
 670
 671       rq = this_rq();
 672
 673       /* Make sure rq->nr_running update is visible after the IPI */
 674       smp_rmb();
 
 675
 676       /* More than one running task need preemption */
 677       if (rq->nr_running > 1)
 678               return false;
 
 
 
 
 
 
 
 679
 680       return true;
 681}
 682#endif /* CONFIG_NO_HZ_FULL */
 
 
 
 
 683
 684void sched_avg_update(struct rq *rq)
 685{
 686	s64 period = sched_avg_period();
 
 
 
 
 687
 688	while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
 689		/*
 690		 * Inline assembly required to prevent the compiler
 691		 * optimising this loop into a divmod call.
 692		 * See __iter_div_u64_rem() for another example of this.
 693		 */
 694		asm("" : "+rm" (rq->age_stamp));
 695		rq->age_stamp += period;
 696		rq->rt_avg /= 2;
 697	}
 698}
 699
 700#endif /* CONFIG_SMP */
 701
 702#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
 703			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
 704/*
 705 * Iterate task_group tree rooted at *from, calling @down when first entering a
 706 * node and @up when leaving it for the final time.
 707 *
 708 * Caller must hold rcu_lock or sufficient equivalent.
 709 */
 710int walk_tg_tree_from(struct task_group *from,
 711			     tg_visitor down, tg_visitor up, void *data)
 712{
 713	struct task_group *parent, *child;
 714	int ret;
 715
 716	parent = from;
 717
 718down:
 719	ret = (*down)(parent, data);
 720	if (ret)
 721		goto out;
 722	list_for_each_entry_rcu(child, &parent->children, siblings) {
 723		parent = child;
 724		goto down;
 725
 726up:
 727		continue;
 728	}
 729	ret = (*up)(parent, data);
 730	if (ret || parent == from)
 731		goto out;
 732
 733	child = parent;
 734	parent = parent->parent;
 735	if (parent)
 736		goto up;
 737out:
 738	return ret;
 739}
 740
 741int tg_nop(struct task_group *tg, void *data)
 742{
 743	return 0;
 744}
 745#endif
 746
 747static void set_load_weight(struct task_struct *p)
 748{
 749	int prio = p->static_prio - MAX_RT_PRIO;
 750	struct load_weight *load = &p->se.load;
 751
 752	/*
 753	 * SCHED_IDLE tasks get minimal weight:
 754	 */
 755	if (p->policy == SCHED_IDLE) {
 756		load->weight = scale_load(WEIGHT_IDLEPRIO);
 757		load->inv_weight = WMULT_IDLEPRIO;
 758		return;
 759	}
 760
 761	load->weight = scale_load(prio_to_weight[prio]);
 762	load->inv_weight = prio_to_wmult[prio];
 
 
 
 
 
 
 
 
 763}
 764
 765static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 766{
 767	update_rq_clock(rq);
 768	sched_info_queued(rq, p);
 769	p->sched_class->enqueue_task(rq, p, flags);
 770}
 771
 772static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 773{
 774	update_rq_clock(rq);
 775	sched_info_dequeued(rq, p);
 776	p->sched_class->dequeue_task(rq, p, flags);
 777}
 778
 779void activate_task(struct rq *rq, struct task_struct *p, int flags)
 780{
 781	if (task_contributes_to_load(p))
 782		rq->nr_uninterruptible--;
 
 
 783
 784	enqueue_task(rq, p, flags);
 
 
 
 
 
 785}
 786
 787void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 
 
 788{
 789	if (task_contributes_to_load(p))
 790		rq->nr_uninterruptible++;
 
 
 
 
 
 
 
 791
 792	dequeue_task(rq, p, flags);
 793}
 794
 795static void update_rq_clock_task(struct rq *rq, s64 delta)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 796{
 
 
 
 
 
 
 
 
 
 
 
 797/*
 798 * In theory, the compile should just see 0 here, and optimize out the call
 799 * to sched_rt_avg_update. But I don't trust it...
 
 
 
 
 
 
 800 */
 801#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
 802	s64 steal = 0, irq_delta = 0;
 803#endif
 804#ifdef CONFIG_IRQ_TIME_ACCOUNTING
 805	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
 
 
 
 
 
 
 
 
 
 
 
 
 806
 807	/*
 808	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
 809	 * this case when a previous update_rq_clock() happened inside a
 810	 * {soft,}irq region.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 811	 *
 812	 * When this happens, we stop ->clock_task and only update the
 813	 * prev_irq_time stamp to account for the part that fit, so that a next
 814	 * update will consume the rest. This ensures ->clock_task is
 815	 * monotonic.
 816	 *
 817	 * It does however cause some slight miss-attribution of {soft,}irq
 818	 * time, a more accurate solution would be to update the irq_time using
 819	 * the current rq->clock timestamp, except that would require using
 820	 * atomic ops.
 
 
 
 
 
 
 
 
 
 
 821	 */
 822	if (irq_delta > delta)
 823		irq_delta = delta;
 824
 825	rq->prev_irq_time += irq_delta;
 826	delta -= irq_delta;
 827#endif
 828#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
 829	if (static_key_false((&paravirt_steal_rq_enabled))) {
 830		steal = paravirt_steal_clock(cpu_of(rq));
 831		steal -= rq->prev_steal_time_rq;
 832
 833		if (unlikely(steal > delta))
 834			steal = delta;
 
 835
 836		rq->prev_steal_time_rq += steal;
 837		delta -= steal;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 838	}
 839#endif
 840
 841	rq->clock_task += delta;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 842
 843#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
 844	if ((irq_delta + steal) && sched_feat(NONTASK_POWER))
 845		sched_rt_avg_update(rq, irq_delta + steal);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 846#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 847}
 848
 849void sched_set_stop_task(int cpu, struct task_struct *stop)
 
 850{
 851	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
 852	struct task_struct *old_stop = cpu_rq(cpu)->stop;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 853
 854	if (stop) {
 855		/*
 856		 * Make it appear like a SCHED_FIFO task, its something
 857		 * userspace knows about and won't get confused about.
 858		 *
 859		 * Also, it will make PI more or less work without too
 860		 * much confusion -- but then, stop work should not
 861		 * rely on PI working anyway.
 862		 */
 863		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
 
 
 
 864
 865		stop->sched_class = &stop_sched_class;
 866	}
 867
 868	cpu_rq(cpu)->stop = stop;
 
 869
 870	if (old_stop) {
 871		/*
 872		 * Reset it back to a normal scheduling class so that
 873		 * it can die in pieces.
 874		 */
 875		old_stop->sched_class = &rt_sched_class;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 876	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 877}
 878
 879/*
 880 * __normal_prio - return the priority that is based on the static prio
 881 */
 882static inline int __normal_prio(struct task_struct *p)
 883{
 884	return p->static_prio;
 885}
 886
 887/*
 888 * Calculate the expected normal priority: i.e. priority
 889 * without taking RT-inheritance into account. Might be
 890 * boosted by interactivity modifiers. Changes upon fork,
 891 * setprio syscalls, and whenever the interactivity
 892 * estimator recalculates.
 893 */
 894static inline int normal_prio(struct task_struct *p)
 895{
 896	int prio;
 897
 898	if (task_has_dl_policy(p))
 899		prio = MAX_DL_PRIO-1;
 900	else if (task_has_rt_policy(p))
 901		prio = MAX_RT_PRIO-1 - p->rt_priority;
 902	else
 903		prio = __normal_prio(p);
 904	return prio;
 905}
 906
 907/*
 908 * Calculate the current priority, i.e. the priority
 909 * taken into account by the scheduler. This value might
 910 * be boosted by RT tasks, or might be boosted by
 911 * interactivity modifiers. Will be RT if the task got
 912 * RT-boosted. If not then it returns p->normal_prio.
 913 */
 914static int effective_prio(struct task_struct *p)
 915{
 916	p->normal_prio = normal_prio(p);
 917	/*
 918	 * If we are RT tasks or we were boosted to RT priority,
 919	 * keep the priority unchanged. Otherwise, update priority
 920	 * to the normal priority:
 921	 */
 922	if (!rt_prio(p->prio))
 923		return p->normal_prio;
 924	return p->prio;
 925}
 926
 927/**
 928 * task_curr - is this task currently executing on a CPU?
 929 * @p: the task in question.
 930 *
 931 * Return: 1 if the task is currently executing. 0 otherwise.
 932 */
 933inline int task_curr(const struct task_struct *p)
 934{
 935	return cpu_curr(task_cpu(p)) == p;
 936}
 937
 
 
 
 
 
 
 
 938static inline void check_class_changed(struct rq *rq, struct task_struct *p,
 939				       const struct sched_class *prev_class,
 940				       int oldprio)
 941{
 942	if (prev_class != p->sched_class) {
 943		if (prev_class->switched_from)
 944			prev_class->switched_from(rq, p);
 
 945		p->sched_class->switched_to(rq, p);
 946	} else if (oldprio != p->prio || dl_task(p))
 947		p->sched_class->prio_changed(rq, p, oldprio);
 948}
 949
 950void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
 951{
 952	const struct sched_class *class;
 953
 954	if (p->sched_class == rq->curr->sched_class) {
 955		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
 956	} else {
 957		for_each_class(class) {
 958			if (class == rq->curr->sched_class)
 959				break;
 960			if (class == p->sched_class) {
 961				resched_task(rq->curr);
 962				break;
 963			}
 964		}
 965	}
 966
 967	/*
 968	 * A queue event has occurred, and we're going to schedule.  In
 969	 * this case, we can save a useless back to back clock update.
 970	 */
 971	if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
 972		rq->skip_clock_update = 1;
 973}
 974
 975#ifdef CONFIG_SMP
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 976void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 977{
 978#ifdef CONFIG_SCHED_DEBUG
 979	/*
 980	 * We should never call set_task_cpu() on a blocked task,
 981	 * ttwu() will sort out the placement.
 982	 */
 983	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
 984			!(task_preempt_count(p) & PREEMPT_ACTIVE));
 
 
 
 
 
 
 
 
 
 985
 986#ifdef CONFIG_LOCKDEP
 987	/*
 988	 * The caller should hold either p->pi_lock or rq->lock, when changing
 989	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
 990	 *
 991	 * sched_move_task() holds both and thus holding either pins the cgroup,
 992	 * see task_group().
 993	 *
 994	 * Furthermore, all task_rq users should acquire both locks, see
 995	 * task_rq_lock().
 996	 */
 997	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
 998				      lockdep_is_held(&task_rq(p)->lock)));
 999#endif
 
 
 
 
1000#endif
1001
1002	trace_sched_migrate_task(p, new_cpu);
1003
1004	if (task_cpu(p) != new_cpu) {
1005		if (p->sched_class->migrate_task_rq)
1006			p->sched_class->migrate_task_rq(p, new_cpu);
1007		p->se.nr_migrations++;
1008		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
 
1009	}
1010
1011	__set_task_cpu(p, new_cpu);
1012}
1013
 
1014static void __migrate_swap_task(struct task_struct *p, int cpu)
1015{
1016	if (p->on_rq) {
1017		struct rq *src_rq, *dst_rq;
 
1018
1019		src_rq = task_rq(p);
1020		dst_rq = cpu_rq(cpu);
1021
 
 
 
1022		deactivate_task(src_rq, p, 0);
1023		set_task_cpu(p, cpu);
1024		activate_task(dst_rq, p, 0);
1025		check_preempt_curr(dst_rq, p, 0);
 
 
 
 
1026	} else {
1027		/*
1028		 * Task isn't running anymore; make it appear like we migrated
1029		 * it before it went to sleep. This means on wakeup we make the
1030		 * previous cpu our targer instead of where it really is.
1031		 */
1032		p->wake_cpu = cpu;
1033	}
1034}
1035
1036struct migration_swap_arg {
1037	struct task_struct *src_task, *dst_task;
1038	int src_cpu, dst_cpu;
1039};
1040
1041static int migrate_swap_stop(void *data)
1042{
1043	struct migration_swap_arg *arg = data;
1044	struct rq *src_rq, *dst_rq;
1045	int ret = -EAGAIN;
1046
 
 
 
1047	src_rq = cpu_rq(arg->src_cpu);
1048	dst_rq = cpu_rq(arg->dst_cpu);
1049
1050	double_raw_lock(&arg->src_task->pi_lock,
1051			&arg->dst_task->pi_lock);
1052	double_rq_lock(src_rq, dst_rq);
 
1053	if (task_cpu(arg->dst_task) != arg->dst_cpu)
1054		goto unlock;
1055
1056	if (task_cpu(arg->src_task) != arg->src_cpu)
1057		goto unlock;
1058
1059	if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
1060		goto unlock;
1061
1062	if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
1063		goto unlock;
1064
1065	__migrate_swap_task(arg->src_task, arg->dst_cpu);
1066	__migrate_swap_task(arg->dst_task, arg->src_cpu);
1067
1068	ret = 0;
1069
1070unlock:
1071	double_rq_unlock(src_rq, dst_rq);
1072	raw_spin_unlock(&arg->dst_task->pi_lock);
1073	raw_spin_unlock(&arg->src_task->pi_lock);
1074
1075	return ret;
1076}
1077
1078/*
1079 * Cross migrate two tasks
1080 */
1081int migrate_swap(struct task_struct *cur, struct task_struct *p)
 
1082{
1083	struct migration_swap_arg arg;
1084	int ret = -EINVAL;
1085
1086	arg = (struct migration_swap_arg){
1087		.src_task = cur,
1088		.src_cpu = task_cpu(cur),
1089		.dst_task = p,
1090		.dst_cpu = task_cpu(p),
1091	};
1092
1093	if (arg.src_cpu == arg.dst_cpu)
1094		goto out;
1095
1096	/*
1097	 * These three tests are all lockless; this is OK since all of them
1098	 * will be re-checked with proper locks held further down the line.
1099	 */
1100	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1101		goto out;
1102
1103	if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
1104		goto out;
1105
1106	if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
1107		goto out;
1108
1109	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1110	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1111
1112out:
1113	return ret;
1114}
1115
1116struct migration_arg {
1117	struct task_struct *task;
1118	int dest_cpu;
1119};
1120
1121static int migration_cpu_stop(void *data);
1122
1123/*
1124 * wait_task_inactive - wait for a thread to unschedule.
1125 *
1126 * If @match_state is nonzero, it's the @p->state value just checked and
1127 * not expected to change.  If it changes, i.e. @p might have woken up,
1128 * then return zero.  When we succeed in waiting for @p to be off its CPU,
1129 * we return a positive number (its total switch count).  If a second call
1130 * a short while later returns the same number, the caller can be sure that
1131 * @p has remained unscheduled the whole time.
1132 *
1133 * The caller must ensure that the task *will* unschedule sometime soon,
1134 * else this function might spin for a *long* time. This function can't
1135 * be called with interrupts off, or it may introduce deadlock with
1136 * smp_call_function() if an IPI is sent by the same process we are
1137 * waiting to become inactive.
1138 */
1139unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1140{
1141	unsigned long flags;
1142	int running, on_rq;
1143	unsigned long ncsw;
1144	struct rq *rq;
1145
1146	for (;;) {
1147		/*
1148		 * We do the initial early heuristics without holding
1149		 * any task-queue locks at all. We'll only try to get
1150		 * the runqueue lock when things look like they will
1151		 * work out!
1152		 */
1153		rq = task_rq(p);
1154
1155		/*
1156		 * If the task is actively running on another CPU
1157		 * still, just relax and busy-wait without holding
1158		 * any locks.
1159		 *
1160		 * NOTE! Since we don't hold any locks, it's not
1161		 * even sure that "rq" stays as the right runqueue!
1162		 * But we don't care, since "task_running()" will
1163		 * return false if the runqueue has changed and p
1164		 * is actually now running somewhere else!
1165		 */
1166		while (task_running(rq, p)) {
1167			if (match_state && unlikely(p->state != match_state))
1168				return 0;
1169			cpu_relax();
1170		}
1171
1172		/*
1173		 * Ok, time to look more closely! We need the rq
1174		 * lock now, to be *sure*. If we're wrong, we'll
1175		 * just go back and repeat.
1176		 */
1177		rq = task_rq_lock(p, &flags);
1178		trace_sched_wait_task(p);
1179		running = task_running(rq, p);
1180		on_rq = p->on_rq;
1181		ncsw = 0;
1182		if (!match_state || p->state == match_state)
1183			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1184		task_rq_unlock(rq, p, &flags);
1185
1186		/*
1187		 * If it changed from the expected state, bail out now.
1188		 */
1189		if (unlikely(!ncsw))
1190			break;
1191
1192		/*
1193		 * Was it really running after all now that we
1194		 * checked with the proper locks actually held?
1195		 *
1196		 * Oops. Go back and try again..
1197		 */
1198		if (unlikely(running)) {
1199			cpu_relax();
1200			continue;
1201		}
1202
1203		/*
1204		 * It's not enough that it's not actively running,
1205		 * it must be off the runqueue _entirely_, and not
1206		 * preempted!
1207		 *
1208		 * So if it was still runnable (but just not actively
1209		 * running right now), it's preempted, and we should
1210		 * yield - it could be a while.
1211		 */
1212		if (unlikely(on_rq)) {
1213			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
1214
1215			set_current_state(TASK_UNINTERRUPTIBLE);
1216			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1217			continue;
1218		}
1219
1220		/*
1221		 * Ahh, all good. It wasn't running, and it wasn't
1222		 * runnable, which means that it will never become
1223		 * running in the future either. We're all done!
1224		 */
1225		break;
1226	}
1227
1228	return ncsw;
1229}
1230
1231/***
1232 * kick_process - kick a running thread to enter/exit the kernel
1233 * @p: the to-be-kicked thread
1234 *
1235 * Cause a process which is running on another CPU to enter
1236 * kernel-mode, without any delay. (to get signals handled.)
1237 *
1238 * NOTE: this function doesn't have to take the runqueue lock,
1239 * because all it wants to ensure is that the remote task enters
1240 * the kernel. If the IPI races and the task has been migrated
1241 * to another CPU then no harm is done and the purpose has been
1242 * achieved as well.
1243 */
1244void kick_process(struct task_struct *p)
1245{
1246	int cpu;
1247
1248	preempt_disable();
1249	cpu = task_cpu(p);
1250	if ((cpu != smp_processor_id()) && task_curr(p))
1251		smp_send_reschedule(cpu);
1252	preempt_enable();
1253}
1254EXPORT_SYMBOL_GPL(kick_process);
1255#endif /* CONFIG_SMP */
1256
1257#ifdef CONFIG_SMP
1258/*
1259 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1260 */
1261static int select_fallback_rq(int cpu, struct task_struct *p)
1262{
1263	int nid = cpu_to_node(cpu);
1264	const struct cpumask *nodemask = NULL;
1265	enum { cpuset, possible, fail } state = cpuset;
1266	int dest_cpu;
1267
1268	/*
1269	 * If the node that the cpu is on has been offlined, cpu_to_node()
1270	 * will return -1. There is no cpu on the node, and we should
1271	 * select the cpu on the other node.
1272	 */
1273	if (nid != -1) {
1274		nodemask = cpumask_of_node(nid);
1275
1276		/* Look for allowed, online CPU in same node. */
1277		for_each_cpu(dest_cpu, nodemask) {
1278			if (!cpu_online(dest_cpu))
1279				continue;
1280			if (!cpu_active(dest_cpu))
1281				continue;
1282			if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1283				return dest_cpu;
1284		}
1285	}
1286
1287	for (;;) {
1288		/* Any allowed, online CPU? */
1289		for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1290			if (!cpu_online(dest_cpu))
1291				continue;
1292			if (!cpu_active(dest_cpu))
1293				continue;
 
1294			goto out;
1295		}
1296
 
1297		switch (state) {
1298		case cpuset:
1299			/* No more Mr. Nice Guy. */
1300			cpuset_cpus_allowed_fallback(p);
1301			state = possible;
1302			break;
1303
 
1304		case possible:
1305			do_set_cpus_allowed(p, cpu_possible_mask);
1306			state = fail;
1307			break;
1308
1309		case fail:
1310			BUG();
1311			break;
1312		}
1313	}
1314
1315out:
1316	if (state != cpuset) {
1317		/*
1318		 * Don't tell them about moving exiting tasks or
1319		 * kernel threads (both mm NULL), since they never
1320		 * leave kernel.
1321		 */
1322		if (p->mm && printk_ratelimit()) {
1323			printk_sched("process %d (%s) no longer affine to cpu%d\n",
1324					task_pid_nr(p), p->comm, cpu);
1325		}
1326	}
1327
1328	return dest_cpu;
1329}
1330
1331/*
1332 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1333 */
1334static inline
1335int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1336{
1337	cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
 
 
 
 
 
1338
1339	/*
1340	 * In order not to call set_task_cpu() on a blocking task we need
1341	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
1342	 * cpu.
1343	 *
1344	 * Since this is common to all placement strategies, this lives here.
1345	 *
1346	 * [ this allows ->select_task() to simply return task_cpu(p) and
1347	 *   not worry about this generic constraint ]
1348	 */
1349	if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
1350		     !cpu_online(cpu)))
1351		cpu = select_fallback_rq(task_cpu(p), p);
1352
1353	return cpu;
1354}
1355
1356static void update_avg(u64 *avg, u64 sample)
1357{
1358	s64 diff = sample - *avg;
1359	*avg += diff >> 3;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1360}
1361#endif
 
 
 
 
 
 
 
 
 
1362
1363static void
1364ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
1365{
1366#ifdef CONFIG_SCHEDSTATS
1367	struct rq *rq = this_rq();
1368
1369#ifdef CONFIG_SMP
1370	int this_cpu = smp_processor_id();
 
 
1371
1372	if (cpu == this_cpu) {
1373		schedstat_inc(rq, ttwu_local);
1374		schedstat_inc(p, se.statistics.nr_wakeups_local);
 
1375	} else {
1376		struct sched_domain *sd;
1377
1378		schedstat_inc(p, se.statistics.nr_wakeups_remote);
1379		rcu_read_lock();
1380		for_each_domain(this_cpu, sd) {
1381			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
1382				schedstat_inc(sd, ttwu_wake_remote);
1383				break;
1384			}
1385		}
1386		rcu_read_unlock();
1387	}
1388
1389	if (wake_flags & WF_MIGRATED)
1390		schedstat_inc(p, se.statistics.nr_wakeups_migrate);
1391
1392#endif /* CONFIG_SMP */
1393
1394	schedstat_inc(rq, ttwu_count);
1395	schedstat_inc(p, se.statistics.nr_wakeups);
1396
1397	if (wake_flags & WF_SYNC)
1398		schedstat_inc(p, se.statistics.nr_wakeups_sync);
1399
1400#endif /* CONFIG_SCHEDSTATS */
1401}
1402
1403static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
1404{
1405	activate_task(rq, p, en_flags);
1406	p->on_rq = 1;
1407
1408	/* if a worker is waking up, notify workqueue */
1409	if (p->flags & PF_WQ_WORKER)
1410		wq_worker_waking_up(p, cpu_of(rq));
1411}
1412
1413/*
1414 * Mark the task runnable and perform wakeup-preemption.
1415 */
1416static void
1417ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
1418{
1419	check_preempt_curr(rq, p, wake_flags);
1420	trace_sched_wakeup(p, true);
1421
1422	p->state = TASK_RUNNING;
 
 
1423#ifdef CONFIG_SMP
1424	if (p->sched_class->task_woken)
 
 
 
 
 
1425		p->sched_class->task_woken(rq, p);
 
 
1426
1427	if (rq->idle_stamp) {
1428		u64 delta = rq_clock(rq) - rq->idle_stamp;
1429		u64 max = 2*rq->max_idle_balance_cost;
1430
1431		update_avg(&rq->avg_idle, delta);
1432
1433		if (rq->avg_idle > max)
1434			rq->avg_idle = max;
1435
1436		rq->idle_stamp = 0;
1437	}
1438#endif
1439}
1440
1441static void
1442ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
 
1443{
1444#ifdef CONFIG_SMP
 
 
 
1445	if (p->sched_contributes_to_load)
1446		rq->nr_uninterruptible--;
 
 
 
 
1447#endif
1448
1449	ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
1450	ttwu_do_wakeup(rq, p, wake_flags);
1451}
1452
1453/*
1454 * Called in case the task @p isn't fully descheduled from its runqueue,
1455 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
1456 * since all we need to do is flip p->state to TASK_RUNNING, since
1457 * the task is still ->on_rq.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1458 */
1459static int ttwu_remote(struct task_struct *p, int wake_flags)
1460{
 
1461	struct rq *rq;
1462	int ret = 0;
1463
1464	rq = __task_rq_lock(p);
1465	if (p->on_rq) {
1466		/* check_preempt_curr() may use rq clock */
1467		update_rq_clock(rq);
1468		ttwu_do_wakeup(rq, p, wake_flags);
1469		ret = 1;
1470	}
1471	__task_rq_unlock(rq);
1472
1473	return ret;
1474}
1475
1476#ifdef CONFIG_SMP
1477static void sched_ttwu_pending(void)
1478{
 
1479	struct rq *rq = this_rq();
1480	struct llist_node *llist = llist_del_all(&rq->wake_list);
1481	struct task_struct *p;
1482
1483	raw_spin_lock(&rq->lock);
 
 
 
 
 
 
 
 
1484
1485	while (llist) {
1486		p = llist_entry(llist, struct task_struct, wake_entry);
1487		llist = llist_next(llist);
1488		ttwu_do_activate(rq, p, 0);
 
 
 
 
 
 
 
1489	}
1490
1491	raw_spin_unlock(&rq->lock);
1492}
1493
1494void scheduler_ipi(void)
1495{
1496	/*
1497	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1498	 * TIF_NEED_RESCHED remotely (for the first time) will also send
1499	 * this IPI.
1500	 */
1501	preempt_fold_need_resched();
1502
1503	if (llist_empty(&this_rq()->wake_list)
1504			&& !tick_nohz_full_cpu(smp_processor_id())
1505			&& !got_nohz_idle_kick())
1506		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1507
 
 
1508	/*
1509	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1510	 * traditionally all their work was done from the interrupt return
1511	 * path. Now that we actually do some work, we need to make sure
1512	 * we do call them.
1513	 *
1514	 * Some archs already do call them, luckily irq_enter/exit nest
1515	 * properly.
1516	 *
1517	 * Arguably we should visit all archs and update all handlers,
1518	 * however a fair share of IPIs are still resched only so this would
1519	 * somewhat pessimize the simple resched case.
1520	 */
1521	irq_enter();
1522	tick_nohz_full_check();
1523	sched_ttwu_pending();
1524
1525	/*
1526	 * Check if someone kicked us for doing the nohz idle load balance.
 
 
 
1527	 */
1528	if (unlikely(got_nohz_idle_kick())) {
1529		this_rq()->idle_balance = 1;
1530		raise_softirq_irqoff(SCHED_SOFTIRQ);
1531	}
1532	irq_exit();
1533}
1534
1535static void ttwu_queue_remote(struct task_struct *p, int cpu)
1536{
1537	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list))
1538		smp_send_reschedule(cpu);
 
 
 
 
 
 
 
 
1539}
1540
1541bool cpus_share_cache(int this_cpu, int that_cpu)
 
 
1542{
1543	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
1544}
 
1545#endif /* CONFIG_SMP */
1546
1547static void ttwu_queue(struct task_struct *p, int cpu)
1548{
1549	struct rq *rq = cpu_rq(cpu);
 
1550
1551#if defined(CONFIG_SMP)
1552	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1553		sched_clock_cpu(cpu); /* sync clocks x-cpu */
1554		ttwu_queue_remote(p, cpu);
1555		return;
1556	}
1557#endif
1558
1559	raw_spin_lock(&rq->lock);
1560	ttwu_do_activate(rq, p, 0);
1561	raw_spin_unlock(&rq->lock);
 
1562}
1563
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1564/**
1565 * try_to_wake_up - wake up a thread
1566 * @p: the thread to be awakened
1567 * @state: the mask of task states that can be woken
1568 * @wake_flags: wake modifier flags (WF_*)
1569 *
1570 * Put it on the run-queue if it's not already there. The "current"
1571 * thread is always on the run-queue (except when the actual
1572 * re-schedule is in progress), and as such you're allowed to do
1573 * the simpler "current->state = TASK_RUNNING" to mark yourself
1574 * runnable without the overhead of this.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1575 *
1576 * Return: %true if @p was woken up, %false if it was already running.
1577 * or @state didn't match @p's state.
 
 
 
1578 */
1579static int
1580try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
1581{
1582	unsigned long flags;
1583	int cpu, success = 0;
1584
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1585	/*
1586	 * If we are going to wake up a thread waiting for CONDITION we
1587	 * need to ensure that CONDITION=1 done by the caller can not be
1588	 * reordered with p->state check below. This pairs with mb() in
1589	 * set_current_state() the waiting thread does.
1590	 */
1591	smp_mb__before_spinlock();
1592	raw_spin_lock_irqsave(&p->pi_lock, flags);
 
1593	if (!(p->state & state))
1594		goto out;
1595
1596	success = 1; /* we're going to change ->state */
1597	cpu = task_cpu(p);
1598
1599	if (p->on_rq && ttwu_remote(p, wake_flags))
1600		goto stat;
1601
1602#ifdef CONFIG_SMP
1603	/*
1604	 * If the owning (remote) cpu is still in the middle of schedule() with
1605	 * this task as prev, wait until its done referencing the task.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1606	 */
1607	while (p->on_cpu)
1608		cpu_relax();
 
 
 
 
 
 
 
 
1609	/*
1610	 * Pairs with the smp_wmb() in finish_lock_switch().
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1611	 */
1612	smp_rmb();
1613
1614	p->sched_contributes_to_load = !!task_contributes_to_load(p);
 
 
 
 
 
1615	p->state = TASK_WAKING;
1616
1617	if (p->sched_class->task_waking)
1618		p->sched_class->task_waking(p);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1619
1620	cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
1621	if (task_cpu(p) != cpu) {
1622		wake_flags |= WF_MIGRATED;
 
1623		set_task_cpu(p, cpu);
1624	}
 
 
1625#endif /* CONFIG_SMP */
1626
1627	ttwu_queue(p, cpu);
1628stat:
1629	ttwu_stat(p, cpu, wake_flags);
1630out:
1631	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
 
 
 
1632
1633	return success;
1634}
1635
1636/**
1637 * try_to_wake_up_local - try to wake up a local task with rq lock held
1638 * @p: the thread to be awakened
1639 *
1640 * Put @p on the run-queue if it's not already there. The caller must
1641 * ensure that this_rq() is locked, @p is bound to this_rq() and not
1642 * the current task.
 
 
 
 
 
 
 
 
 
 
1643 */
1644static void try_to_wake_up_local(struct task_struct *p)
1645{
1646	struct rq *rq = task_rq(p);
1647
1648	if (WARN_ON_ONCE(rq != this_rq()) ||
1649	    WARN_ON_ONCE(p == current))
1650		return;
1651
1652	lockdep_assert_held(&rq->lock);
1653
1654	if (!raw_spin_trylock(&p->pi_lock)) {
1655		raw_spin_unlock(&rq->lock);
1656		raw_spin_lock(&p->pi_lock);
1657		raw_spin_lock(&rq->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
1658	}
1659
1660	if (!(p->state & TASK_NORMAL))
1661		goto out;
1662
1663	if (!p->on_rq)
1664		ttwu_activate(rq, p, ENQUEUE_WAKEUP);
1665
1666	ttwu_do_wakeup(rq, p, 0);
1667	ttwu_stat(p, smp_processor_id(), 0);
1668out:
1669	raw_spin_unlock(&p->pi_lock);
1670}
1671
1672/**
1673 * wake_up_process - Wake up a specific process
1674 * @p: The process to be woken up.
1675 *
1676 * Attempt to wake up the nominated process and move it to the set of runnable
1677 * processes.
1678 *
1679 * Return: 1 if the process was woken up, 0 if it was already running.
1680 *
1681 * It may be assumed that this function implies a write memory barrier before
1682 * changing the task state if and only if any tasks are woken up.
1683 */
1684int wake_up_process(struct task_struct *p)
1685{
1686	WARN_ON(task_is_stopped_or_traced(p));
1687	return try_to_wake_up(p, TASK_NORMAL, 0);
1688}
1689EXPORT_SYMBOL(wake_up_process);
1690
1691int wake_up_state(struct task_struct *p, unsigned int state)
1692{
1693	return try_to_wake_up(p, state, 0);
1694}
1695
1696/*
1697 * Perform scheduler related setup for a newly forked process p.
1698 * p is forked by current.
1699 *
1700 * __sched_fork() is basic setup used by init_idle() too:
1701 */
1702static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
1703{
1704	p->on_rq			= 0;
1705
1706	p->se.on_rq			= 0;
1707	p->se.exec_start		= 0;
1708	p->se.sum_exec_runtime		= 0;
1709	p->se.prev_sum_exec_runtime	= 0;
1710	p->se.nr_migrations		= 0;
1711	p->se.vruntime			= 0;
1712	INIT_LIST_HEAD(&p->se.group_node);
1713
 
 
 
 
1714#ifdef CONFIG_SCHEDSTATS
 
1715	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
1716#endif
1717
1718	RB_CLEAR_NODE(&p->dl.rb_node);
1719	hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1720	p->dl.dl_runtime = p->dl.runtime = 0;
1721	p->dl.dl_deadline = p->dl.deadline = 0;
1722	p->dl.dl_period = 0;
1723	p->dl.flags = 0;
1724
1725	INIT_LIST_HEAD(&p->rt.run_list);
 
 
 
 
1726
1727#ifdef CONFIG_PREEMPT_NOTIFIERS
1728	INIT_HLIST_HEAD(&p->preempt_notifiers);
1729#endif
1730
 
 
 
 
 
 
 
 
 
 
 
1731#ifdef CONFIG_NUMA_BALANCING
1732	if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
1733		p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
1734		p->mm->numa_scan_seq = 0;
1735	}
1736
1737	if (clone_flags & CLONE_VM)
1738		p->numa_preferred_nid = current->numa_preferred_nid;
 
 
1739	else
1740		p->numa_preferred_nid = -1;
 
1741
1742	p->node_stamp = 0ULL;
1743	p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
1744	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
1745	p->numa_work.next = &p->numa_work;
1746	p->numa_faults_memory = NULL;
1747	p->numa_faults_buffer_memory = NULL;
1748	p->last_task_numa_placement = 0;
1749	p->last_sum_exec_runtime = 0;
1750
1751	INIT_LIST_HEAD(&p->numa_entry);
1752	p->numa_group = NULL;
1753#endif /* CONFIG_NUMA_BALANCING */
 
 
 
 
 
 
 
 
1754}
 
 
1755
1756#ifdef CONFIG_NUMA_BALANCING
1757#ifdef CONFIG_SCHED_DEBUG
1758void set_numabalancing_state(bool enabled)
 
 
 
1759{
1760	if (enabled)
1761		sched_feat_set("NUMA");
1762	else
1763		sched_feat_set("NO_NUMA");
1764}
1765#else
1766__read_mostly bool numabalancing_enabled;
1767
1768void set_numabalancing_state(bool enabled)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1769{
1770	numabalancing_enabled = enabled;
1771}
1772#endif /* CONFIG_SCHED_DEBUG */
1773
1774#ifdef CONFIG_PROC_SYSCTL
1775int sysctl_numa_balancing(struct ctl_table *table, int write,
1776			 void __user *buffer, size_t *lenp, loff_t *ppos)
1777{
1778	struct ctl_table t;
1779	int err;
1780	int state = numabalancing_enabled;
1781
1782	if (write && !capable(CAP_SYS_ADMIN))
1783		return -EPERM;
1784
1785	t = *table;
1786	t.data = &state;
1787	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
1788	if (err < 0)
1789		return err;
1790	if (write)
1791		set_numabalancing_state(state);
1792	return err;
1793}
1794#endif
1795#endif
 
 
1796
1797/*
1798 * fork()/clone()-time setup:
1799 */
1800int sched_fork(unsigned long clone_flags, struct task_struct *p)
1801{
1802	unsigned long flags;
1803	int cpu = get_cpu();
1804
1805	__sched_fork(clone_flags, p);
1806	/*
1807	 * We mark the process as running here. This guarantees that
1808	 * nobody will actually run it, and a signal or other external
1809	 * event cannot wake it up and insert it on the runqueue either.
1810	 */
1811	p->state = TASK_RUNNING;
1812
1813	/*
1814	 * Make sure we do not leak PI boosting priority to the child.
1815	 */
1816	p->prio = current->normal_prio;
1817
 
 
1818	/*
1819	 * Revert to default priority/policy on fork if requested.
1820	 */
1821	if (unlikely(p->sched_reset_on_fork)) {
1822		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
1823			p->policy = SCHED_NORMAL;
1824			p->static_prio = NICE_TO_PRIO(0);
1825			p->rt_priority = 0;
1826		} else if (PRIO_TO_NICE(p->static_prio) < 0)
1827			p->static_prio = NICE_TO_PRIO(0);
1828
1829		p->prio = p->normal_prio = __normal_prio(p);
1830		set_load_weight(p);
1831
1832		/*
1833		 * We don't need the reset flag anymore after the fork. It has
1834		 * fulfilled its duty:
1835		 */
1836		p->sched_reset_on_fork = 0;
1837	}
1838
1839	if (dl_prio(p->prio)) {
1840		put_cpu();
1841		return -EAGAIN;
1842	} else if (rt_prio(p->prio)) {
1843		p->sched_class = &rt_sched_class;
1844	} else {
1845		p->sched_class = &fair_sched_class;
1846	}
1847
1848	if (p->sched_class->task_fork)
1849		p->sched_class->task_fork(p);
1850
1851	/*
1852	 * The child is not yet in the pid-hash so no cgroup attach races,
1853	 * and the cgroup is pinned to this child due to cgroup_fork()
1854	 * is ran before sched_fork().
1855	 *
1856	 * Silence PROVE_RCU.
1857	 */
1858	raw_spin_lock_irqsave(&p->pi_lock, flags);
1859	set_task_cpu(p, cpu);
 
 
 
 
 
 
 
1860	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1861
1862#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
1863	if (likely(sched_info_on()))
1864		memset(&p->sched_info, 0, sizeof(p->sched_info));
1865#endif
1866#if defined(CONFIG_SMP)
1867	p->on_cpu = 0;
1868#endif
1869	init_task_preempt_count(p);
1870#ifdef CONFIG_SMP
1871	plist_node_init(&p->pushable_tasks, MAX_PRIO);
1872	RB_CLEAR_NODE(&p->pushable_dl_tasks);
1873#endif
1874
1875	put_cpu();
1876	return 0;
1877}
1878
 
 
 
 
 
1879unsigned long to_ratio(u64 period, u64 runtime)
1880{
1881	if (runtime == RUNTIME_INF)
1882		return 1ULL << 20;
1883
1884	/*
1885	 * Doing this here saves a lot of checks in all
1886	 * the calling paths, and returning zero seems
1887	 * safe for them anyway.
1888	 */
1889	if (period == 0)
1890		return 0;
1891
1892	return div64_u64(runtime << 20, period);
1893}
1894
1895#ifdef CONFIG_SMP
1896inline struct dl_bw *dl_bw_of(int i)
1897{
1898	return &cpu_rq(i)->rd->dl_bw;
1899}
1900
1901static inline int dl_bw_cpus(int i)
1902{
1903	struct root_domain *rd = cpu_rq(i)->rd;
1904	int cpus = 0;
1905
1906	for_each_cpu_and(i, rd->span, cpu_active_mask)
1907		cpus++;
1908
1909	return cpus;
1910}
1911#else
1912inline struct dl_bw *dl_bw_of(int i)
1913{
1914	return &cpu_rq(i)->dl.dl_bw;
1915}
1916
1917static inline int dl_bw_cpus(int i)
1918{
1919	return 1;
1920}
1921#endif
1922
1923static inline
1924void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw)
1925{
1926	dl_b->total_bw -= tsk_bw;
1927}
1928
1929static inline
1930void __dl_add(struct dl_bw *dl_b, u64 tsk_bw)
1931{
1932	dl_b->total_bw += tsk_bw;
1933}
1934
1935static inline
1936bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
1937{
1938	return dl_b->bw != -1 &&
1939	       dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
1940}
1941
1942/*
1943 * We must be sure that accepting a new task (or allowing changing the
1944 * parameters of an existing one) is consistent with the bandwidth
1945 * constraints. If yes, this function also accordingly updates the currently
1946 * allocated bandwidth to reflect the new situation.
1947 *
1948 * This function is called while holding p's rq->lock.
1949 */
1950static int dl_overflow(struct task_struct *p, int policy,
1951		       const struct sched_attr *attr)
1952{
1953
1954	struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
1955	u64 period = attr->sched_period ?: attr->sched_deadline;
1956	u64 runtime = attr->sched_runtime;
1957	u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
1958	int cpus, err = -1;
1959
1960	if (new_bw == p->dl.dl_bw)
1961		return 0;
1962
1963	/*
1964	 * Either if a task, enters, leave, or stays -deadline but changes
1965	 * its parameters, we may need to update accordingly the total
1966	 * allocated bandwidth of the container.
1967	 */
1968	raw_spin_lock(&dl_b->lock);
1969	cpus = dl_bw_cpus(task_cpu(p));
1970	if (dl_policy(policy) && !task_has_dl_policy(p) &&
1971	    !__dl_overflow(dl_b, cpus, 0, new_bw)) {
1972		__dl_add(dl_b, new_bw);
1973		err = 0;
1974	} else if (dl_policy(policy) && task_has_dl_policy(p) &&
1975		   !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
1976		__dl_clear(dl_b, p->dl.dl_bw);
1977		__dl_add(dl_b, new_bw);
1978		err = 0;
1979	} else if (!dl_policy(policy) && task_has_dl_policy(p)) {
1980		__dl_clear(dl_b, p->dl.dl_bw);
1981		err = 0;
1982	}
1983	raw_spin_unlock(&dl_b->lock);
1984
1985	return err;
1986}
1987
1988extern void init_dl_bw(struct dl_bw *dl_b);
1989
1990/*
1991 * wake_up_new_task - wake up a newly created task for the first time.
1992 *
1993 * This function will do some initial scheduler statistics housekeeping
1994 * that must be done for every newly created context, then puts the task
1995 * on the runqueue and wakes it.
1996 */
1997void wake_up_new_task(struct task_struct *p)
1998{
1999	unsigned long flags;
2000	struct rq *rq;
2001
2002	raw_spin_lock_irqsave(&p->pi_lock, flags);
 
2003#ifdef CONFIG_SMP
2004	/*
2005	 * Fork balancing, do it here and not earlier because:
2006	 *  - cpus_allowed can change in the fork path
2007	 *  - any previously selected cpu might disappear through hotplug
 
 
 
2008	 */
2009	set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
 
 
2010#endif
 
 
 
2011
2012	/* Initialize new task's runnable average */
2013	init_task_runnable_average(p);
2014	rq = __task_rq_lock(p);
2015	activate_task(rq, p, 0);
2016	p->on_rq = 1;
2017	trace_sched_wakeup_new(p, true);
2018	check_preempt_curr(rq, p, WF_FORK);
2019#ifdef CONFIG_SMP
2020	if (p->sched_class->task_woken)
 
 
 
 
 
2021		p->sched_class->task_woken(rq, p);
 
 
2022#endif
2023	task_rq_unlock(rq, p, &flags);
2024}
2025
2026#ifdef CONFIG_PREEMPT_NOTIFIERS
2027
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2028/**
2029 * preempt_notifier_register - tell me when current is being preempted & rescheduled
2030 * @notifier: notifier struct to register
2031 */
2032void preempt_notifier_register(struct preempt_notifier *notifier)
2033{
 
 
 
2034	hlist_add_head(&notifier->link, &current->preempt_notifiers);
2035}
2036EXPORT_SYMBOL_GPL(preempt_notifier_register);
2037
2038/**
2039 * preempt_notifier_unregister - no longer interested in preemption notifications
2040 * @notifier: notifier struct to unregister
2041 *
2042 * This is safe to call from within a preemption notifier.
2043 */
2044void preempt_notifier_unregister(struct preempt_notifier *notifier)
2045{
2046	hlist_del(&notifier->link);
2047}
2048EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2049
2050static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2051{
2052	struct preempt_notifier *notifier;
2053
2054	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2055		notifier->ops->sched_in(notifier, raw_smp_processor_id());
2056}
2057
 
 
 
 
 
 
2058static void
2059fire_sched_out_preempt_notifiers(struct task_struct *curr,
2060				 struct task_struct *next)
2061{
2062	struct preempt_notifier *notifier;
2063
2064	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2065		notifier->ops->sched_out(notifier, next);
2066}
2067
 
 
 
 
 
 
 
 
2068#else /* !CONFIG_PREEMPT_NOTIFIERS */
2069
2070static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2071{
2072}
2073
2074static void
2075fire_sched_out_preempt_notifiers(struct task_struct *curr,
2076				 struct task_struct *next)
2077{
2078}
2079
2080#endif /* CONFIG_PREEMPT_NOTIFIERS */
2081
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2082/**
2083 * prepare_task_switch - prepare to switch tasks
2084 * @rq: the runqueue preparing to switch
2085 * @prev: the current task that is being switched out
2086 * @next: the task we are going to switch to.
2087 *
2088 * This is called with the rq lock held and interrupts off. It must
2089 * be paired with a subsequent finish_task_switch after the context
2090 * switch.
2091 *
2092 * prepare_task_switch sets up locking and calls architecture specific
2093 * hooks.
2094 */
2095static inline void
2096prepare_task_switch(struct rq *rq, struct task_struct *prev,
2097		    struct task_struct *next)
2098{
2099	trace_sched_switch(prev, next);
2100	sched_info_switch(rq, prev, next);
2101	perf_event_task_sched_out(prev, next);
 
2102	fire_sched_out_preempt_notifiers(prev, next);
2103	prepare_lock_switch(rq, next);
2104	prepare_arch_switch(next);
2105}
2106
2107/**
2108 * finish_task_switch - clean up after a task-switch
2109 * @rq: runqueue associated with task-switch
2110 * @prev: the thread we just switched away from.
2111 *
2112 * finish_task_switch must be called after the context switch, paired
2113 * with a prepare_task_switch call before the context switch.
2114 * finish_task_switch will reconcile locking set up by prepare_task_switch,
2115 * and do any other architecture-specific cleanup actions.
2116 *
2117 * Note that we may have delayed dropping an mm in context_switch(). If
2118 * so, we finish that here outside of the runqueue lock. (Doing it
2119 * with the lock held can cause deadlocks; see schedule() for
2120 * details.)
 
 
 
 
 
2121 */
2122static void finish_task_switch(struct rq *rq, struct task_struct *prev)
2123	__releases(rq->lock)
2124{
 
2125	struct mm_struct *mm = rq->prev_mm;
2126	long prev_state;
2127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2128	rq->prev_mm = NULL;
2129
2130	/*
2131	 * A task struct has one reference for the use as "current".
2132	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
2133	 * schedule one last time. The schedule call will never return, and
2134	 * the scheduled task must drop that reference.
2135	 * The test for TASK_DEAD must occur while the runqueue locks are
2136	 * still held, otherwise prev could be scheduled on another cpu, die
2137	 * there before we look at prev->state, and then the reference would
2138	 * be dropped twice.
2139	 *		Manfred Spraul <manfred@colorfullife.com>
2140	 */
2141	prev_state = prev->state;
2142	vtime_task_switch(prev);
2143	finish_arch_switch(prev);
2144	perf_event_task_sched_in(prev, current);
2145	finish_lock_switch(rq, prev);
 
2146	finish_arch_post_lock_switch();
 
2147
2148	fire_sched_in_preempt_notifiers(current);
2149	if (mm)
 
 
 
 
 
 
 
 
 
 
 
 
 
2150		mmdrop(mm);
 
2151	if (unlikely(prev_state == TASK_DEAD)) {
2152		if (prev->sched_class->task_dead)
2153			prev->sched_class->task_dead(prev);
2154
2155		/*
2156		 * Remove function-return probe instances associated with this
2157		 * task and put them back on the free list.
2158		 */
2159		kprobe_flush_task(prev);
2160		put_task_struct(prev);
 
 
 
 
2161	}
2162
2163	tick_nohz_task_switch(current);
 
2164}
2165
2166#ifdef CONFIG_SMP
2167
2168/* rq->lock is NOT held, but preemption is disabled */
2169static inline void post_schedule(struct rq *rq)
2170{
2171	if (rq->post_schedule) {
2172		unsigned long flags;
 
2173
2174		raw_spin_lock_irqsave(&rq->lock, flags);
2175		if (rq->curr->sched_class->post_schedule)
2176			rq->curr->sched_class->post_schedule(rq);
2177		raw_spin_unlock_irqrestore(&rq->lock, flags);
 
 
 
 
2178
2179		rq->post_schedule = 0;
2180	}
 
 
 
 
 
 
 
2181}
2182
2183#else
2184
2185static inline void post_schedule(struct rq *rq)
2186{
2187}
2188
2189#endif
2190
2191/**
2192 * schedule_tail - first thing a freshly forked thread must call.
2193 * @prev: the thread we just switched away from.
2194 */
2195asmlinkage __visible void schedule_tail(struct task_struct *prev)
2196	__releases(rq->lock)
2197{
2198	struct rq *rq = this_rq();
2199
2200	finish_task_switch(rq, prev);
2201
2202	/*
2203	 * FIXME: do we need to worry about rq being invalidated by the
2204	 * task_switch?
 
 
 
 
2205	 */
2206	post_schedule(rq);
2207
2208#ifdef __ARCH_WANT_UNLOCKED_CTXSW
2209	/* In this case, finish_task_switch does not reenable preemption */
2210	preempt_enable();
2211#endif
2212	if (current->set_child_tid)
2213		put_user(task_pid_vnr(current), current->set_child_tid);
 
 
2214}
2215
2216/*
2217 * context_switch - switch to the new MM and the new
2218 * thread's register state.
2219 */
2220static inline void
2221context_switch(struct rq *rq, struct task_struct *prev,
2222	       struct task_struct *next)
2223{
2224	struct mm_struct *mm, *oldmm;
2225
2226	prepare_task_switch(rq, prev, next);
2227
2228	mm = next->mm;
2229	oldmm = prev->active_mm;
2230	/*
2231	 * For paravirt, this is coupled with an exit in switch_to to
2232	 * combine the page table reload and the switch backend into
2233	 * one hypercall.
2234	 */
2235	arch_start_context_switch(prev);
2236
2237	if (!mm) {
2238		next->active_mm = oldmm;
2239		atomic_inc(&oldmm->mm_count);
2240		enter_lazy_tlb(oldmm, next);
2241	} else
2242		switch_mm(oldmm, mm, next);
2243
2244	if (!prev->mm) {
2245		prev->active_mm = NULL;
2246		rq->prev_mm = oldmm;
2247	}
2248	/*
2249	 * Since the runqueue lock will be released by the next
2250	 * task (which is an invalid locking op but in the case
2251	 * of the scheduler it's an obvious special-case), so we
2252	 * do an early lockdep release here:
 
2253	 */
2254#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2255	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2256#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2257
2258	context_tracking_task_switch(prev, next);
2259	/* Here we just switch the register state and the stack. */
2260	switch_to(prev, next, prev);
2261
2262	barrier();
2263	/*
2264	 * this_rq must be evaluated again because prev may have moved
2265	 * CPUs since it called schedule(), thus the 'rq' on its stack
2266	 * frame will be invalid.
2267	 */
2268	finish_task_switch(this_rq(), prev);
2269}
2270
2271/*
2272 * nr_running and nr_context_switches:
2273 *
2274 * externally visible scheduler statistics: current number of runnable
2275 * threads, total number of context switches performed since bootup.
2276 */
2277unsigned long nr_running(void)
2278{
2279	unsigned long i, sum = 0;
2280
2281	for_each_online_cpu(i)
2282		sum += cpu_rq(i)->nr_running;
2283
2284	return sum;
2285}
2286
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2287unsigned long long nr_context_switches(void)
2288{
2289	int i;
2290	unsigned long long sum = 0;
2291
2292	for_each_possible_cpu(i)
2293		sum += cpu_rq(i)->nr_switches;
2294
2295	return sum;
2296}
2297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2298unsigned long nr_iowait(void)
2299{
2300	unsigned long i, sum = 0;
2301
2302	for_each_possible_cpu(i)
2303		sum += atomic_read(&cpu_rq(i)->nr_iowait);
2304
2305	return sum;
2306}
2307
2308unsigned long nr_iowait_cpu(int cpu)
2309{
2310	struct rq *this = cpu_rq(cpu);
2311	return atomic_read(&this->nr_iowait);
2312}
2313
2314#ifdef CONFIG_SMP
2315
2316/*
2317 * sched_exec - execve() is a valuable balancing opportunity, because at
2318 * this point the task has the smallest effective memory and cache footprint.
2319 */
2320void sched_exec(void)
2321{
2322	struct task_struct *p = current;
2323	unsigned long flags;
2324	int dest_cpu;
2325
2326	raw_spin_lock_irqsave(&p->pi_lock, flags);
2327	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
2328	if (dest_cpu == smp_processor_id())
2329		goto unlock;
2330
2331	if (likely(cpu_active(dest_cpu))) {
2332		struct migration_arg arg = { p, dest_cpu };
2333
2334		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2335		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
2336		return;
2337	}
2338unlock:
2339	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2340}
2341
2342#endif
2343
2344DEFINE_PER_CPU(struct kernel_stat, kstat);
2345DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
2346
2347EXPORT_PER_CPU_SYMBOL(kstat);
2348EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
2349
2350/*
2351 * Return any ns on the sched_clock that have not yet been accounted in
2352 * @p in case that task is currently running.
2353 *
2354 * Called with task_rq_lock() held on @rq.
2355 */
2356static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
2357{
2358	u64 ns = 0;
2359
2360	if (task_current(rq, p)) {
2361		update_rq_clock(rq);
2362		ns = rq_clock_task(rq) - p->se.exec_start;
2363		if ((s64)ns < 0)
2364			ns = 0;
2365	}
2366
2367	return ns;
2368}
2369
2370unsigned long long task_delta_exec(struct task_struct *p)
2371{
2372	unsigned long flags;
2373	struct rq *rq;
2374	u64 ns = 0;
2375
2376	rq = task_rq_lock(p, &flags);
2377	ns = do_task_delta_exec(p, rq);
2378	task_rq_unlock(rq, p, &flags);
2379
2380	return ns;
2381}
2382
2383/*
2384 * Return accounted runtime for the task.
2385 * In case the task is currently running, return the runtime plus current's
2386 * pending runtime that have not been accounted yet.
2387 */
2388unsigned long long task_sched_runtime(struct task_struct *p)
2389{
2390	unsigned long flags;
2391	struct rq *rq;
2392	u64 ns = 0;
2393
2394#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
2395	/*
2396	 * 64-bit doesn't need locks to atomically read a 64bit value.
2397	 * So we have a optimization chance when the task's delta_exec is 0.
2398	 * Reading ->on_cpu is racy, but this is ok.
2399	 *
2400	 * If we race with it leaving cpu, we'll take a lock. So we're correct.
2401	 * If we race with it entering cpu, unaccounted time is 0. This is
2402	 * indistinguishable from the read occurring a few cycles earlier.
 
 
2403	 */
2404	if (!p->on_cpu)
2405		return p->se.sum_exec_runtime;
2406#endif
2407
2408	rq = task_rq_lock(p, &flags);
2409	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
2410	task_rq_unlock(rq, p, &flags);
 
 
 
 
 
 
 
 
 
 
2411
2412	return ns;
2413}
2414
2415/*
2416 * This function gets called by the timer code, with HZ frequency.
2417 * We call it with interrupts disabled.
2418 */
2419void scheduler_tick(void)
2420{
2421	int cpu = smp_processor_id();
2422	struct rq *rq = cpu_rq(cpu);
2423	struct task_struct *curr = rq->curr;
 
 
2424
 
2425	sched_clock_tick();
2426
2427	raw_spin_lock(&rq->lock);
 
2428	update_rq_clock(rq);
 
 
2429	curr->sched_class->task_tick(rq, curr, 0);
2430	update_cpu_load_active(rq);
2431	raw_spin_unlock(&rq->lock);
 
 
2432
2433	perf_event_task_tick();
2434
2435#ifdef CONFIG_SMP
2436	rq->idle_balance = idle_cpu(cpu);
2437	trigger_load_balance(rq);
2438#endif
2439	rq_last_tick_reset(rq);
2440}
2441
2442#ifdef CONFIG_NO_HZ_FULL
2443/**
2444 * scheduler_tick_max_deferment
 
 
 
 
 
 
 
 
 
 
 
 
2445 *
2446 * Keep at least one tick per second when a single
2447 * active task is running because the scheduler doesn't
2448 * yet completely support full dynticks environment.
 
 
 
 
 
 
 
 
 
 
2449 *
2450 * This makes sure that uptime, CFS vruntime, load
2451 * balancing, etc... continue to move forward, even
2452 * with a very low granularity.
2453 *
2454 * Return: Maximum deferment in nanoseconds.
 
2455 */
2456u64 scheduler_tick_max_deferment(void)
 
 
 
2457{
2458	struct rq *rq = this_rq();
2459	unsigned long next, now = ACCESS_ONCE(jiffies);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2460
2461	next = rq->last_sched_tick + HZ;
2462
2463	if (time_before_eq(next, now))
2464		return 0;
 
 
 
 
 
 
 
2465
2466	return jiffies_to_nsecs(next - now);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2467}
2468#endif
2469
2470notrace unsigned long get_parent_ip(unsigned long addr)
2471{
2472	if (in_lock_functions(addr)) {
2473		addr = CALLER_ADDR2;
2474		if (in_lock_functions(addr))
2475			addr = CALLER_ADDR3;
 
 
 
 
 
 
 
 
 
 
 
2476	}
2477	return addr;
2478}
2479
2480#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
2481				defined(CONFIG_PREEMPT_TRACER))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2482
2483void __kprobes preempt_count_add(int val)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2484{
2485#ifdef CONFIG_DEBUG_PREEMPT
2486	/*
2487	 * Underflow?
2488	 */
2489	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
2490		return;
2491#endif
2492	__preempt_count_add(val);
2493#ifdef CONFIG_DEBUG_PREEMPT
2494	/*
2495	 * Spinlock count overflowing soon?
2496	 */
2497	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
2498				PREEMPT_MASK - 10);
2499#endif
2500	if (preempt_count() == val) {
2501		unsigned long ip = get_parent_ip(CALLER_ADDR1);
2502#ifdef CONFIG_DEBUG_PREEMPT
2503		current->preempt_disable_ip = ip;
2504#endif
2505		trace_preempt_off(CALLER_ADDR0, ip);
2506	}
2507}
2508EXPORT_SYMBOL(preempt_count_add);
 
2509
2510void __kprobes preempt_count_sub(int val)
 
 
 
 
 
 
 
 
 
 
2511{
2512#ifdef CONFIG_DEBUG_PREEMPT
2513	/*
2514	 * Underflow?
2515	 */
2516	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
2517		return;
2518	/*
2519	 * Is the spinlock portion underflowing?
2520	 */
2521	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
2522			!(preempt_count() & PREEMPT_MASK)))
2523		return;
2524#endif
2525
2526	if (preempt_count() == val)
2527		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
2528	__preempt_count_sub(val);
2529}
2530EXPORT_SYMBOL(preempt_count_sub);
 
2531
 
 
 
2532#endif
2533
 
 
 
 
 
 
 
 
 
2534/*
2535 * Print scheduling while atomic bug:
2536 */
2537static noinline void __schedule_bug(struct task_struct *prev)
2538{
 
 
 
2539	if (oops_in_progress)
2540		return;
2541
2542	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
2543		prev->comm, prev->pid, preempt_count());
2544
2545	debug_show_held_locks(prev);
2546	print_modules();
2547	if (irqs_disabled())
2548		print_irqtrace_events(prev);
2549#ifdef CONFIG_DEBUG_PREEMPT
2550	if (in_atomic_preempt_off()) {
2551		pr_err("Preemption disabled at:");
2552		print_ip_sym(current->preempt_disable_ip);
2553		pr_cont("\n");
2554	}
2555#endif
 
 
2556	dump_stack();
2557	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
2558}
2559
2560/*
2561 * Various schedule()-time debugging checks and statistics:
2562 */
2563static inline void schedule_debug(struct task_struct *prev)
2564{
2565	/*
2566	 * Test if we are atomic. Since do_exit() needs to call into
2567	 * schedule() atomically, we ignore that path. Otherwise whine
2568	 * if we are scheduling when we should not.
2569	 */
2570	if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
 
 
 
 
 
 
 
 
 
 
 
 
2571		__schedule_bug(prev);
 
 
2572	rcu_sleep_check();
2573
2574	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
2575
2576	schedstat_inc(this_rq(), sched_count);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2577}
2578
2579/*
2580 * Pick up the highest-prio task:
2581 */
2582static inline struct task_struct *
2583pick_next_task(struct rq *rq, struct task_struct *prev)
2584{
2585	const struct sched_class *class = &fair_sched_class;
2586	struct task_struct *p;
2587
2588	/*
2589	 * Optimization: we know that if all tasks are in
2590	 * the fair class we can call that function directly:
 
 
2591	 */
2592	if (likely(prev->sched_class == class &&
2593		   rq->nr_running == rq->cfs.h_nr_running)) {
2594		p = fair_sched_class.pick_next_task(rq, prev);
 
2595		if (unlikely(p == RETRY_TASK))
2596			goto again;
2597
2598		/* assumes fair_sched_class->next == idle_sched_class */
2599		if (unlikely(!p))
2600			p = idle_sched_class.pick_next_task(rq, prev);
 
 
2601
2602		return p;
2603	}
2604
2605again:
 
 
2606	for_each_class(class) {
2607		p = class->pick_next_task(rq, prev);
2608		if (p) {
2609			if (unlikely(p == RETRY_TASK))
2610				goto again;
2611			return p;
2612		}
2613	}
2614
2615	BUG(); /* the idle class will always have a runnable task */
 
2616}
2617
2618/*
2619 * __schedule() is the main scheduler function.
2620 *
2621 * The main means of driving the scheduler and thus entering this function are:
2622 *
2623 *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
2624 *
2625 *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
2626 *      paths. For example, see arch/x86/entry_64.S.
2627 *
2628 *      To drive preemption between tasks, the scheduler sets the flag in timer
2629 *      interrupt handler scheduler_tick().
2630 *
2631 *   3. Wakeups don't really cause entry into schedule(). They add a
2632 *      task to the run-queue and that's it.
2633 *
2634 *      Now, if the new task added to the run-queue preempts the current
2635 *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
2636 *      called on the nearest possible occasion:
2637 *
2638 *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
2639 *
2640 *         - in syscall or exception context, at the next outmost
2641 *           preempt_enable(). (this might be as soon as the wake_up()'s
2642 *           spin_unlock()!)
2643 *
2644 *         - in IRQ context, return from interrupt-handler to
2645 *           preemptible context
2646 *
2647 *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
2648 *         then at the next:
2649 *
2650 *          - cond_resched() call
2651 *          - explicit schedule() call
2652 *          - return from syscall or exception to user-space
2653 *          - return from interrupt-handler to user-space
 
 
2654 */
2655static void __sched __schedule(void)
2656{
2657	struct task_struct *prev, *next;
2658	unsigned long *switch_count;
 
 
2659	struct rq *rq;
2660	int cpu;
2661
2662need_resched:
2663	preempt_disable();
2664	cpu = smp_processor_id();
2665	rq = cpu_rq(cpu);
2666	rcu_note_context_switch(cpu);
2667	prev = rq->curr;
2668
2669	schedule_debug(prev);
2670
2671	if (sched_feat(HRTICK))
2672		hrtick_clear(rq);
2673
 
 
 
2674	/*
2675	 * Make sure that signal_pending_state()->signal_pending() below
2676	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
2677	 * done by the caller to avoid the race with signal_wake_up().
 
 
 
 
 
 
 
 
 
 
2678	 */
2679	smp_mb__before_spinlock();
2680	raw_spin_lock_irq(&rq->lock);
 
 
 
 
2681
2682	switch_count = &prev->nivcsw;
2683	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
2684		if (unlikely(signal_pending_state(prev->state, prev))) {
 
 
 
 
 
 
 
 
 
2685			prev->state = TASK_RUNNING;
2686		} else {
2687			deactivate_task(rq, prev, DEQUEUE_SLEEP);
2688			prev->on_rq = 0;
 
 
 
 
 
2689
2690			/*
2691			 * If a worker went to sleep, notify and ask workqueue
2692			 * whether it wants to wake up a task to maintain
2693			 * concurrency.
 
 
 
 
 
 
2694			 */
2695			if (prev->flags & PF_WQ_WORKER) {
2696				struct task_struct *to_wakeup;
2697
2698				to_wakeup = wq_worker_sleeping(prev, cpu);
2699				if (to_wakeup)
2700					try_to_wake_up_local(to_wakeup);
2701			}
2702		}
2703		switch_count = &prev->nvcsw;
2704	}
2705
2706	if (prev->on_rq || rq->skip_clock_update < 0)
2707		update_rq_clock(rq);
2708
2709	next = pick_next_task(rq, prev);
2710	clear_tsk_need_resched(prev);
2711	clear_preempt_need_resched();
2712	rq->skip_clock_update = 0;
2713
2714	if (likely(prev != next)) {
2715		rq->nr_switches++;
2716		rq->curr = next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2717		++*switch_count;
2718
2719		context_switch(rq, prev, next); /* unlocks the rq */
2720		/*
2721		 * The context switch have flipped the stack from under us
2722		 * and restored the local variables which were saved when
2723		 * this task called schedule() in the past. prev == current
2724		 * is still correct, but it can be moved to another cpu/rq.
2725		 */
2726		cpu = smp_processor_id();
2727		rq = cpu_rq(cpu);
2728	} else
2729		raw_spin_unlock_irq(&rq->lock);
2730
2731	post_schedule(rq);
2732
2733	sched_preempt_enable_no_resched();
2734	if (need_resched())
2735		goto need_resched;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2736}
2737
2738static inline void sched_submit_work(struct task_struct *tsk)
2739{
2740	if (!tsk->state || tsk_is_pi_blocked(tsk))
2741		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2742	/*
2743	 * If we are going to sleep and we have plugged IO queued,
2744	 * make sure to submit it to avoid deadlocks.
2745	 */
2746	if (blk_needs_flush_plug(tsk))
2747		blk_schedule_flush_plug(tsk);
2748}
2749
 
 
 
 
 
 
 
 
 
 
2750asmlinkage __visible void __sched schedule(void)
2751{
2752	struct task_struct *tsk = current;
2753
2754	sched_submit_work(tsk);
2755	__schedule();
 
 
 
 
 
2756}
2757EXPORT_SYMBOL(schedule);
2758
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2759#ifdef CONFIG_CONTEXT_TRACKING
2760asmlinkage __visible void __sched schedule_user(void)
2761{
2762	/*
2763	 * If we come here after a random call to set_need_resched(),
2764	 * or we have been woken up remotely but the IPI has not yet arrived,
2765	 * we haven't yet exited the RCU idle mode. Do it here manually until
2766	 * we find a better solution.
 
 
 
 
2767	 */
2768	user_exit();
2769	schedule();
2770	user_enter();
2771}
2772#endif
2773
2774/**
2775 * schedule_preempt_disabled - called with preemption disabled
2776 *
2777 * Returns with preemption disabled. Note: preempt_count must be 1
2778 */
2779void __sched schedule_preempt_disabled(void)
2780{
2781	sched_preempt_enable_no_resched();
2782	schedule();
2783	preempt_disable();
2784}
2785
2786#ifdef CONFIG_PREEMPT
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2787/*
2788 * this is the entry point to schedule() from in-kernel preemption
2789 * off of preempt_enable. Kernel preemptions off return from interrupt
2790 * occur there and call schedule directly.
2791 */
2792asmlinkage __visible void __sched notrace preempt_schedule(void)
2793{
2794	/*
2795	 * If there is a non-zero preempt_count or interrupts are disabled,
2796	 * we do not want to preempt the current task. Just return..
2797	 */
2798	if (likely(!preemptible()))
2799		return;
2800
2801	do {
2802		__preempt_count_add(PREEMPT_ACTIVE);
2803		__schedule();
2804		__preempt_count_sub(PREEMPT_ACTIVE);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2805
 
 
 
 
2806		/*
2807		 * Check again in case we missed a preemption opportunity
2808		 * between schedule and now.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2809		 */
2810		barrier();
 
 
 
 
 
2811	} while (need_resched());
2812}
2813EXPORT_SYMBOL(preempt_schedule);
2814#endif /* CONFIG_PREEMPT */
 
2815
2816/*
2817 * this is the entry point to schedule() from kernel preemption
2818 * off of irq context.
2819 * Note, that this is called and return with irqs disabled. This will
2820 * protect us against recursive calling from irq.
2821 */
2822asmlinkage __visible void __sched preempt_schedule_irq(void)
2823{
2824	enum ctx_state prev_state;
2825
2826	/* Catch callers which need to be fixed */
2827	BUG_ON(preempt_count() || !irqs_disabled());
2828
2829	prev_state = exception_enter();
2830
2831	do {
2832		__preempt_count_add(PREEMPT_ACTIVE);
2833		local_irq_enable();
2834		__schedule();
2835		local_irq_disable();
2836		__preempt_count_sub(PREEMPT_ACTIVE);
2837
2838		/*
2839		 * Check again in case we missed a preemption opportunity
2840		 * between schedule and now.
2841		 */
2842		barrier();
2843	} while (need_resched());
2844
2845	exception_exit(prev_state);
2846}
2847
2848int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
2849			  void *key)
2850{
 
2851	return try_to_wake_up(curr->private, mode, wake_flags);
2852}
2853EXPORT_SYMBOL(default_wake_function);
2854
2855#ifdef CONFIG_RT_MUTEXES
2856
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2857/*
2858 * rt_mutex_setprio - set the current priority of a task
2859 * @p: task
2860 * @prio: prio value (kernel-internal form)
2861 *
2862 * This function changes the 'effective' priority of a task. It does
2863 * not touch ->normal_prio like __setscheduler().
2864 *
2865 * Used by the rt_mutex code to implement priority inheritance
2866 * logic. Call site only calls if the priority of the task changed.
2867 */
2868void rt_mutex_setprio(struct task_struct *p, int prio)
2869{
2870	int oldprio, on_rq, running, enqueue_flag = 0;
2871	struct rq *rq;
2872	const struct sched_class *prev_class;
 
 
2873
2874	BUG_ON(prio > MAX_PRIO);
 
2875
2876	rq = __task_rq_lock(p);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2877
2878	/*
2879	 * Idle task boosting is a nono in general. There is one
2880	 * exception, when PREEMPT_RT and NOHZ is active:
2881	 *
2882	 * The idle task calls get_next_timer_interrupt() and holds
2883	 * the timer wheel base->lock on the CPU and another CPU wants
2884	 * to access the timer (probably to cancel it). We can safely
2885	 * ignore the boosting request, as the idle CPU runs this code
2886	 * with interrupts disabled and will complete the lock
2887	 * protected section without being interrupted. So there is no
2888	 * real need to boost.
2889	 */
2890	if (unlikely(p == rq->idle)) {
2891		WARN_ON(p != rq->curr);
2892		WARN_ON(p->pi_blocked_on);
2893		goto out_unlock;
2894	}
2895
2896	trace_sched_pi_setprio(p, prio);
2897	p->pi_top_task = rt_mutex_get_top_task(p);
2898	oldprio = p->prio;
 
 
 
 
2899	prev_class = p->sched_class;
2900	on_rq = p->on_rq;
2901	running = task_current(rq, p);
2902	if (on_rq)
2903		dequeue_task(rq, p, 0);
2904	if (running)
2905		p->sched_class->put_prev_task(rq, p);
2906
2907	/*
2908	 * Boosting condition are:
2909	 * 1. -rt task is running and holds mutex A
2910	 *      --> -dl task blocks on mutex A
2911	 *
2912	 * 2. -dl task is running and holds mutex A
2913	 *      --> -dl task blocks on mutex A and could preempt the
2914	 *          running task
2915	 */
2916	if (dl_prio(prio)) {
2917		if (!dl_prio(p->normal_prio) || (p->pi_top_task &&
2918			dl_entity_preempt(&p->pi_top_task->dl, &p->dl))) {
 
2919			p->dl.dl_boosted = 1;
2920			p->dl.dl_throttled = 0;
2921			enqueue_flag = ENQUEUE_REPLENISH;
2922		} else
2923			p->dl.dl_boosted = 0;
2924		p->sched_class = &dl_sched_class;
2925	} else if (rt_prio(prio)) {
2926		if (dl_prio(oldprio))
2927			p->dl.dl_boosted = 0;
2928		if (oldprio < prio)
2929			enqueue_flag = ENQUEUE_HEAD;
2930		p->sched_class = &rt_sched_class;
2931	} else {
2932		if (dl_prio(oldprio))
2933			p->dl.dl_boosted = 0;
 
 
2934		p->sched_class = &fair_sched_class;
2935	}
2936
2937	p->prio = prio;
2938
 
 
2939	if (running)
2940		p->sched_class->set_curr_task(rq);
2941	if (on_rq)
2942		enqueue_task(rq, p, enqueue_flag);
2943
2944	check_class_changed(rq, p, prev_class, oldprio);
2945out_unlock:
2946	__task_rq_unlock(rq);
 
 
 
 
 
 
 
 
 
 
2947}
2948#endif
2949
2950void set_user_nice(struct task_struct *p, long nice)
2951{
2952	int old_prio, delta, on_rq;
2953	unsigned long flags;
 
2954	struct rq *rq;
2955
2956	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
2957		return;
2958	/*
2959	 * We have to be careful, if called from sys_setpriority(),
2960	 * the task might be in the middle of scheduling on another CPU.
2961	 */
2962	rq = task_rq_lock(p, &flags);
 
 
2963	/*
2964	 * The RT priorities are set via sched_setscheduler(), but we still
2965	 * allow the 'normal' nice value to be set - but as expected
2966	 * it wont have any effect on scheduling until the task is
2967	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
2968	 */
2969	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
2970		p->static_prio = NICE_TO_PRIO(nice);
2971		goto out_unlock;
2972	}
2973	on_rq = p->on_rq;
2974	if (on_rq)
2975		dequeue_task(rq, p, 0);
 
 
 
2976
2977	p->static_prio = NICE_TO_PRIO(nice);
2978	set_load_weight(p);
2979	old_prio = p->prio;
2980	p->prio = effective_prio(p);
2981	delta = p->prio - old_prio;
2982
2983	if (on_rq) {
2984		enqueue_task(rq, p, 0);
2985		/*
2986		 * If the task increased its priority or is running and
2987		 * lowered its priority, then reschedule its CPU:
2988		 */
2989		if (delta < 0 || (delta > 0 && task_running(rq, p)))
2990			resched_task(rq->curr);
2991	}
 
 
2992out_unlock:
2993	task_rq_unlock(rq, p, &flags);
2994}
2995EXPORT_SYMBOL(set_user_nice);
2996
2997/*
2998 * can_nice - check if a task can reduce its nice value
2999 * @p: task
3000 * @nice: nice value
3001 */
3002int can_nice(const struct task_struct *p, const int nice)
3003{
3004	/* convert nice value [19,-20] to rlimit style value [1,40] */
3005	int nice_rlim = 20 - nice;
3006
3007	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
3008		capable(CAP_SYS_NICE));
3009}
3010
3011#ifdef __ARCH_WANT_SYS_NICE
3012
3013/*
3014 * sys_nice - change the priority of the current process.
3015 * @increment: priority increment
3016 *
3017 * sys_setpriority is a more generic, but much slower function that
3018 * does similar things.
3019 */
3020SYSCALL_DEFINE1(nice, int, increment)
3021{
3022	long nice, retval;
3023
3024	/*
3025	 * Setpriority might change our priority at the same moment.
3026	 * We don't have to worry. Conceptually one call occurs first
3027	 * and we have a single winner.
3028	 */
3029	if (increment < -40)
3030		increment = -40;
3031	if (increment > 40)
3032		increment = 40;
3033
3034	nice = task_nice(current) + increment;
3035	if (nice < MIN_NICE)
3036		nice = MIN_NICE;
3037	if (nice > MAX_NICE)
3038		nice = MAX_NICE;
3039
 
3040	if (increment < 0 && !can_nice(current, nice))
3041		return -EPERM;
3042
3043	retval = security_task_setnice(current, nice);
3044	if (retval)
3045		return retval;
3046
3047	set_user_nice(current, nice);
3048	return 0;
3049}
3050
3051#endif
3052
3053/**
3054 * task_prio - return the priority value of a given task.
3055 * @p: the task in question.
3056 *
3057 * Return: The priority value as seen by users in /proc.
3058 * RT tasks are offset by -200. Normal tasks are centered
3059 * around 0, value goes from -16 to +15.
3060 */
3061int task_prio(const struct task_struct *p)
3062{
3063	return p->prio - MAX_RT_PRIO;
3064}
3065
3066/**
3067 * idle_cpu - is a given cpu idle currently?
3068 * @cpu: the processor in question.
3069 *
3070 * Return: 1 if the CPU is currently idle. 0 otherwise.
3071 */
3072int idle_cpu(int cpu)
3073{
3074	struct rq *rq = cpu_rq(cpu);
3075
3076	if (rq->curr != rq->idle)
3077		return 0;
3078
3079	if (rq->nr_running)
3080		return 0;
3081
3082#ifdef CONFIG_SMP
3083	if (!llist_empty(&rq->wake_list))
3084		return 0;
3085#endif
3086
3087	return 1;
3088}
3089
3090/**
3091 * idle_task - return the idle task for a given cpu.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3092 * @cpu: the processor in question.
3093 *
3094 * Return: The idle task for the cpu @cpu.
3095 */
3096struct task_struct *idle_task(int cpu)
3097{
3098	return cpu_rq(cpu)->idle;
3099}
3100
3101/**
3102 * find_process_by_pid - find a process with a matching PID value.
3103 * @pid: the pid in question.
3104 *
3105 * The task of @pid, if found. %NULL otherwise.
3106 */
3107static struct task_struct *find_process_by_pid(pid_t pid)
3108{
3109	return pid ? find_task_by_vpid(pid) : current;
3110}
3111
3112/*
3113 * This function initializes the sched_dl_entity of a newly becoming
3114 * SCHED_DEADLINE task.
3115 *
3116 * Only the static values are considered here, the actual runtime and the
3117 * absolute deadline will be properly calculated when the task is enqueued
3118 * for the first time with its new policy.
3119 */
3120static void
3121__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
3122{
3123	struct sched_dl_entity *dl_se = &p->dl;
3124
3125	init_dl_task_timer(dl_se);
3126	dl_se->dl_runtime = attr->sched_runtime;
3127	dl_se->dl_deadline = attr->sched_deadline;
3128	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
3129	dl_se->flags = attr->sched_flags;
3130	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
3131	dl_se->dl_throttled = 0;
3132	dl_se->dl_new = 1;
3133	dl_se->dl_yielded = 0;
3134}
3135
3136static void __setscheduler_params(struct task_struct *p,
3137		const struct sched_attr *attr)
3138{
3139	int policy = attr->sched_policy;
3140
3141	if (policy == -1) /* setparam */
3142		policy = p->policy;
3143
3144	p->policy = policy;
3145
3146	if (dl_policy(policy))
3147		__setparam_dl(p, attr);
3148	else if (fair_policy(policy))
3149		p->static_prio = NICE_TO_PRIO(attr->sched_nice);
3150
3151	/*
3152	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
3153	 * !rt_policy. Always setting this ensures that things like
3154	 * getparam()/getattr() don't report silly values for !rt tasks.
3155	 */
3156	p->rt_priority = attr->sched_priority;
3157	p->normal_prio = normal_prio(p);
3158	set_load_weight(p);
3159}
3160
3161/* Actually do priority change: must hold pi & rq lock. */
3162static void __setscheduler(struct rq *rq, struct task_struct *p,
3163			   const struct sched_attr *attr)
3164{
 
 
 
 
 
 
 
3165	__setscheduler_params(p, attr);
3166
3167	/*
3168	 * If we get here, there was no pi waiters boosting the
3169	 * task. It is safe to use the normal prio.
3170	 */
3171	p->prio = normal_prio(p);
 
 
3172
3173	if (dl_prio(p->prio))
3174		p->sched_class = &dl_sched_class;
3175	else if (rt_prio(p->prio))
3176		p->sched_class = &rt_sched_class;
3177	else
3178		p->sched_class = &fair_sched_class;
3179}
3180
3181static void
3182__getparam_dl(struct task_struct *p, struct sched_attr *attr)
3183{
3184	struct sched_dl_entity *dl_se = &p->dl;
3185
3186	attr->sched_priority = p->rt_priority;
3187	attr->sched_runtime = dl_se->dl_runtime;
3188	attr->sched_deadline = dl_se->dl_deadline;
3189	attr->sched_period = dl_se->dl_period;
3190	attr->sched_flags = dl_se->flags;
3191}
3192
3193/*
3194 * This function validates the new parameters of a -deadline task.
3195 * We ask for the deadline not being zero, and greater or equal
3196 * than the runtime, as well as the period of being zero or
3197 * greater than deadline. Furthermore, we have to be sure that
3198 * user parameters are above the internal resolution of 1us (we
3199 * check sched_runtime only since it is always the smaller one) and
3200 * below 2^63 ns (we have to check both sched_deadline and
3201 * sched_period, as the latter can be zero).
3202 */
3203static bool
3204__checkparam_dl(const struct sched_attr *attr)
3205{
3206	/* deadline != 0 */
3207	if (attr->sched_deadline == 0)
3208		return false;
3209
3210	/*
3211	 * Since we truncate DL_SCALE bits, make sure we're at least
3212	 * that big.
3213	 */
3214	if (attr->sched_runtime < (1ULL << DL_SCALE))
3215		return false;
3216
3217	/*
3218	 * Since we use the MSB for wrap-around and sign issues, make
3219	 * sure it's not set (mind that period can be equal to zero).
3220	 */
3221	if (attr->sched_deadline & (1ULL << 63) ||
3222	    attr->sched_period & (1ULL << 63))
3223		return false;
3224
3225	/* runtime <= deadline <= period (if period != 0) */
3226	if ((attr->sched_period != 0 &&
3227	     attr->sched_period < attr->sched_deadline) ||
3228	    attr->sched_deadline < attr->sched_runtime)
3229		return false;
3230
3231	return true;
3232}
3233
3234/*
3235 * check the target process has a UID that matches the current process's
3236 */
3237static bool check_same_owner(struct task_struct *p)
3238{
3239	const struct cred *cred = current_cred(), *pcred;
3240	bool match;
3241
3242	rcu_read_lock();
3243	pcred = __task_cred(p);
3244	match = (uid_eq(cred->euid, pcred->euid) ||
3245		 uid_eq(cred->euid, pcred->uid));
3246	rcu_read_unlock();
3247	return match;
3248}
3249
3250static int __sched_setscheduler(struct task_struct *p,
3251				const struct sched_attr *attr,
3252				bool user)
3253{
3254	int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
3255		      MAX_RT_PRIO - 1 - attr->sched_priority;
3256	int retval, oldprio, oldpolicy = -1, on_rq, running;
3257	int policy = attr->sched_policy;
3258	unsigned long flags;
3259	const struct sched_class *prev_class;
3260	struct rq *rq;
3261	int reset_on_fork;
 
 
3262
3263	/* may grab non-irq protected spin_locks */
3264	BUG_ON(in_interrupt());
3265recheck:
3266	/* double check policy once rq lock held */
3267	if (policy < 0) {
3268		reset_on_fork = p->sched_reset_on_fork;
3269		policy = oldpolicy = p->policy;
3270	} else {
3271		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
3272
3273		if (policy != SCHED_DEADLINE &&
3274				policy != SCHED_FIFO && policy != SCHED_RR &&
3275				policy != SCHED_NORMAL && policy != SCHED_BATCH &&
3276				policy != SCHED_IDLE)
3277			return -EINVAL;
3278	}
3279
3280	if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
3281		return -EINVAL;
3282
3283	/*
3284	 * Valid priorities for SCHED_FIFO and SCHED_RR are
3285	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
3286	 * SCHED_BATCH and SCHED_IDLE is 0.
3287	 */
3288	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
3289	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
3290		return -EINVAL;
3291	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
3292	    (rt_policy(policy) != (attr->sched_priority != 0)))
3293		return -EINVAL;
3294
3295	/*
3296	 * Allow unprivileged RT tasks to decrease priority:
3297	 */
3298	if (user && !capable(CAP_SYS_NICE)) {
3299		if (fair_policy(policy)) {
3300			if (attr->sched_nice < task_nice(p) &&
3301			    !can_nice(p, attr->sched_nice))
3302				return -EPERM;
3303		}
3304
3305		if (rt_policy(policy)) {
3306			unsigned long rlim_rtprio =
3307					task_rlimit(p, RLIMIT_RTPRIO);
3308
3309			/* can't set/change the rt policy */
3310			if (policy != p->policy && !rlim_rtprio)
3311				return -EPERM;
3312
3313			/* can't increase priority */
3314			if (attr->sched_priority > p->rt_priority &&
3315			    attr->sched_priority > rlim_rtprio)
3316				return -EPERM;
3317		}
3318
3319		 /*
3320		  * Can't set/change SCHED_DEADLINE policy at all for now
3321		  * (safest behavior); in the future we would like to allow
3322		  * unprivileged DL tasks to increase their relative deadline
3323		  * or reduce their runtime (both ways reducing utilization)
3324		  */
3325		if (dl_policy(policy))
3326			return -EPERM;
3327
3328		/*
3329		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
3330		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
3331		 */
3332		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
3333			if (!can_nice(p, task_nice(p)))
3334				return -EPERM;
3335		}
3336
3337		/* can't change other user's priorities */
3338		if (!check_same_owner(p))
3339			return -EPERM;
3340
3341		/* Normal users shall not reset the sched_reset_on_fork flag */
3342		if (p->sched_reset_on_fork && !reset_on_fork)
3343			return -EPERM;
3344	}
3345
3346	if (user) {
 
 
 
3347		retval = security_task_setscheduler(p);
3348		if (retval)
3349			return retval;
3350	}
3351
 
 
 
 
 
 
 
 
 
 
3352	/*
3353	 * make sure no PI-waiters arrive (or leave) while we are
3354	 * changing the priority of the task:
3355	 *
3356	 * To be able to change p->policy safely, the appropriate
3357	 * runqueue lock must be held.
3358	 */
3359	rq = task_rq_lock(p, &flags);
 
3360
3361	/*
3362	 * Changing the policy of the stop threads its a very bad idea
3363	 */
3364	if (p == rq->stop) {
3365		task_rq_unlock(rq, p, &flags);
3366		return -EINVAL;
3367	}
3368
3369	/*
3370	 * If not changing anything there's no need to proceed further,
3371	 * but store a possible modification of reset_on_fork.
3372	 */
3373	if (unlikely(policy == p->policy)) {
3374		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
3375			goto change;
3376		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
3377			goto change;
3378		if (dl_policy(policy))
 
 
3379			goto change;
3380
3381		p->sched_reset_on_fork = reset_on_fork;
3382		task_rq_unlock(rq, p, &flags);
3383		return 0;
3384	}
3385change:
3386
3387	if (user) {
3388#ifdef CONFIG_RT_GROUP_SCHED
3389		/*
3390		 * Do not allow realtime tasks into groups that have no runtime
3391		 * assigned.
3392		 */
3393		if (rt_bandwidth_enabled() && rt_policy(policy) &&
3394				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
3395				!task_group_is_autogroup(task_group(p))) {
3396			task_rq_unlock(rq, p, &flags);
3397			return -EPERM;
3398		}
3399#endif
3400#ifdef CONFIG_SMP
3401		if (dl_bandwidth_enabled() && dl_policy(policy)) {
 
3402			cpumask_t *span = rq->rd->span;
3403
3404			/*
3405			 * Don't allow tasks with an affinity mask smaller than
3406			 * the entire root_domain to become SCHED_DEADLINE. We
3407			 * will also fail if there's no bandwidth available.
3408			 */
3409			if (!cpumask_subset(span, &p->cpus_allowed) ||
3410			    rq->rd->dl_bw.bw == 0) {
3411				task_rq_unlock(rq, p, &flags);
3412				return -EPERM;
3413			}
3414		}
3415#endif
3416	}
3417
3418	/* recheck policy now with rq lock held */
3419	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
3420		policy = oldpolicy = -1;
3421		task_rq_unlock(rq, p, &flags);
 
 
3422		goto recheck;
3423	}
3424
3425	/*
3426	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
3427	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
3428	 * is available.
3429	 */
3430	if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
3431		task_rq_unlock(rq, p, &flags);
3432		return -EBUSY;
3433	}
3434
3435	p->sched_reset_on_fork = reset_on_fork;
3436	oldprio = p->prio;
3437
3438	/*
3439	 * Special case for priority boosted tasks.
3440	 *
3441	 * If the new priority is lower or equal (user space view)
3442	 * than the current (boosted) priority, we just store the new
3443	 * normal parameters and do not touch the scheduler class and
3444	 * the runqueue. This will be done when the task deboost
3445	 * itself.
3446	 */
3447	if (rt_mutex_check_prio(p, newprio)) {
3448		__setscheduler_params(p, attr);
3449		task_rq_unlock(rq, p, &flags);
3450		return 0;
3451	}
3452
3453	on_rq = p->on_rq;
3454	running = task_current(rq, p);
3455	if (on_rq)
3456		dequeue_task(rq, p, 0);
3457	if (running)
3458		p->sched_class->put_prev_task(rq, p);
3459
3460	prev_class = p->sched_class;
3461	__setscheduler(rq, p, attr);
3462
3463	if (running)
3464		p->sched_class->set_curr_task(rq);
3465	if (on_rq) {
 
3466		/*
3467		 * We enqueue to tail when the priority of a task is
3468		 * increased (user space view).
3469		 */
3470		enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
 
 
 
3471	}
 
 
3472
3473	check_class_changed(rq, p, prev_class, oldprio);
3474	task_rq_unlock(rq, p, &flags);
3475
3476	rt_mutex_adjust_pi(p);
 
 
 
 
 
 
 
 
 
 
 
3477
3478	return 0;
 
 
 
 
 
 
3479}
3480
3481static int _sched_setscheduler(struct task_struct *p, int policy,
3482			       const struct sched_param *param, bool check)
3483{
3484	struct sched_attr attr = {
3485		.sched_policy   = policy,
3486		.sched_priority = param->sched_priority,
3487		.sched_nice	= PRIO_TO_NICE(p->static_prio),
3488	};
3489
3490	/*
3491	 * Fixup the legacy SCHED_RESET_ON_FORK hack
3492	 */
3493	if (policy & SCHED_RESET_ON_FORK) {
3494		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3495		policy &= ~SCHED_RESET_ON_FORK;
3496		attr.sched_policy = policy;
3497	}
3498
3499	return __sched_setscheduler(p, &attr, check);
3500}
3501/**
3502 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
3503 * @p: the task in question.
3504 * @policy: new policy.
3505 * @param: structure containing the new RT priority.
3506 *
 
 
3507 * Return: 0 on success. An error code otherwise.
3508 *
3509 * NOTE that the task may be already dead.
3510 */
3511int sched_setscheduler(struct task_struct *p, int policy,
3512		       const struct sched_param *param)
3513{
3514	return _sched_setscheduler(p, policy, param, true);
3515}
3516EXPORT_SYMBOL_GPL(sched_setscheduler);
3517
3518int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
3519{
3520	return __sched_setscheduler(p, attr, true);
 
 
 
 
 
3521}
3522EXPORT_SYMBOL_GPL(sched_setattr);
3523
3524/**
3525 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
3526 * @p: the task in question.
3527 * @policy: new policy.
3528 * @param: structure containing the new RT priority.
3529 *
3530 * Just like sched_setscheduler, only don't bother checking if the
3531 * current context has permission.  For example, this is needed in
3532 * stop_machine(): we create temporary high priority worker threads,
3533 * but our caller might not have that capability.
3534 *
3535 * Return: 0 on success. An error code otherwise.
3536 */
3537int sched_setscheduler_nocheck(struct task_struct *p, int policy,
3538			       const struct sched_param *param)
3539{
3540	return _sched_setscheduler(p, policy, param, false);
3541}
3542
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3543static int
3544do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
3545{
3546	struct sched_param lparam;
3547	struct task_struct *p;
3548	int retval;
3549
3550	if (!param || pid < 0)
3551		return -EINVAL;
3552	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
3553		return -EFAULT;
3554
3555	rcu_read_lock();
3556	retval = -ESRCH;
3557	p = find_process_by_pid(pid);
3558	if (p != NULL)
3559		retval = sched_setscheduler(p, policy, &lparam);
3560	rcu_read_unlock();
3561
 
 
 
 
 
3562	return retval;
3563}
3564
3565/*
3566 * Mimics kernel/events/core.c perf_copy_attr().
3567 */
3568static int sched_copy_attr(struct sched_attr __user *uattr,
3569			   struct sched_attr *attr)
3570{
3571	u32 size;
3572	int ret;
3573
3574	if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
3575		return -EFAULT;
3576
3577	/*
3578	 * zero the full structure, so that a short copy will be nice.
3579	 */
3580	memset(attr, 0, sizeof(*attr));
3581
3582	ret = get_user(size, &uattr->size);
3583	if (ret)
3584		return ret;
3585
3586	if (size > PAGE_SIZE)	/* silly large */
3587		goto err_size;
3588
3589	if (!size)		/* abi compat */
3590		size = SCHED_ATTR_SIZE_VER0;
3591
3592	if (size < SCHED_ATTR_SIZE_VER0)
3593		goto err_size;
3594
3595	/*
3596	 * If we're handed a bigger struct than we know of,
3597	 * ensure all the unknown bits are 0 - i.e. new
3598	 * user-space does not rely on any kernel feature
3599	 * extensions we dont know about yet.
3600	 */
3601	if (size > sizeof(*attr)) {
3602		unsigned char __user *addr;
3603		unsigned char __user *end;
3604		unsigned char val;
3605
3606		addr = (void __user *)uattr + sizeof(*attr);
3607		end  = (void __user *)uattr + size;
3608
3609		for (; addr < end; addr++) {
3610			ret = get_user(val, addr);
3611			if (ret)
3612				return ret;
3613			if (val)
3614				goto err_size;
3615		}
3616		size = sizeof(*attr);
3617	}
3618
3619	ret = copy_from_user(attr, uattr, size);
3620	if (ret)
3621		return -EFAULT;
3622
3623	/*
3624	 * XXX: do we want to be lenient like existing syscalls; or do we want
3625	 * to be strict and return an error on out-of-bounds values?
3626	 */
3627	attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
3628
3629out:
3630	return ret;
3631
3632err_size:
3633	put_user(sizeof(*attr), &uattr->size);
3634	ret = -E2BIG;
3635	goto out;
3636}
3637
3638/**
3639 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
3640 * @pid: the pid in question.
3641 * @policy: new policy.
3642 * @param: structure containing the new RT priority.
3643 *
3644 * Return: 0 on success. An error code otherwise.
3645 */
3646SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
3647		struct sched_param __user *, param)
3648{
3649	/* negative values for policy are not valid */
3650	if (policy < 0)
3651		return -EINVAL;
3652
3653	return do_sched_setscheduler(pid, policy, param);
3654}
3655
3656/**
3657 * sys_sched_setparam - set/change the RT priority of a thread
3658 * @pid: the pid in question.
3659 * @param: structure containing the new RT priority.
3660 *
3661 * Return: 0 on success. An error code otherwise.
3662 */
3663SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
3664{
3665	return do_sched_setscheduler(pid, -1, param);
3666}
3667
3668/**
3669 * sys_sched_setattr - same as above, but with extended sched_attr
3670 * @pid: the pid in question.
3671 * @uattr: structure containing the extended parameters.
3672 * @flags: for future extension.
3673 */
3674SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
3675			       unsigned int, flags)
3676{
3677	struct sched_attr attr;
3678	struct task_struct *p;
3679	int retval;
3680
3681	if (!uattr || pid < 0 || flags)
3682		return -EINVAL;
3683
3684	retval = sched_copy_attr(uattr, &attr);
3685	if (retval)
3686		return retval;
3687
3688	if ((int)attr.sched_policy < 0)
3689		return -EINVAL;
 
 
3690
3691	rcu_read_lock();
3692	retval = -ESRCH;
3693	p = find_process_by_pid(pid);
3694	if (p != NULL)
3695		retval = sched_setattr(p, &attr);
3696	rcu_read_unlock();
3697
 
 
 
 
 
3698	return retval;
3699}
3700
3701/**
3702 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
3703 * @pid: the pid in question.
3704 *
3705 * Return: On success, the policy of the thread. Otherwise, a negative error
3706 * code.
3707 */
3708SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
3709{
3710	struct task_struct *p;
3711	int retval;
3712
3713	if (pid < 0)
3714		return -EINVAL;
3715
3716	retval = -ESRCH;
3717	rcu_read_lock();
3718	p = find_process_by_pid(pid);
3719	if (p) {
3720		retval = security_task_getscheduler(p);
3721		if (!retval)
3722			retval = p->policy
3723				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
3724	}
3725	rcu_read_unlock();
3726	return retval;
3727}
3728
3729/**
3730 * sys_sched_getparam - get the RT priority of a thread
3731 * @pid: the pid in question.
3732 * @param: structure containing the RT priority.
3733 *
3734 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
3735 * code.
3736 */
3737SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
3738{
3739	struct sched_param lp = { .sched_priority = 0 };
3740	struct task_struct *p;
3741	int retval;
3742
3743	if (!param || pid < 0)
3744		return -EINVAL;
3745
3746	rcu_read_lock();
3747	p = find_process_by_pid(pid);
3748	retval = -ESRCH;
3749	if (!p)
3750		goto out_unlock;
3751
3752	retval = security_task_getscheduler(p);
3753	if (retval)
3754		goto out_unlock;
3755
3756	if (task_has_rt_policy(p))
3757		lp.sched_priority = p->rt_priority;
3758	rcu_read_unlock();
3759
3760	/*
3761	 * This one might sleep, we cannot do it with a spinlock held ...
3762	 */
3763	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
3764
3765	return retval;
3766
3767out_unlock:
3768	rcu_read_unlock();
3769	return retval;
3770}
3771
3772static int sched_read_attr(struct sched_attr __user *uattr,
3773			   struct sched_attr *attr,
3774			   unsigned int usize)
 
 
 
 
 
 
 
 
 
3775{
3776	int ret;
3777
3778	if (!access_ok(VERIFY_WRITE, uattr, usize))
3779		return -EFAULT;
3780
3781	/*
3782	 * If we're handed a smaller struct than we know of,
3783	 * ensure all the unknown bits are 0 - i.e. old
3784	 * user-space does not get uncomplete information.
3785	 */
3786	if (usize < sizeof(*attr)) {
3787		unsigned char *addr;
3788		unsigned char *end;
3789
3790		addr = (void *)attr + usize;
3791		end  = (void *)attr + sizeof(*attr);
3792
3793		for (; addr < end; addr++) {
3794			if (*addr)
3795				goto err_size;
3796		}
3797
3798		attr->size = usize;
3799	}
3800
3801	ret = copy_to_user(uattr, attr, attr->size);
3802	if (ret)
3803		return -EFAULT;
3804
3805out:
3806	return ret;
3807
3808err_size:
3809	ret = -E2BIG;
3810	goto out;
3811}
3812
3813/**
3814 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
3815 * @pid: the pid in question.
3816 * @uattr: structure containing the extended parameters.
3817 * @size: sizeof(attr) for fwd/bwd comp.
3818 * @flags: for future extension.
3819 */
3820SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
3821		unsigned int, size, unsigned int, flags)
3822{
3823	struct sched_attr attr = {
3824		.size = sizeof(struct sched_attr),
3825	};
3826	struct task_struct *p;
3827	int retval;
3828
3829	if (!uattr || pid < 0 || size > PAGE_SIZE ||
3830	    size < SCHED_ATTR_SIZE_VER0 || flags)
3831		return -EINVAL;
3832
3833	rcu_read_lock();
3834	p = find_process_by_pid(pid);
3835	retval = -ESRCH;
3836	if (!p)
3837		goto out_unlock;
3838
3839	retval = security_task_getscheduler(p);
3840	if (retval)
3841		goto out_unlock;
3842
3843	attr.sched_policy = p->policy;
3844	if (p->sched_reset_on_fork)
3845		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3846	if (task_has_dl_policy(p))
3847		__getparam_dl(p, &attr);
3848	else if (task_has_rt_policy(p))
3849		attr.sched_priority = p->rt_priority;
3850	else
3851		attr.sched_nice = task_nice(p);
 
 
 
 
 
 
 
 
 
 
3852
3853	rcu_read_unlock();
3854
3855	retval = sched_read_attr(uattr, &attr, size);
3856	return retval;
3857
3858out_unlock:
3859	rcu_read_unlock();
3860	return retval;
3861}
3862
3863long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
3864{
3865	cpumask_var_t cpus_allowed, new_mask;
3866	struct task_struct *p;
3867	int retval;
3868
3869	rcu_read_lock();
3870
3871	p = find_process_by_pid(pid);
3872	if (!p) {
3873		rcu_read_unlock();
3874		return -ESRCH;
3875	}
3876
3877	/* Prevent p going away */
3878	get_task_struct(p);
3879	rcu_read_unlock();
3880
3881	if (p->flags & PF_NO_SETAFFINITY) {
3882		retval = -EINVAL;
3883		goto out_put_task;
3884	}
3885	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
3886		retval = -ENOMEM;
3887		goto out_put_task;
3888	}
3889	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
3890		retval = -ENOMEM;
3891		goto out_free_cpus_allowed;
3892	}
3893	retval = -EPERM;
3894	if (!check_same_owner(p)) {
3895		rcu_read_lock();
3896		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
3897			rcu_read_unlock();
3898			goto out_unlock;
3899		}
3900		rcu_read_unlock();
3901	}
3902
3903	retval = security_task_setscheduler(p);
3904	if (retval)
3905		goto out_unlock;
3906
3907
3908	cpuset_cpus_allowed(p, cpus_allowed);
3909	cpumask_and(new_mask, in_mask, cpus_allowed);
3910
3911	/*
3912	 * Since bandwidth control happens on root_domain basis,
3913	 * if admission test is enabled, we only admit -deadline
3914	 * tasks allowed to run on all the CPUs in the task's
3915	 * root_domain.
3916	 */
3917#ifdef CONFIG_SMP
3918	if (task_has_dl_policy(p)) {
3919		const struct cpumask *span = task_rq(p)->rd->span;
3920
3921		if (dl_bandwidth_enabled() && !cpumask_subset(span, new_mask)) {
3922			retval = -EBUSY;
3923			goto out_unlock;
 
3924		}
 
3925	}
3926#endif
3927again:
3928	retval = set_cpus_allowed_ptr(p, new_mask);
3929
3930	if (!retval) {
3931		cpuset_cpus_allowed(p, cpus_allowed);
3932		if (!cpumask_subset(new_mask, cpus_allowed)) {
3933			/*
3934			 * We must have raced with a concurrent cpuset
3935			 * update. Just reset the cpus_allowed to the
3936			 * cpuset's cpus_allowed
3937			 */
3938			cpumask_copy(new_mask, cpus_allowed);
3939			goto again;
3940		}
3941	}
3942out_unlock:
3943	free_cpumask_var(new_mask);
3944out_free_cpus_allowed:
3945	free_cpumask_var(cpus_allowed);
3946out_put_task:
3947	put_task_struct(p);
3948	return retval;
3949}
3950
3951static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
3952			     struct cpumask *new_mask)
3953{
3954	if (len < cpumask_size())
3955		cpumask_clear(new_mask);
3956	else if (len > cpumask_size())
3957		len = cpumask_size();
3958
3959	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
3960}
3961
3962/**
3963 * sys_sched_setaffinity - set the cpu affinity of a process
3964 * @pid: pid of the process
3965 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
3966 * @user_mask_ptr: user-space pointer to the new cpu mask
3967 *
3968 * Return: 0 on success. An error code otherwise.
3969 */
3970SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
3971		unsigned long __user *, user_mask_ptr)
3972{
3973	cpumask_var_t new_mask;
3974	int retval;
3975
3976	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
3977		return -ENOMEM;
3978
3979	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
3980	if (retval == 0)
3981		retval = sched_setaffinity(pid, new_mask);
3982	free_cpumask_var(new_mask);
3983	return retval;
3984}
3985
3986long sched_getaffinity(pid_t pid, struct cpumask *mask)
3987{
3988	struct task_struct *p;
3989	unsigned long flags;
3990	int retval;
3991
3992	rcu_read_lock();
3993
3994	retval = -ESRCH;
3995	p = find_process_by_pid(pid);
3996	if (!p)
3997		goto out_unlock;
3998
3999	retval = security_task_getscheduler(p);
4000	if (retval)
4001		goto out_unlock;
4002
4003	raw_spin_lock_irqsave(&p->pi_lock, flags);
4004	cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
4005	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4006
4007out_unlock:
4008	rcu_read_unlock();
4009
4010	return retval;
4011}
4012
4013/**
4014 * sys_sched_getaffinity - get the cpu affinity of a process
4015 * @pid: pid of the process
4016 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4017 * @user_mask_ptr: user-space pointer to hold the current cpu mask
4018 *
4019 * Return: 0 on success. An error code otherwise.
 
4020 */
4021SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4022		unsigned long __user *, user_mask_ptr)
4023{
4024	int ret;
4025	cpumask_var_t mask;
4026
4027	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
4028		return -EINVAL;
4029	if (len & (sizeof(unsigned long)-1))
4030		return -EINVAL;
4031
4032	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4033		return -ENOMEM;
4034
4035	ret = sched_getaffinity(pid, mask);
4036	if (ret == 0) {
4037		size_t retlen = min_t(size_t, len, cpumask_size());
4038
4039		if (copy_to_user(user_mask_ptr, mask, retlen))
4040			ret = -EFAULT;
4041		else
4042			ret = retlen;
4043	}
4044	free_cpumask_var(mask);
4045
4046	return ret;
4047}
4048
4049/**
4050 * sys_sched_yield - yield the current processor to other threads.
4051 *
4052 * This function yields the current CPU to other tasks. If there are no
4053 * other threads running on this CPU then this function will return.
4054 *
4055 * Return: 0.
4056 */
4057SYSCALL_DEFINE0(sched_yield)
4058{
4059	struct rq *rq = this_rq_lock();
 
4060
4061	schedstat_inc(rq, yld_count);
 
 
4062	current->sched_class->yield_task(rq);
4063
4064	/*
4065	 * Since we are going to call schedule() anyway, there's
4066	 * no need to preempt or enable interrupts:
4067	 */
4068	__release(rq->lock);
4069	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4070	do_raw_spin_unlock(&rq->lock);
4071	sched_preempt_enable_no_resched();
4072
4073	schedule();
4074
4075	return 0;
4076}
4077
4078static void __cond_resched(void)
4079{
4080	__preempt_count_add(PREEMPT_ACTIVE);
4081	__schedule();
4082	__preempt_count_sub(PREEMPT_ACTIVE);
4083}
4084
 
4085int __sched _cond_resched(void)
4086{
4087	if (should_resched()) {
4088		__cond_resched();
4089		return 1;
4090	}
 
4091	return 0;
4092}
4093EXPORT_SYMBOL(_cond_resched);
 
4094
4095/*
4096 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
4097 * call schedule, and on return reacquire the lock.
4098 *
4099 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
4100 * operations here to prevent schedule() from being called twice (once via
4101 * spin_unlock(), once by hand).
4102 */
4103int __cond_resched_lock(spinlock_t *lock)
4104{
4105	int resched = should_resched();
4106	int ret = 0;
4107
4108	lockdep_assert_held(lock);
4109
4110	if (spin_needbreak(lock) || resched) {
4111		spin_unlock(lock);
4112		if (resched)
4113			__cond_resched();
4114		else
4115			cpu_relax();
4116		ret = 1;
4117		spin_lock(lock);
4118	}
4119	return ret;
4120}
4121EXPORT_SYMBOL(__cond_resched_lock);
4122
4123int __sched __cond_resched_softirq(void)
4124{
4125	BUG_ON(!in_softirq());
4126
4127	if (should_resched()) {
4128		local_bh_enable();
4129		__cond_resched();
4130		local_bh_disable();
4131		return 1;
4132	}
4133	return 0;
4134}
4135EXPORT_SYMBOL(__cond_resched_softirq);
4136
4137/**
4138 * yield - yield the current processor to other threads.
4139 *
4140 * Do not ever use this function, there's a 99% chance you're doing it wrong.
4141 *
4142 * The scheduler is at all times free to pick the calling task as the most
4143 * eligible task to run, if removing the yield() call from your code breaks
4144 * it, its already broken.
4145 *
4146 * Typical broken usage is:
4147 *
4148 * while (!event)
4149 * 	yield();
4150 *
4151 * where one assumes that yield() will let 'the other' process run that will
4152 * make event true. If the current task is a SCHED_FIFO task that will never
4153 * happen. Never use yield() as a progress guarantee!!
4154 *
4155 * If you want to use yield() to wait for something, use wait_event().
4156 * If you want to use yield() to be 'nice' for others, use cond_resched().
4157 * If you still want to use yield(), do not!
4158 */
4159void __sched yield(void)
4160{
4161	set_current_state(TASK_RUNNING);
4162	sys_sched_yield();
4163}
4164EXPORT_SYMBOL(yield);
4165
4166/**
4167 * yield_to - yield the current processor to another thread in
4168 * your thread group, or accelerate that thread toward the
4169 * processor it's on.
4170 * @p: target task
4171 * @preempt: whether task preemption is allowed or not
4172 *
4173 * It's the caller's job to ensure that the target task struct
4174 * can't go away on us before we can do any checks.
4175 *
4176 * Return:
4177 *	true (>0) if we indeed boosted the target task.
4178 *	false (0) if we failed to boost the target.
4179 *	-ESRCH if there's no task to yield to.
4180 */
4181bool __sched yield_to(struct task_struct *p, bool preempt)
4182{
4183	struct task_struct *curr = current;
4184	struct rq *rq, *p_rq;
4185	unsigned long flags;
4186	int yielded = 0;
4187
4188	local_irq_save(flags);
4189	rq = this_rq();
4190
4191again:
4192	p_rq = task_rq(p);
4193	/*
4194	 * If we're the only runnable task on the rq and target rq also
4195	 * has only one task, there's absolutely no point in yielding.
4196	 */
4197	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
4198		yielded = -ESRCH;
4199		goto out_irq;
4200	}
4201
4202	double_rq_lock(rq, p_rq);
4203	if (task_rq(p) != p_rq) {
4204		double_rq_unlock(rq, p_rq);
4205		goto again;
4206	}
4207
4208	if (!curr->sched_class->yield_to_task)
4209		goto out_unlock;
4210
4211	if (curr->sched_class != p->sched_class)
4212		goto out_unlock;
4213
4214	if (task_running(p_rq, p) || p->state)
4215		goto out_unlock;
4216
4217	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
4218	if (yielded) {
4219		schedstat_inc(rq, yld_count);
4220		/*
4221		 * Make p's CPU reschedule; pick_next_entity takes care of
4222		 * fairness.
4223		 */
4224		if (preempt && rq != p_rq)
4225			resched_task(p_rq->curr);
4226	}
4227
4228out_unlock:
4229	double_rq_unlock(rq, p_rq);
4230out_irq:
4231	local_irq_restore(flags);
4232
4233	if (yielded > 0)
4234		schedule();
4235
4236	return yielded;
4237}
4238EXPORT_SYMBOL_GPL(yield_to);
4239
4240/*
4241 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
4242 * that process accounting knows that this is a task in IO wait state.
4243 */
4244void __sched io_schedule(void)
4245{
4246	struct rq *rq = raw_rq();
4247
4248	delayacct_blkio_start();
4249	atomic_inc(&rq->nr_iowait);
4250	blk_flush_plug(current);
4251	current->in_iowait = 1;
4252	schedule();
4253	current->in_iowait = 0;
4254	atomic_dec(&rq->nr_iowait);
4255	delayacct_blkio_end();
4256}
4257EXPORT_SYMBOL(io_schedule);
4258
 
 
 
 
 
 
 
 
 
4259long __sched io_schedule_timeout(long timeout)
4260{
4261	struct rq *rq = raw_rq();
4262	long ret;
4263
4264	delayacct_blkio_start();
4265	atomic_inc(&rq->nr_iowait);
4266	blk_flush_plug(current);
4267	current->in_iowait = 1;
4268	ret = schedule_timeout(timeout);
4269	current->in_iowait = 0;
4270	atomic_dec(&rq->nr_iowait);
4271	delayacct_blkio_end();
4272	return ret;
4273}
 
 
 
 
 
 
 
 
 
 
 
4274
4275/**
4276 * sys_sched_get_priority_max - return maximum RT priority.
4277 * @policy: scheduling class.
4278 *
4279 * Return: On success, this syscall returns the maximum
4280 * rt_priority that can be used by a given scheduling class.
4281 * On failure, a negative error code is returned.
4282 */
4283SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
4284{
4285	int ret = -EINVAL;
4286
4287	switch (policy) {
4288	case SCHED_FIFO:
4289	case SCHED_RR:
4290		ret = MAX_USER_RT_PRIO-1;
4291		break;
4292	case SCHED_DEADLINE:
4293	case SCHED_NORMAL:
4294	case SCHED_BATCH:
4295	case SCHED_IDLE:
4296		ret = 0;
4297		break;
4298	}
4299	return ret;
4300}
4301
4302/**
4303 * sys_sched_get_priority_min - return minimum RT priority.
4304 * @policy: scheduling class.
4305 *
4306 * Return: On success, this syscall returns the minimum
4307 * rt_priority that can be used by a given scheduling class.
4308 * On failure, a negative error code is returned.
4309 */
4310SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
4311{
4312	int ret = -EINVAL;
4313
4314	switch (policy) {
4315	case SCHED_FIFO:
4316	case SCHED_RR:
4317		ret = 1;
4318		break;
4319	case SCHED_DEADLINE:
4320	case SCHED_NORMAL:
4321	case SCHED_BATCH:
4322	case SCHED_IDLE:
4323		ret = 0;
4324	}
4325	return ret;
4326}
4327
4328/**
4329 * sys_sched_rr_get_interval - return the default timeslice of a process.
4330 * @pid: pid of the process.
4331 * @interval: userspace pointer to the timeslice value.
4332 *
4333 * this syscall writes the default timeslice value of a given process
4334 * into the user-space timespec buffer. A value of '0' means infinity.
4335 *
4336 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
4337 * an error code.
4338 */
4339SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
4340		struct timespec __user *, interval)
4341{
4342	struct task_struct *p;
4343	unsigned int time_slice;
4344	unsigned long flags;
4345	struct rq *rq;
4346	int retval;
4347	struct timespec t;
4348
4349	if (pid < 0)
4350		return -EINVAL;
4351
4352	retval = -ESRCH;
4353	rcu_read_lock();
4354	p = find_process_by_pid(pid);
4355	if (!p)
4356		goto out_unlock;
4357
4358	retval = security_task_getscheduler(p);
4359	if (retval)
4360		goto out_unlock;
4361
4362	rq = task_rq_lock(p, &flags);
4363	time_slice = 0;
4364	if (p->sched_class->get_rr_interval)
4365		time_slice = p->sched_class->get_rr_interval(rq, p);
4366	task_rq_unlock(rq, p, &flags);
4367
4368	rcu_read_unlock();
4369	jiffies_to_timespec(time_slice, &t);
4370	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
4371	return retval;
4372
4373out_unlock:
4374	rcu_read_unlock();
4375	return retval;
4376}
4377
4378static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4379
4380void sched_show_task(struct task_struct *p)
4381{
4382	unsigned long free = 0;
4383	int ppid;
4384	unsigned state;
4385
4386	state = p->state ? __ffs(p->state) + 1 : 0;
4387	printk(KERN_INFO "%-15.15s %c", p->comm,
4388		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4389#if BITS_PER_LONG == 32
4390	if (state == TASK_RUNNING)
4391		printk(KERN_CONT " running  ");
4392	else
4393		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
4394#else
4395	if (state == TASK_RUNNING)
4396		printk(KERN_CONT "  running task    ");
4397	else
4398		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
4399#endif
4400#ifdef CONFIG_DEBUG_STACK_USAGE
4401	free = stack_not_used(p);
4402#endif
 
4403	rcu_read_lock();
4404	ppid = task_pid_nr(rcu_dereference(p->real_parent));
 
4405	rcu_read_unlock();
4406	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4407		task_pid_nr(p), ppid,
4408		(unsigned long)task_thread_info(p)->flags);
4409
4410	print_worker_info(KERN_INFO, p);
4411	show_stack(p, NULL);
 
4412}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4413
4414void show_state_filter(unsigned long state_filter)
4415{
4416	struct task_struct *g, *p;
4417
4418#if BITS_PER_LONG == 32
4419	printk(KERN_INFO
4420		"  task                PC stack   pid father\n");
4421#else
4422	printk(KERN_INFO
4423		"  task                        PC stack   pid father\n");
4424#endif
4425	rcu_read_lock();
4426	do_each_thread(g, p) {
4427		/*
4428		 * reset the NMI-timeout, listing all files on a slow
4429		 * console might take a lot of time:
 
 
 
4430		 */
4431		touch_nmi_watchdog();
4432		if (!state_filter || (p->state & state_filter))
 
4433			sched_show_task(p);
4434	} while_each_thread(g, p);
4435
4436	touch_all_softlockup_watchdogs();
4437
4438#ifdef CONFIG_SCHED_DEBUG
4439	sysrq_sched_debug_show();
 
4440#endif
4441	rcu_read_unlock();
4442	/*
4443	 * Only show locks if all tasks are dumped:
4444	 */
4445	if (!state_filter)
4446		debug_show_all_locks();
4447}
4448
4449void init_idle_bootup_task(struct task_struct *idle)
4450{
4451	idle->sched_class = &idle_sched_class;
4452}
4453
4454/**
4455 * init_idle - set up an idle thread for a given CPU
4456 * @idle: task in question
4457 * @cpu: cpu the idle task belongs to
4458 *
4459 * NOTE: this function does not set the idle thread's NEED_RESCHED
4460 * flag, to make booting more robust.
4461 */
4462void init_idle(struct task_struct *idle, int cpu)
4463{
4464	struct rq *rq = cpu_rq(cpu);
4465	unsigned long flags;
4466
4467	raw_spin_lock_irqsave(&rq->lock, flags);
4468
4469	__sched_fork(0, idle);
 
 
 
 
4470	idle->state = TASK_RUNNING;
4471	idle->se.exec_start = sched_clock();
 
 
 
 
4472
4473	do_set_cpus_allowed(idle, cpumask_of(cpu));
 
 
 
 
 
 
 
 
4474	/*
4475	 * We're having a chicken and egg problem, even though we are
4476	 * holding rq->lock, the cpu isn't yet set to this cpu so the
4477	 * lockdep check in task_group() will fail.
4478	 *
4479	 * Similar case to sched_fork(). / Alternatively we could
4480	 * use task_rq_lock() here and obtain the other rq->lock.
4481	 *
4482	 * Silence PROVE_RCU
4483	 */
4484	rcu_read_lock();
4485	__set_task_cpu(idle, cpu);
4486	rcu_read_unlock();
4487
4488	rq->curr = rq->idle = idle;
4489	idle->on_rq = 1;
4490#if defined(CONFIG_SMP)
 
4491	idle->on_cpu = 1;
4492#endif
4493	raw_spin_unlock_irqrestore(&rq->lock, flags);
 
4494
4495	/* Set the preempt count _outside_ the spinlocks! */
4496	init_idle_preempt_count(idle, cpu);
4497
4498	/*
4499	 * The idle tasks have their own, simple scheduling class:
4500	 */
4501	idle->sched_class = &idle_sched_class;
4502	ftrace_graph_init_idle_task(idle, cpu);
4503	vtime_init_idle(idle, cpu);
4504#if defined(CONFIG_SMP)
4505	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
4506#endif
4507}
4508
4509#ifdef CONFIG_SMP
4510void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
4511{
4512	if (p->sched_class && p->sched_class->set_cpus_allowed)
4513		p->sched_class->set_cpus_allowed(p, new_mask);
4514
4515	cpumask_copy(&p->cpus_allowed, new_mask);
4516	p->nr_cpus_allowed = cpumask_weight(new_mask);
4517}
4518
4519/*
4520 * This is how migration works:
4521 *
4522 * 1) we invoke migration_cpu_stop() on the target CPU using
4523 *    stop_one_cpu().
4524 * 2) stopper starts to run (implicitly forcing the migrated thread
4525 *    off the CPU)
4526 * 3) it checks whether the migrated task is still in the wrong runqueue.
4527 * 4) if it's in the wrong runqueue then the migration thread removes
4528 *    it and puts it into the right queue.
4529 * 5) stopper completes and stop_one_cpu() returns and the migration
4530 *    is done.
4531 */
4532
4533/*
4534 * Change a given task's CPU affinity. Migrate the thread to a
4535 * proper CPU and schedule it away if the CPU it's executing on
4536 * is removed from the allowed bitmask.
4537 *
4538 * NOTE: the caller must have a valid reference to the task, the
4539 * task must not exit() & deallocate itself prematurely. The
4540 * call is not atomic; no spinlocks may be held.
4541 */
4542int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
4543{
4544	unsigned long flags;
4545	struct rq *rq;
4546	unsigned int dest_cpu;
4547	int ret = 0;
4548
4549	rq = task_rq_lock(p, &flags);
4550
4551	if (cpumask_equal(&p->cpus_allowed, new_mask))
4552		goto out;
4553
4554	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
4555		ret = -EINVAL;
4556		goto out;
4557	}
4558
4559	do_set_cpus_allowed(p, new_mask);
4560
4561	/* Can the task run on the task's current CPU? If so, we're done */
4562	if (cpumask_test_cpu(task_cpu(p), new_mask))
4563		goto out;
4564
4565	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
4566	if (p->on_rq) {
4567		struct migration_arg arg = { p, dest_cpu };
4568		/* Need help from migration thread: drop lock and wait. */
4569		task_rq_unlock(rq, p, &flags);
4570		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
4571		tlb_migrate_finish(p->mm);
4572		return 0;
4573	}
4574out:
4575	task_rq_unlock(rq, p, &flags);
4576
4577	return ret;
4578}
4579EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
4580
4581/*
4582 * Move (not current) task off this cpu, onto dest cpu. We're doing
4583 * this because either it can't run here any more (set_cpus_allowed()
4584 * away from this CPU, or CPU going down), or because we're
4585 * attempting to rebalance this task on exec (sched_exec).
4586 *
4587 * So we race with normal scheduler movements, but that's OK, as long
4588 * as the task is no longer on this CPU.
4589 *
4590 * Returns non-zero if task was successfully migrated.
4591 */
4592static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
4593{
4594	struct rq *rq_dest, *rq_src;
4595	int ret = 0;
4596
4597	if (unlikely(!cpu_active(dest_cpu)))
4598		return ret;
4599
4600	rq_src = cpu_rq(src_cpu);
4601	rq_dest = cpu_rq(dest_cpu);
4602
4603	raw_spin_lock(&p->pi_lock);
4604	double_rq_lock(rq_src, rq_dest);
4605	/* Already moved. */
4606	if (task_cpu(p) != src_cpu)
4607		goto done;
4608	/* Affinity changed (again). */
4609	if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
4610		goto fail;
4611
4612	/*
4613	 * If we're not on a rq, the next wake-up will ensure we're
4614	 * placed properly.
 
 
 
 
 
4615	 */
4616	if (p->on_rq) {
4617		dequeue_task(rq_src, p, 0);
4618		set_task_cpu(p, dest_cpu);
4619		enqueue_task(rq_dest, p, 0);
4620		check_preempt_curr(rq_dest, p, 0);
4621	}
4622done:
4623	ret = 1;
4624fail:
4625	double_rq_unlock(rq_src, rq_dest);
4626	raw_spin_unlock(&p->pi_lock);
 
4627	return ret;
4628}
4629
 
 
4630#ifdef CONFIG_NUMA_BALANCING
4631/* Migrate current task p to target_cpu */
4632int migrate_task_to(struct task_struct *p, int target_cpu)
4633{
4634	struct migration_arg arg = { p, target_cpu };
4635	int curr_cpu = task_cpu(p);
4636
4637	if (curr_cpu == target_cpu)
4638		return 0;
4639
4640	if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
4641		return -EINVAL;
4642
4643	/* TODO: This is not properly updating schedstats */
4644
4645	trace_sched_move_numa(p, curr_cpu, target_cpu);
4646	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
4647}
4648
4649/*
4650 * Requeue a task on a given node and accurately track the number of NUMA
4651 * tasks on the runqueues
4652 */
4653void sched_setnuma(struct task_struct *p, int nid)
4654{
 
 
4655	struct rq *rq;
4656	unsigned long flags;
4657	bool on_rq, running;
4658
4659	rq = task_rq_lock(p, &flags);
4660	on_rq = p->on_rq;
4661	running = task_current(rq, p);
4662
4663	if (on_rq)
4664		dequeue_task(rq, p, 0);
4665	if (running)
4666		p->sched_class->put_prev_task(rq, p);
4667
4668	p->numa_preferred_nid = nid;
4669
 
 
4670	if (running)
4671		p->sched_class->set_curr_task(rq);
4672	if (on_rq)
4673		enqueue_task(rq, p, 0);
4674	task_rq_unlock(rq, p, &flags);
4675}
4676#endif
4677
4678/*
4679 * migration_cpu_stop - this will be executed by a highprio stopper thread
4680 * and performs thread migration by bumping thread off CPU then
4681 * 'pushing' onto another runqueue.
4682 */
4683static int migration_cpu_stop(void *data)
4684{
4685	struct migration_arg *arg = data;
4686
4687	/*
4688	 * The original target cpu might have gone down and we might
4689	 * be on another cpu but it doesn't matter.
4690	 */
4691	local_irq_disable();
4692	__migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
4693	local_irq_enable();
4694	return 0;
4695}
 
4696
4697#ifdef CONFIG_HOTPLUG_CPU
4698
4699/*
4700 * Ensures that the idle task is using init_mm right before its cpu goes
4701 * offline.
4702 */
4703void idle_task_exit(void)
4704{
4705	struct mm_struct *mm = current->active_mm;
4706
4707	BUG_ON(cpu_online(smp_processor_id()));
 
4708
4709	if (mm != &init_mm) {
4710		switch_mm(mm, &init_mm, current);
4711		finish_arch_post_lock_switch();
4712	}
4713	mmdrop(mm);
 
4714}
4715
4716/*
4717 * Since this CPU is going 'away' for a while, fold any nr_active delta
4718 * we might have. Assumes we're called after migrate_tasks() so that the
4719 * nr_active count is stable.
 
 
4720 *
4721 * Also see the comment "Global load-average calculations".
4722 */
4723static void calc_load_migrate(struct rq *rq)
4724{
4725	long delta = calc_load_fold_active(rq);
4726	if (delta)
4727		atomic_long_add(delta, &calc_load_tasks);
4728}
4729
4730static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
4731{
4732}
 
4733
4734static const struct sched_class fake_sched_class = {
4735	.put_prev_task = put_prev_task_fake,
4736};
 
 
 
 
4737
4738static struct task_struct fake_task = {
4739	/*
4740	 * Avoid pull_{rt,dl}_task()
4741	 */
4742	.prio = MAX_PRIO + 1,
4743	.sched_class = &fake_sched_class,
4744};
4745
4746/*
4747 * Migrate all tasks from the rq, sleeping tasks will be migrated by
4748 * try_to_wake_up()->select_task_rq().
4749 *
4750 * Called with rq->lock held even though we'er in stop_machine() and
4751 * there's no concurrency possible, we hold the required locks anyway
4752 * because of lock validation efforts.
4753 */
4754static void migrate_tasks(unsigned int dead_cpu)
4755{
4756	struct rq *rq = cpu_rq(dead_cpu);
4757	struct task_struct *next, *stop = rq->stop;
 
4758	int dest_cpu;
4759
4760	/*
4761	 * Fudge the rq selection such that the below task selection loop
4762	 * doesn't get stuck on the currently eligible stop task.
4763	 *
4764	 * We're currently inside stop_machine() and the rq is either stuck
4765	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
4766	 * either way we should never end up calling schedule() until we're
4767	 * done here.
4768	 */
4769	rq->stop = NULL;
4770
4771	/*
4772	 * put_prev_task() and pick_next_task() sched
4773	 * class method both need to have an up-to-date
4774	 * value of rq->clock[_task]
4775	 */
4776	update_rq_clock(rq);
4777
4778	for ( ; ; ) {
4779		/*
4780		 * There's this thread running, bail when that's the only
4781		 * remaining thread.
4782		 */
4783		if (rq->nr_running == 1)
4784			break;
4785
4786		next = pick_next_task(rq, &fake_task);
4787		BUG_ON(!next);
4788		next->sched_class->put_prev_task(rq, next);
4789
4790		/* Find suitable destination for @next, with force if needed. */
4791		dest_cpu = select_fallback_rq(dead_cpu, next);
4792		raw_spin_unlock(&rq->lock);
 
 
 
 
 
 
 
 
 
4793
4794		__migrate_task(next, dead_cpu, dest_cpu);
 
 
 
 
 
 
 
 
4795
4796		raw_spin_lock(&rq->lock);
 
 
 
 
 
 
 
 
 
4797	}
4798
4799	rq->stop = stop;
4800}
4801
4802#endif /* CONFIG_HOTPLUG_CPU */
4803
4804#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
4805
4806static struct ctl_table sd_ctl_dir[] = {
4807	{
4808		.procname	= "sched_domain",
4809		.mode		= 0555,
4810	},
4811	{}
4812};
4813
4814static struct ctl_table sd_ctl_root[] = {
4815	{
4816		.procname	= "kernel",
4817		.mode		= 0555,
4818		.child		= sd_ctl_dir,
4819	},
4820	{}
4821};
4822
4823static struct ctl_table *sd_alloc_ctl_entry(int n)
4824{
4825	struct ctl_table *entry =
4826		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
4827
4828	return entry;
4829}
4830
4831static void sd_free_ctl_entry(struct ctl_table **tablep)
4832{
4833	struct ctl_table *entry;
4834
4835	/*
4836	 * In the intermediate directories, both the child directory and
4837	 * procname are dynamically allocated and could fail but the mode
4838	 * will always be set. In the lowest directory the names are
4839	 * static strings and all have proc handlers.
4840	 */
4841	for (entry = *tablep; entry->mode; entry++) {
4842		if (entry->child)
4843			sd_free_ctl_entry(&entry->child);
4844		if (entry->proc_handler == NULL)
4845			kfree(entry->procname);
4846	}
4847
4848	kfree(*tablep);
4849	*tablep = NULL;
4850}
4851
4852static int min_load_idx = 0;
4853static int max_load_idx = CPU_LOAD_IDX_MAX-1;
4854
4855static void
4856set_table_entry(struct ctl_table *entry,
4857		const char *procname, void *data, int maxlen,
4858		umode_t mode, proc_handler *proc_handler,
4859		bool load_idx)
4860{
4861	entry->procname = procname;
4862	entry->data = data;
4863	entry->maxlen = maxlen;
4864	entry->mode = mode;
4865	entry->proc_handler = proc_handler;
4866
4867	if (load_idx) {
4868		entry->extra1 = &min_load_idx;
4869		entry->extra2 = &max_load_idx;
4870	}
4871}
4872
4873static struct ctl_table *
4874sd_alloc_ctl_domain_table(struct sched_domain *sd)
4875{
4876	struct ctl_table *table = sd_alloc_ctl_entry(14);
4877
4878	if (table == NULL)
4879		return NULL;
4880
4881	set_table_entry(&table[0], "min_interval", &sd->min_interval,
4882		sizeof(long), 0644, proc_doulongvec_minmax, false);
4883	set_table_entry(&table[1], "max_interval", &sd->max_interval,
4884		sizeof(long), 0644, proc_doulongvec_minmax, false);
4885	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
4886		sizeof(int), 0644, proc_dointvec_minmax, true);
4887	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
4888		sizeof(int), 0644, proc_dointvec_minmax, true);
4889	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
4890		sizeof(int), 0644, proc_dointvec_minmax, true);
4891	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
4892		sizeof(int), 0644, proc_dointvec_minmax, true);
4893	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
4894		sizeof(int), 0644, proc_dointvec_minmax, true);
4895	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
4896		sizeof(int), 0644, proc_dointvec_minmax, false);
4897	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
4898		sizeof(int), 0644, proc_dointvec_minmax, false);
4899	set_table_entry(&table[9], "cache_nice_tries",
4900		&sd->cache_nice_tries,
4901		sizeof(int), 0644, proc_dointvec_minmax, false);
4902	set_table_entry(&table[10], "flags", &sd->flags,
4903		sizeof(int), 0644, proc_dointvec_minmax, false);
4904	set_table_entry(&table[11], "max_newidle_lb_cost",
4905		&sd->max_newidle_lb_cost,
4906		sizeof(long), 0644, proc_doulongvec_minmax, false);
4907	set_table_entry(&table[12], "name", sd->name,
4908		CORENAME_MAX_SIZE, 0444, proc_dostring, false);
4909	/* &table[13] is terminator */
4910
4911	return table;
4912}
4913
4914static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
4915{
4916	struct ctl_table *entry, *table;
4917	struct sched_domain *sd;
4918	int domain_num = 0, i;
4919	char buf[32];
4920
4921	for_each_domain(cpu, sd)
4922		domain_num++;
4923	entry = table = sd_alloc_ctl_entry(domain_num + 1);
4924	if (table == NULL)
4925		return NULL;
4926
4927	i = 0;
4928	for_each_domain(cpu, sd) {
4929		snprintf(buf, 32, "domain%d", i);
4930		entry->procname = kstrdup(buf, GFP_KERNEL);
4931		entry->mode = 0555;
4932		entry->child = sd_alloc_ctl_domain_table(sd);
4933		entry++;
4934		i++;
4935	}
4936	return table;
4937}
4938
4939static struct ctl_table_header *sd_sysctl_header;
4940static void register_sched_domain_sysctl(void)
4941{
4942	int i, cpu_num = num_possible_cpus();
4943	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
4944	char buf[32];
4945
4946	WARN_ON(sd_ctl_dir[0].child);
4947	sd_ctl_dir[0].child = entry;
4948
4949	if (entry == NULL)
4950		return;
4951
4952	for_each_possible_cpu(i) {
4953		snprintf(buf, 32, "cpu%d", i);
4954		entry->procname = kstrdup(buf, GFP_KERNEL);
4955		entry->mode = 0555;
4956		entry->child = sd_alloc_ctl_cpu_table(i);
4957		entry++;
4958	}
4959
4960	WARN_ON(sd_sysctl_header);
4961	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
4962}
4963
4964/* may be called multiple times per register */
4965static void unregister_sched_domain_sysctl(void)
4966{
4967	if (sd_sysctl_header)
4968		unregister_sysctl_table(sd_sysctl_header);
4969	sd_sysctl_header = NULL;
4970	if (sd_ctl_dir[0].child)
4971		sd_free_ctl_entry(&sd_ctl_dir[0].child);
4972}
4973#else
4974static void register_sched_domain_sysctl(void)
4975{
4976}
4977static void unregister_sched_domain_sysctl(void)
4978{
4979}
4980#endif
4981
4982static void set_rq_online(struct rq *rq)
4983{
4984	if (!rq->online) {
4985		const struct sched_class *class;
4986
4987		cpumask_set_cpu(rq->cpu, rq->rd->online);
4988		rq->online = 1;
4989
4990		for_each_class(class) {
4991			if (class->rq_online)
4992				class->rq_online(rq);
4993		}
4994	}
4995}
4996
4997static void set_rq_offline(struct rq *rq)
4998{
4999	if (rq->online) {
5000		const struct sched_class *class;
5001
5002		for_each_class(class) {
5003			if (class->rq_offline)
5004				class->rq_offline(rq);
5005		}
5006
5007		cpumask_clear_cpu(rq->cpu, rq->rd->online);
5008		rq->online = 0;
5009	}
5010}
5011
5012/*
5013 * migration_call - callback that gets triggered when a CPU is added.
5014 * Here we can start up the necessary migration thread for the new CPU.
5015 */
5016static int
5017migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
5018{
5019	int cpu = (long)hcpu;
5020	unsigned long flags;
5021	struct rq *rq = cpu_rq(cpu);
5022
5023	switch (action & ~CPU_TASKS_FROZEN) {
5024
5025	case CPU_UP_PREPARE:
5026		rq->calc_load_update = calc_load_update;
5027		break;
5028
5029	case CPU_ONLINE:
5030		/* Update our root-domain */
5031		raw_spin_lock_irqsave(&rq->lock, flags);
5032		if (rq->rd) {
5033			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5034
5035			set_rq_online(rq);
5036		}
5037		raw_spin_unlock_irqrestore(&rq->lock, flags);
5038		break;
5039
5040#ifdef CONFIG_HOTPLUG_CPU
5041	case CPU_DYING:
5042		sched_ttwu_pending();
5043		/* Update our root-domain */
5044		raw_spin_lock_irqsave(&rq->lock, flags);
5045		if (rq->rd) {
5046			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5047			set_rq_offline(rq);
5048		}
5049		migrate_tasks(cpu);
5050		BUG_ON(rq->nr_running != 1); /* the migration thread */
5051		raw_spin_unlock_irqrestore(&rq->lock, flags);
5052		break;
5053
5054	case CPU_DEAD:
5055		calc_load_migrate(rq);
5056		break;
5057#endif
5058	}
5059
5060	update_max_interval();
5061
5062	return NOTIFY_OK;
5063}
5064
5065/*
5066 * Register at high priority so that task migration (migrate_all_tasks)
5067 * happens before everything else.  This has to be lower priority than
5068 * the notifier in the perf_event subsystem, though.
5069 */
5070static struct notifier_block migration_notifier = {
5071	.notifier_call = migration_call,
5072	.priority = CPU_PRI_MIGRATION,
5073};
5074
5075static int sched_cpu_active(struct notifier_block *nfb,
5076				      unsigned long action, void *hcpu)
5077{
5078	switch (action & ~CPU_TASKS_FROZEN) {
5079	case CPU_DOWN_FAILED:
5080		set_cpu_active((long)hcpu, true);
5081		return NOTIFY_OK;
5082	default:
5083		return NOTIFY_DONE;
5084	}
5085}
5086
5087static int sched_cpu_inactive(struct notifier_block *nfb,
5088					unsigned long action, void *hcpu)
5089{
5090	unsigned long flags;
5091	long cpu = (long)hcpu;
5092
5093	switch (action & ~CPU_TASKS_FROZEN) {
5094	case CPU_DOWN_PREPARE:
5095		set_cpu_active(cpu, false);
5096
5097		/* explicitly allow suspend */
5098		if (!(action & CPU_TASKS_FROZEN)) {
5099			struct dl_bw *dl_b = dl_bw_of(cpu);
5100			bool overflow;
5101			int cpus;
5102
5103			raw_spin_lock_irqsave(&dl_b->lock, flags);
5104			cpus = dl_bw_cpus(cpu);
5105			overflow = __dl_overflow(dl_b, cpus, 0, 0);
5106			raw_spin_unlock_irqrestore(&dl_b->lock, flags);
5107
5108			if (overflow)
5109				return notifier_from_errno(-EBUSY);
5110		}
5111		return NOTIFY_OK;
5112	}
5113
5114	return NOTIFY_DONE;
5115}
5116
5117static int __init migration_init(void)
5118{
5119	void *cpu = (void *)(long)smp_processor_id();
5120	int err;
5121
5122	/* Initialize migration for the boot CPU */
5123	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
5124	BUG_ON(err == NOTIFY_BAD);
5125	migration_call(&migration_notifier, CPU_ONLINE, cpu);
5126	register_cpu_notifier(&migration_notifier);
5127
5128	/* Register cpu active notifiers */
5129	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
5130	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
5131
5132	return 0;
5133}
5134early_initcall(migration_init);
5135#endif
5136
5137#ifdef CONFIG_SMP
5138
5139static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
5140
5141#ifdef CONFIG_SCHED_DEBUG
5142
5143static __read_mostly int sched_debug_enabled;
5144
5145static int __init sched_debug_setup(char *str)
5146{
5147	sched_debug_enabled = 1;
5148
5149	return 0;
5150}
5151early_param("sched_debug", sched_debug_setup);
5152
5153static inline bool sched_debug(void)
5154{
5155	return sched_debug_enabled;
5156}
5157
5158static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5159				  struct cpumask *groupmask)
5160{
5161	struct sched_group *group = sd->groups;
5162	char str[256];
5163
5164	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
5165	cpumask_clear(groupmask);
5166
5167	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5168
5169	if (!(sd->flags & SD_LOAD_BALANCE)) {
5170		printk("does not load-balance\n");
5171		if (sd->parent)
5172			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5173					" has parent");
5174		return -1;
5175	}
5176
5177	printk(KERN_CONT "span %s level %s\n", str, sd->name);
5178
5179	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
5180		printk(KERN_ERR "ERROR: domain->span does not contain "
5181				"CPU%d\n", cpu);
5182	}
5183	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
5184		printk(KERN_ERR "ERROR: domain->groups does not contain"
5185				" CPU%d\n", cpu);
5186	}
5187
5188	printk(KERN_DEBUG "%*s groups:", level + 1, "");
5189	do {
5190		if (!group) {
5191			printk("\n");
5192			printk(KERN_ERR "ERROR: group is NULL\n");
5193			break;
5194		}
5195
5196		/*
5197		 * Even though we initialize ->power to something semi-sane,
5198		 * we leave power_orig unset. This allows us to detect if
5199		 * domain iteration is still funny without causing /0 traps.
5200		 */
5201		if (!group->sgp->power_orig) {
5202			printk(KERN_CONT "\n");
5203			printk(KERN_ERR "ERROR: domain->cpu_power not "
5204					"set\n");
5205			break;
5206		}
5207
5208		if (!cpumask_weight(sched_group_cpus(group))) {
5209			printk(KERN_CONT "\n");
5210			printk(KERN_ERR "ERROR: empty group\n");
5211			break;
5212		}
5213
5214		if (!(sd->flags & SD_OVERLAP) &&
5215		    cpumask_intersects(groupmask, sched_group_cpus(group))) {
5216			printk(KERN_CONT "\n");
5217			printk(KERN_ERR "ERROR: repeated CPUs\n");
5218			break;
5219		}
5220
5221		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
5222
5223		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5224
5225		printk(KERN_CONT " %s", str);
5226		if (group->sgp->power != SCHED_POWER_SCALE) {
5227			printk(KERN_CONT " (cpu_power = %d)",
5228				group->sgp->power);
5229		}
5230
5231		group = group->next;
5232	} while (group != sd->groups);
5233	printk(KERN_CONT "\n");
5234
5235	if (!cpumask_equal(sched_domain_span(sd), groupmask))
5236		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
5237
5238	if (sd->parent &&
5239	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
5240		printk(KERN_ERR "ERROR: parent span is not a superset "
5241			"of domain->span\n");
5242	return 0;
5243}
5244
5245static void sched_domain_debug(struct sched_domain *sd, int cpu)
5246{
5247	int level = 0;
5248
5249	if (!sched_debug_enabled)
5250		return;
5251
5252	if (!sd) {
5253		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5254		return;
5255	}
5256
5257	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5258
5259	for (;;) {
5260		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
5261			break;
5262		level++;
5263		sd = sd->parent;
5264		if (!sd)
5265			break;
5266	}
5267}
5268#else /* !CONFIG_SCHED_DEBUG */
5269# define sched_domain_debug(sd, cpu) do { } while (0)
5270static inline bool sched_debug(void)
5271{
5272	return false;
5273}
5274#endif /* CONFIG_SCHED_DEBUG */
5275
5276static int sd_degenerate(struct sched_domain *sd)
5277{
5278	if (cpumask_weight(sched_domain_span(sd)) == 1)
5279		return 1;
5280
5281	/* Following flags need at least 2 groups */
5282	if (sd->flags & (SD_LOAD_BALANCE |
5283			 SD_BALANCE_NEWIDLE |
5284			 SD_BALANCE_FORK |
5285			 SD_BALANCE_EXEC |
5286			 SD_SHARE_CPUPOWER |
5287			 SD_SHARE_PKG_RESOURCES)) {
5288		if (sd->groups != sd->groups->next)
5289			return 0;
5290	}
5291
5292	/* Following flags don't use groups */
5293	if (sd->flags & (SD_WAKE_AFFINE))
5294		return 0;
5295
5296	return 1;
5297}
5298
5299static int
5300sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5301{
5302	unsigned long cflags = sd->flags, pflags = parent->flags;
5303
5304	if (sd_degenerate(parent))
5305		return 1;
5306
5307	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5308		return 0;
5309
5310	/* Flags needing groups don't count if only 1 group in parent */
5311	if (parent->groups == parent->groups->next) {
5312		pflags &= ~(SD_LOAD_BALANCE |
5313				SD_BALANCE_NEWIDLE |
5314				SD_BALANCE_FORK |
5315				SD_BALANCE_EXEC |
5316				SD_SHARE_CPUPOWER |
5317				SD_SHARE_PKG_RESOURCES |
5318				SD_PREFER_SIBLING);
5319		if (nr_node_ids == 1)
5320			pflags &= ~SD_SERIALIZE;
5321	}
5322	if (~cflags & pflags)
5323		return 0;
5324
5325	return 1;
5326}
5327
5328static void free_rootdomain(struct rcu_head *rcu)
5329{
5330	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5331
5332	cpupri_cleanup(&rd->cpupri);
5333	cpudl_cleanup(&rd->cpudl);
5334	free_cpumask_var(rd->dlo_mask);
5335	free_cpumask_var(rd->rto_mask);
5336	free_cpumask_var(rd->online);
5337	free_cpumask_var(rd->span);
5338	kfree(rd);
5339}
5340
5341static void rq_attach_root(struct rq *rq, struct root_domain *rd)
5342{
5343	struct root_domain *old_rd = NULL;
5344	unsigned long flags;
5345
5346	raw_spin_lock_irqsave(&rq->lock, flags);
5347
5348	if (rq->rd) {
5349		old_rd = rq->rd;
5350
5351		if (cpumask_test_cpu(rq->cpu, old_rd->online))
5352			set_rq_offline(rq);
5353
5354		cpumask_clear_cpu(rq->cpu, old_rd->span);
5355
5356		/*
5357		 * If we dont want to free the old_rd yet then
5358		 * set old_rd to NULL to skip the freeing later
5359		 * in this function:
5360		 */
5361		if (!atomic_dec_and_test(&old_rd->refcount))
5362			old_rd = NULL;
5363	}
5364
5365	atomic_inc(&rd->refcount);
5366	rq->rd = rd;
5367
5368	cpumask_set_cpu(rq->cpu, rd->span);
5369	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5370		set_rq_online(rq);
5371
5372	raw_spin_unlock_irqrestore(&rq->lock, flags);
5373
5374	if (old_rd)
5375		call_rcu_sched(&old_rd->rcu, free_rootdomain);
5376}
5377
5378static int init_rootdomain(struct root_domain *rd)
5379{
5380	memset(rd, 0, sizeof(*rd));
5381
5382	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
5383		goto out;
5384	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
5385		goto free_span;
5386	if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
5387		goto free_online;
5388	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5389		goto free_dlo_mask;
5390
5391	init_dl_bw(&rd->dl_bw);
5392	if (cpudl_init(&rd->cpudl) != 0)
5393		goto free_dlo_mask;
5394
5395	if (cpupri_init(&rd->cpupri) != 0)
5396		goto free_rto_mask;
5397	return 0;
5398
5399free_rto_mask:
5400	free_cpumask_var(rd->rto_mask);
5401free_dlo_mask:
5402	free_cpumask_var(rd->dlo_mask);
5403free_online:
5404	free_cpumask_var(rd->online);
5405free_span:
5406	free_cpumask_var(rd->span);
5407out:
5408	return -ENOMEM;
5409}
5410
5411/*
5412 * By default the system creates a single root-domain with all cpus as
5413 * members (mimicking the global state we have today).
5414 */
5415struct root_domain def_root_domain;
5416
5417static void init_defrootdomain(void)
5418{
5419	init_rootdomain(&def_root_domain);
5420
5421	atomic_set(&def_root_domain.refcount, 1);
5422}
5423
5424static struct root_domain *alloc_rootdomain(void)
5425{
5426	struct root_domain *rd;
5427
5428	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
5429	if (!rd)
5430		return NULL;
5431
5432	if (init_rootdomain(rd) != 0) {
5433		kfree(rd);
5434		return NULL;
5435	}
5436
5437	return rd;
5438}
5439
5440static void free_sched_groups(struct sched_group *sg, int free_sgp)
5441{
5442	struct sched_group *tmp, *first;
5443
5444	if (!sg)
5445		return;
5446
5447	first = sg;
5448	do {
5449		tmp = sg->next;
5450
5451		if (free_sgp && atomic_dec_and_test(&sg->sgp->ref))
5452			kfree(sg->sgp);
5453
5454		kfree(sg);
5455		sg = tmp;
5456	} while (sg != first);
5457}
5458
5459static void free_sched_domain(struct rcu_head *rcu)
5460{
5461	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5462
5463	/*
5464	 * If its an overlapping domain it has private groups, iterate and
5465	 * nuke them all.
5466	 */
5467	if (sd->flags & SD_OVERLAP) {
5468		free_sched_groups(sd->groups, 1);
5469	} else if (atomic_dec_and_test(&sd->groups->ref)) {
5470		kfree(sd->groups->sgp);
5471		kfree(sd->groups);
5472	}
5473	kfree(sd);
5474}
5475
5476static void destroy_sched_domain(struct sched_domain *sd, int cpu)
5477{
5478	call_rcu(&sd->rcu, free_sched_domain);
5479}
5480
5481static void destroy_sched_domains(struct sched_domain *sd, int cpu)
5482{
5483	for (; sd; sd = sd->parent)
5484		destroy_sched_domain(sd, cpu);
5485}
5486
5487/*
5488 * Keep a special pointer to the highest sched_domain that has
5489 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
5490 * allows us to avoid some pointer chasing select_idle_sibling().
5491 *
5492 * Also keep a unique ID per domain (we use the first cpu number in
5493 * the cpumask of the domain), this allows us to quickly tell if
5494 * two cpus are in the same cache domain, see cpus_share_cache().
5495 */
5496DEFINE_PER_CPU(struct sched_domain *, sd_llc);
5497DEFINE_PER_CPU(int, sd_llc_size);
5498DEFINE_PER_CPU(int, sd_llc_id);
5499DEFINE_PER_CPU(struct sched_domain *, sd_numa);
5500DEFINE_PER_CPU(struct sched_domain *, sd_busy);
5501DEFINE_PER_CPU(struct sched_domain *, sd_asym);
5502
5503static void update_top_cache_domain(int cpu)
5504{
5505	struct sched_domain *sd;
5506	struct sched_domain *busy_sd = NULL;
5507	int id = cpu;
5508	int size = 1;
5509
5510	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
5511	if (sd) {
5512		id = cpumask_first(sched_domain_span(sd));
5513		size = cpumask_weight(sched_domain_span(sd));
5514		busy_sd = sd->parent; /* sd_busy */
5515	}
5516	rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
5517
5518	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
5519	per_cpu(sd_llc_size, cpu) = size;
5520	per_cpu(sd_llc_id, cpu) = id;
5521
5522	sd = lowest_flag_domain(cpu, SD_NUMA);
5523	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
5524
5525	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
5526	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
5527}
5528
5529/*
5530 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
5531 * hold the hotplug lock.
5532 */
5533static void
5534cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
5535{
5536	struct rq *rq = cpu_rq(cpu);
5537	struct sched_domain *tmp;
5538
5539	/* Remove the sched domains which do not contribute to scheduling. */
5540	for (tmp = sd; tmp; ) {
5541		struct sched_domain *parent = tmp->parent;
5542		if (!parent)
5543			break;
5544
5545		if (sd_parent_degenerate(tmp, parent)) {
5546			tmp->parent = parent->parent;
5547			if (parent->parent)
5548				parent->parent->child = tmp;
5549			/*
5550			 * Transfer SD_PREFER_SIBLING down in case of a
5551			 * degenerate parent; the spans match for this
5552			 * so the property transfers.
5553			 */
5554			if (parent->flags & SD_PREFER_SIBLING)
5555				tmp->flags |= SD_PREFER_SIBLING;
5556			destroy_sched_domain(parent, cpu);
5557		} else
5558			tmp = tmp->parent;
5559	}
5560
5561	if (sd && sd_degenerate(sd)) {
5562		tmp = sd;
5563		sd = sd->parent;
5564		destroy_sched_domain(tmp, cpu);
5565		if (sd)
5566			sd->child = NULL;
5567	}
5568
5569	sched_domain_debug(sd, cpu);
5570
5571	rq_attach_root(rq, rd);
5572	tmp = rq->sd;
5573	rcu_assign_pointer(rq->sd, sd);
5574	destroy_sched_domains(tmp, cpu);
5575
5576	update_top_cache_domain(cpu);
5577}
5578
5579/* cpus with isolated domains */
5580static cpumask_var_t cpu_isolated_map;
5581
5582/* Setup the mask of cpus configured for isolated domains */
5583static int __init isolated_cpu_setup(char *str)
5584{
5585	alloc_bootmem_cpumask_var(&cpu_isolated_map);
5586	cpulist_parse(str, cpu_isolated_map);
5587	return 1;
5588}
5589
5590__setup("isolcpus=", isolated_cpu_setup);
5591
5592static const struct cpumask *cpu_cpu_mask(int cpu)
5593{
5594	return cpumask_of_node(cpu_to_node(cpu));
5595}
5596
5597struct sd_data {
5598	struct sched_domain **__percpu sd;
5599	struct sched_group **__percpu sg;
5600	struct sched_group_power **__percpu sgp;
5601};
5602
5603struct s_data {
5604	struct sched_domain ** __percpu sd;
5605	struct root_domain	*rd;
5606};
5607
5608enum s_alloc {
5609	sa_rootdomain,
5610	sa_sd,
5611	sa_sd_storage,
5612	sa_none,
5613};
5614
5615struct sched_domain_topology_level;
5616
5617typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu);
5618typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
5619
5620#define SDTL_OVERLAP	0x01
5621
5622struct sched_domain_topology_level {
5623	sched_domain_init_f init;
5624	sched_domain_mask_f mask;
5625	int		    flags;
5626	int		    numa_level;
5627	struct sd_data      data;
5628};
5629
5630/*
5631 * Build an iteration mask that can exclude certain CPUs from the upwards
5632 * domain traversal.
5633 *
5634 * Asymmetric node setups can result in situations where the domain tree is of
5635 * unequal depth, make sure to skip domains that already cover the entire
5636 * range.
5637 *
5638 * In that case build_sched_domains() will have terminated the iteration early
5639 * and our sibling sd spans will be empty. Domains should always include the
5640 * cpu they're built on, so check that.
5641 *
 
 
5642 */
5643static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
5644{
5645	const struct cpumask *span = sched_domain_span(sd);
5646	struct sd_data *sdd = sd->private;
5647	struct sched_domain *sibling;
5648	int i;
5649
5650	for_each_cpu(i, span) {
5651		sibling = *per_cpu_ptr(sdd->sd, i);
5652		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
5653			continue;
5654
5655		cpumask_set_cpu(i, sched_group_mask(sg));
5656	}
5657}
5658
5659/*
5660 * Return the canonical balance cpu for this group, this is the first cpu
5661 * of this group that's also in the iteration mask.
5662 */
5663int group_balance_cpu(struct sched_group *sg)
5664{
5665	return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
5666}
5667
5668static int
5669build_overlap_sched_groups(struct sched_domain *sd, int cpu)
5670{
5671	struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
5672	const struct cpumask *span = sched_domain_span(sd);
5673	struct cpumask *covered = sched_domains_tmpmask;
5674	struct sd_data *sdd = sd->private;
5675	struct sched_domain *child;
5676	int i;
5677
5678	cpumask_clear(covered);
5679
5680	for_each_cpu(i, span) {
5681		struct cpumask *sg_span;
5682
5683		if (cpumask_test_cpu(i, covered))
5684			continue;
5685
5686		child = *per_cpu_ptr(sdd->sd, i);
5687
5688		/* See the comment near build_group_mask(). */
5689		if (!cpumask_test_cpu(i, sched_domain_span(child)))
5690			continue;
5691
5692		sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
5693				GFP_KERNEL, cpu_to_node(cpu));
5694
5695		if (!sg)
5696			goto fail;
5697
5698		sg_span = sched_group_cpus(sg);
5699		if (child->child) {
5700			child = child->child;
5701			cpumask_copy(sg_span, sched_domain_span(child));
5702		} else
5703			cpumask_set_cpu(i, sg_span);
5704
5705		cpumask_or(covered, covered, sg_span);
5706
5707		sg->sgp = *per_cpu_ptr(sdd->sgp, i);
5708		if (atomic_inc_return(&sg->sgp->ref) == 1)
5709			build_group_mask(sd, sg);
5710
5711		/*
5712		 * Initialize sgp->power such that even if we mess up the
5713		 * domains and no possible iteration will get us here, we won't
5714		 * die on a /0 trap.
 
5715		 */
5716		sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span);
5717		sg->sgp->power_orig = sg->sgp->power;
5718
5719		/*
5720		 * Make sure the first group of this domain contains the
5721		 * canonical balance cpu. Otherwise the sched_domain iteration
5722		 * breaks. See update_sg_lb_stats().
5723		 */
5724		if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
5725		    group_balance_cpu(sg) == cpu)
5726			groups = sg;
5727
5728		if (!first)
5729			first = sg;
5730		if (last)
5731			last->next = sg;
5732		last = sg;
5733		last->next = first;
5734	}
5735	sd->groups = groups;
5736
5737	return 0;
5738
5739fail:
5740	free_sched_groups(first, 0);
5741
5742	return -ENOMEM;
5743}
5744
5745static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
5746{
5747	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
5748	struct sched_domain *child = sd->child;
5749
5750	if (child)
5751		cpu = cpumask_first(sched_domain_span(child));
5752
5753	if (sg) {
5754		*sg = *per_cpu_ptr(sdd->sg, cpu);
5755		(*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu);
5756		atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */
5757	}
5758
5759	return cpu;
5760}
5761
5762/*
5763 * build_sched_groups will build a circular linked list of the groups
5764 * covered by the given span, and will set each group's ->cpumask correctly,
5765 * and ->cpu_power to 0.
5766 *
5767 * Assumes the sched_domain tree is fully constructed
5768 */
5769static int
5770build_sched_groups(struct sched_domain *sd, int cpu)
5771{
5772	struct sched_group *first = NULL, *last = NULL;
5773	struct sd_data *sdd = sd->private;
5774	const struct cpumask *span = sched_domain_span(sd);
5775	struct cpumask *covered;
5776	int i;
5777
5778	get_group(cpu, sdd, &sd->groups);
5779	atomic_inc(&sd->groups->ref);
5780
5781	if (cpu != cpumask_first(span))
5782		return 0;
5783
5784	lockdep_assert_held(&sched_domains_mutex);
5785	covered = sched_domains_tmpmask;
5786
5787	cpumask_clear(covered);
5788
5789	for_each_cpu(i, span) {
5790		struct sched_group *sg;
5791		int group, j;
5792
5793		if (cpumask_test_cpu(i, covered))
5794			continue;
5795
5796		group = get_group(i, sdd, &sg);
5797		cpumask_clear(sched_group_cpus(sg));
5798		sg->sgp->power = 0;
5799		cpumask_setall(sched_group_mask(sg));
5800
5801		for_each_cpu(j, span) {
5802			if (get_group(j, sdd, NULL) != group)
5803				continue;
5804
5805			cpumask_set_cpu(j, covered);
5806			cpumask_set_cpu(j, sched_group_cpus(sg));
5807		}
5808
5809		if (!first)
5810			first = sg;
5811		if (last)
5812			last->next = sg;
5813		last = sg;
5814	}
5815	last->next = first;
5816
5817	return 0;
5818}
5819
5820/*
5821 * Initialize sched groups cpu_power.
5822 *
5823 * cpu_power indicates the capacity of sched group, which is used while
5824 * distributing the load between different sched groups in a sched domain.
5825 * Typically cpu_power for all the groups in a sched domain will be same unless
5826 * there are asymmetries in the topology. If there are asymmetries, group
5827 * having more cpu_power will pickup more load compared to the group having
5828 * less cpu_power.
5829 */
5830static void init_sched_groups_power(int cpu, struct sched_domain *sd)
5831{
5832	struct sched_group *sg = sd->groups;
5833
5834	WARN_ON(!sg);
5835
5836	do {
5837		sg->group_weight = cpumask_weight(sched_group_cpus(sg));
5838		sg = sg->next;
5839	} while (sg != sd->groups);
5840
5841	if (cpu != group_balance_cpu(sg))
5842		return;
5843
5844	update_group_power(sd, cpu);
5845	atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight);
5846}
5847
5848int __weak arch_sd_sibling_asym_packing(void)
5849{
5850       return 0*SD_ASYM_PACKING;
5851}
5852
5853/*
5854 * Initializers for schedule domains
5855 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
5856 */
5857
5858#ifdef CONFIG_SCHED_DEBUG
5859# define SD_INIT_NAME(sd, type)		sd->name = #type
5860#else
5861# define SD_INIT_NAME(sd, type)		do { } while (0)
5862#endif
5863
5864#define SD_INIT_FUNC(type)						\
5865static noinline struct sched_domain *					\
5866sd_init_##type(struct sched_domain_topology_level *tl, int cpu) 	\
5867{									\
5868	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);	\
5869	*sd = SD_##type##_INIT;						\
5870	SD_INIT_NAME(sd, type);						\
5871	sd->private = &tl->data;					\
5872	return sd;							\
5873}
5874
5875SD_INIT_FUNC(CPU)
5876#ifdef CONFIG_SCHED_SMT
5877 SD_INIT_FUNC(SIBLING)
5878#endif
5879#ifdef CONFIG_SCHED_MC
5880 SD_INIT_FUNC(MC)
5881#endif
5882#ifdef CONFIG_SCHED_BOOK
5883 SD_INIT_FUNC(BOOK)
5884#endif
 
5885
5886static int default_relax_domain_level = -1;
5887int sched_domain_level_max;
5888
5889static int __init setup_relax_domain_level(char *str)
5890{
5891	if (kstrtoint(str, 0, &default_relax_domain_level))
5892		pr_warn("Unable to set relax_domain_level\n");
5893
5894	return 1;
5895}
5896__setup("relax_domain_level=", setup_relax_domain_level);
5897
5898static void set_domain_attribute(struct sched_domain *sd,
5899				 struct sched_domain_attr *attr)
5900{
5901	int request;
5902
5903	if (!attr || attr->relax_domain_level < 0) {
5904		if (default_relax_domain_level < 0)
5905			return;
5906		else
5907			request = default_relax_domain_level;
5908	} else
5909		request = attr->relax_domain_level;
5910	if (request < sd->level) {
5911		/* turn off idle balance on this domain */
5912		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5913	} else {
5914		/* turn on idle balance on this domain */
5915		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5916	}
5917}
5918
5919static void __sdt_free(const struct cpumask *cpu_map);
5920static int __sdt_alloc(const struct cpumask *cpu_map);
5921
5922static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
5923				 const struct cpumask *cpu_map)
5924{
5925	switch (what) {
5926	case sa_rootdomain:
5927		if (!atomic_read(&d->rd->refcount))
5928			free_rootdomain(&d->rd->rcu); /* fall through */
5929	case sa_sd:
5930		free_percpu(d->sd); /* fall through */
5931	case sa_sd_storage:
5932		__sdt_free(cpu_map); /* fall through */
5933	case sa_none:
5934		break;
5935	}
5936}
5937
5938static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
5939						   const struct cpumask *cpu_map)
5940{
5941	memset(d, 0, sizeof(*d));
5942
5943	if (__sdt_alloc(cpu_map))
5944		return sa_sd_storage;
5945	d->sd = alloc_percpu(struct sched_domain *);
5946	if (!d->sd)
5947		return sa_sd_storage;
5948	d->rd = alloc_rootdomain();
5949	if (!d->rd)
5950		return sa_sd;
5951	return sa_rootdomain;
5952}
5953
5954/*
5955 * NULL the sd_data elements we've used to build the sched_domain and
5956 * sched_group structure so that the subsequent __free_domain_allocs()
5957 * will not free the data we're using.
5958 */
5959static void claim_allocations(int cpu, struct sched_domain *sd)
5960{
5961	struct sd_data *sdd = sd->private;
5962
5963	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
5964	*per_cpu_ptr(sdd->sd, cpu) = NULL;
5965
5966	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
5967		*per_cpu_ptr(sdd->sg, cpu) = NULL;
5968
5969	if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref))
5970		*per_cpu_ptr(sdd->sgp, cpu) = NULL;
5971}
5972
5973#ifdef CONFIG_SCHED_SMT
5974static const struct cpumask *cpu_smt_mask(int cpu)
5975{
5976	return topology_thread_cpumask(cpu);
5977}
5978#endif
5979
5980/*
5981 * Topology list, bottom-up.
5982 */
5983static struct sched_domain_topology_level default_topology[] = {
5984#ifdef CONFIG_SCHED_SMT
5985	{ sd_init_SIBLING, cpu_smt_mask, },
5986#endif
5987#ifdef CONFIG_SCHED_MC
5988	{ sd_init_MC, cpu_coregroup_mask, },
5989#endif
5990#ifdef CONFIG_SCHED_BOOK
5991	{ sd_init_BOOK, cpu_book_mask, },
5992#endif
5993	{ sd_init_CPU, cpu_cpu_mask, },
5994	{ NULL, },
5995};
5996
5997static struct sched_domain_topology_level *sched_domain_topology = default_topology;
5998
5999#define for_each_sd_topology(tl)			\
6000	for (tl = sched_domain_topology; tl->init; tl++)
6001
6002#ifdef CONFIG_NUMA
6003
6004static int sched_domains_numa_levels;
6005static int *sched_domains_numa_distance;
6006static struct cpumask ***sched_domains_numa_masks;
6007static int sched_domains_curr_level;
6008
6009static inline int sd_local_flags(int level)
6010{
6011	if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE)
6012		return 0;
6013
6014	return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE;
6015}
6016
6017static struct sched_domain *
6018sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
6019{
6020	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
6021	int level = tl->numa_level;
6022	int sd_weight = cpumask_weight(
6023			sched_domains_numa_masks[level][cpu_to_node(cpu)]);
6024
6025	*sd = (struct sched_domain){
6026		.min_interval		= sd_weight,
6027		.max_interval		= 2*sd_weight,
6028		.busy_factor		= 32,
6029		.imbalance_pct		= 125,
6030		.cache_nice_tries	= 2,
6031		.busy_idx		= 3,
6032		.idle_idx		= 2,
6033		.newidle_idx		= 0,
6034		.wake_idx		= 0,
6035		.forkexec_idx		= 0,
6036
6037		.flags			= 1*SD_LOAD_BALANCE
6038					| 1*SD_BALANCE_NEWIDLE
6039					| 0*SD_BALANCE_EXEC
6040					| 0*SD_BALANCE_FORK
6041					| 0*SD_BALANCE_WAKE
6042					| 0*SD_WAKE_AFFINE
6043					| 0*SD_SHARE_CPUPOWER
6044					| 0*SD_SHARE_PKG_RESOURCES
6045					| 1*SD_SERIALIZE
6046					| 0*SD_PREFER_SIBLING
6047					| 1*SD_NUMA
6048					| sd_local_flags(level)
6049					,
6050		.last_balance		= jiffies,
6051		.balance_interval	= sd_weight,
6052		.max_newidle_lb_cost	= 0,
6053		.next_decay_max_lb_cost	= jiffies,
6054	};
6055	SD_INIT_NAME(sd, NUMA);
6056	sd->private = &tl->data;
6057
6058	/*
6059	 * Ugly hack to pass state to sd_numa_mask()...
 
 
 
 
 
 
6060	 */
6061	sched_domains_curr_level = tl->numa_level;
6062
6063	return sd;
6064}
6065
6066static const struct cpumask *sd_numa_mask(int cpu)
6067{
6068	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
6069}
6070
6071static void sched_numa_warn(const char *str)
6072{
6073	static int done = false;
6074	int i,j;
6075
6076	if (done)
6077		return;
6078
6079	done = true;
6080
6081	printk(KERN_WARNING "ERROR: %s\n\n", str);
6082
6083	for (i = 0; i < nr_node_ids; i++) {
6084		printk(KERN_WARNING "  ");
6085		for (j = 0; j < nr_node_ids; j++)
6086			printk(KERN_CONT "%02d ", node_distance(i,j));
6087		printk(KERN_CONT "\n");
6088	}
6089	printk(KERN_WARNING "\n");
6090}
6091
6092static bool find_numa_distance(int distance)
6093{
6094	int i;
6095
6096	if (distance == node_distance(0, 0))
6097		return true;
6098
6099	for (i = 0; i < sched_domains_numa_levels; i++) {
6100		if (sched_domains_numa_distance[i] == distance)
6101			return true;
6102	}
 
6103
6104	return false;
6105}
6106
6107static void sched_init_numa(void)
6108{
6109	int next_distance, curr_distance = node_distance(0, 0);
6110	struct sched_domain_topology_level *tl;
6111	int level = 0;
6112	int i, j, k;
6113
6114	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
6115	if (!sched_domains_numa_distance)
6116		return;
6117
6118	/*
6119	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
6120	 * unique distances in the node_distance() table.
6121	 *
6122	 * Assumes node_distance(0,j) includes all distances in
6123	 * node_distance(i,j) in order to avoid cubic time.
6124	 */
6125	next_distance = curr_distance;
6126	for (i = 0; i < nr_node_ids; i++) {
6127		for (j = 0; j < nr_node_ids; j++) {
6128			for (k = 0; k < nr_node_ids; k++) {
6129				int distance = node_distance(i, k);
6130
6131				if (distance > curr_distance &&
6132				    (distance < next_distance ||
6133				     next_distance == curr_distance))
6134					next_distance = distance;
6135
6136				/*
6137				 * While not a strong assumption it would be nice to know
6138				 * about cases where if node A is connected to B, B is not
6139				 * equally connected to A.
6140				 */
6141				if (sched_debug() && node_distance(k, i) != distance)
6142					sched_numa_warn("Node-distance not symmetric");
6143
6144				if (sched_debug() && i && !find_numa_distance(distance))
6145					sched_numa_warn("Node-0 not representative");
6146			}
6147			if (next_distance != curr_distance) {
6148				sched_domains_numa_distance[level++] = next_distance;
6149				sched_domains_numa_levels = level;
6150				curr_distance = next_distance;
6151			} else break;
6152		}
6153
6154		/*
6155		 * In case of sched_debug() we verify the above assumption.
6156		 */
6157		if (!sched_debug())
6158			break;
6159	}
6160	/*
6161	 * 'level' contains the number of unique distances, excluding the
6162	 * identity distance node_distance(i,i).
6163	 *
6164	 * The sched_domains_numa_distance[] array includes the actual distance
6165	 * numbers.
6166	 */
6167
 
6168	/*
6169	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
6170	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
6171	 * the array will contain less then 'level' members. This could be
6172	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
6173	 * in other functions.
6174	 *
6175	 * We reset it to 'level' at the end of this function.
6176	 */
6177	sched_domains_numa_levels = 0;
6178
6179	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
6180	if (!sched_domains_numa_masks)
6181		return;
6182
6183	/*
6184	 * Now for each level, construct a mask per node which contains all
6185	 * cpus of nodes that are that many hops away from us.
6186	 */
6187	for (i = 0; i < level; i++) {
6188		sched_domains_numa_masks[i] =
6189			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
6190		if (!sched_domains_numa_masks[i])
6191			return;
6192
6193		for (j = 0; j < nr_node_ids; j++) {
6194			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6195			if (!mask)
6196				return;
6197
6198			sched_domains_numa_masks[i][j] = mask;
6199
6200			for (k = 0; k < nr_node_ids; k++) {
6201				if (node_distance(j, k) > sched_domains_numa_distance[i])
6202					continue;
6203
6204				cpumask_or(mask, mask, cpumask_of_node(k));
6205			}
6206		}
6207	}
6208
6209	tl = kzalloc((ARRAY_SIZE(default_topology) + level) *
6210			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
6211	if (!tl)
6212		return;
6213
6214	/*
6215	 * Copy the default topology bits..
6216	 */
6217	for (i = 0; default_topology[i].init; i++)
6218		tl[i] = default_topology[i];
6219
 
6220	/*
6221	 * .. and append 'j' levels of NUMA goodness.
6222	 */
6223	for (j = 0; j < level; i++, j++) {
6224		tl[i] = (struct sched_domain_topology_level){
6225			.init = sd_numa_init,
6226			.mask = sd_numa_mask,
6227			.flags = SDTL_OVERLAP,
6228			.numa_level = j,
6229		};
6230	}
6231
6232	sched_domain_topology = tl;
6233
6234	sched_domains_numa_levels = level;
6235}
6236
6237static void sched_domains_numa_masks_set(int cpu)
6238{
6239	int i, j;
6240	int node = cpu_to_node(cpu);
6241
6242	for (i = 0; i < sched_domains_numa_levels; i++) {
6243		for (j = 0; j < nr_node_ids; j++) {
6244			if (node_distance(j, node) <= sched_domains_numa_distance[i])
6245				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
6246		}
6247	}
6248}
6249
6250static void sched_domains_numa_masks_clear(int cpu)
6251{
6252	int i, j;
6253	for (i = 0; i < sched_domains_numa_levels; i++) {
6254		for (j = 0; j < nr_node_ids; j++)
6255			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
6256	}
6257}
6258
6259/*
6260 * Update sched_domains_numa_masks[level][node] array when new cpus
6261 * are onlined.
6262 */
6263static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6264					   unsigned long action,
6265					   void *hcpu)
6266{
6267	int cpu = (long)hcpu;
6268
6269	switch (action & ~CPU_TASKS_FROZEN) {
6270	case CPU_ONLINE:
6271		sched_domains_numa_masks_set(cpu);
6272		break;
6273
6274	case CPU_DEAD:
6275		sched_domains_numa_masks_clear(cpu);
6276		break;
6277
6278	default:
6279		return NOTIFY_DONE;
6280	}
6281
6282	return NOTIFY_OK;
6283}
6284#else
6285static inline void sched_init_numa(void)
6286{
6287}
6288
6289static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6290					   unsigned long action,
6291					   void *hcpu)
6292{
6293	return 0;
6294}
6295#endif /* CONFIG_NUMA */
6296
6297static int __sdt_alloc(const struct cpumask *cpu_map)
6298{
6299	struct sched_domain_topology_level *tl;
6300	int j;
6301
6302	for_each_sd_topology(tl) {
6303		struct sd_data *sdd = &tl->data;
6304
6305		sdd->sd = alloc_percpu(struct sched_domain *);
6306		if (!sdd->sd)
6307			return -ENOMEM;
6308
6309		sdd->sg = alloc_percpu(struct sched_group *);
6310		if (!sdd->sg)
6311			return -ENOMEM;
6312
6313		sdd->sgp = alloc_percpu(struct sched_group_power *);
6314		if (!sdd->sgp)
6315			return -ENOMEM;
6316
6317		for_each_cpu(j, cpu_map) {
6318			struct sched_domain *sd;
6319			struct sched_group *sg;
6320			struct sched_group_power *sgp;
6321
6322		       	sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6323					GFP_KERNEL, cpu_to_node(j));
6324			if (!sd)
6325				return -ENOMEM;
6326
6327			*per_cpu_ptr(sdd->sd, j) = sd;
6328
6329			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6330					GFP_KERNEL, cpu_to_node(j));
6331			if (!sg)
6332				return -ENOMEM;
6333
6334			sg->next = sg;
6335
6336			*per_cpu_ptr(sdd->sg, j) = sg;
6337
6338			sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(),
6339					GFP_KERNEL, cpu_to_node(j));
6340			if (!sgp)
6341				return -ENOMEM;
6342
6343			*per_cpu_ptr(sdd->sgp, j) = sgp;
6344		}
 
 
6345	}
6346
6347	return 0;
6348}
6349
6350static void __sdt_free(const struct cpumask *cpu_map)
6351{
6352	struct sched_domain_topology_level *tl;
6353	int j;
6354
6355	for_each_sd_topology(tl) {
6356		struct sd_data *sdd = &tl->data;
6357
6358		for_each_cpu(j, cpu_map) {
6359			struct sched_domain *sd;
6360
6361			if (sdd->sd) {
6362				sd = *per_cpu_ptr(sdd->sd, j);
6363				if (sd && (sd->flags & SD_OVERLAP))
6364					free_sched_groups(sd->groups, 0);
6365				kfree(*per_cpu_ptr(sdd->sd, j));
6366			}
6367
6368			if (sdd->sg)
6369				kfree(*per_cpu_ptr(sdd->sg, j));
6370			if (sdd->sgp)
6371				kfree(*per_cpu_ptr(sdd->sgp, j));
6372		}
6373		free_percpu(sdd->sd);
6374		sdd->sd = NULL;
6375		free_percpu(sdd->sg);
6376		sdd->sg = NULL;
6377		free_percpu(sdd->sgp);
6378		sdd->sgp = NULL;
6379	}
6380}
6381
6382struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
6383		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6384		struct sched_domain *child, int cpu)
6385{
6386	struct sched_domain *sd = tl->init(tl, cpu);
6387	if (!sd)
6388		return child;
6389
6390	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6391	if (child) {
6392		sd->level = child->level + 1;
6393		sched_domain_level_max = max(sched_domain_level_max, sd->level);
6394		child->parent = sd;
6395		sd->child = child;
6396	}
6397	set_domain_attribute(sd, attr);
6398
6399	return sd;
6400}
6401
6402/*
6403 * Build sched domains for a given set of cpus and attach the sched domains
6404 * to the individual cpus
6405 */
6406static int build_sched_domains(const struct cpumask *cpu_map,
6407			       struct sched_domain_attr *attr)
6408{
6409	enum s_alloc alloc_state;
6410	struct sched_domain *sd;
6411	struct s_data d;
6412	int i, ret = -ENOMEM;
6413
6414	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
6415	if (alloc_state != sa_rootdomain)
6416		goto error;
6417
6418	/* Set up domains for cpus specified by the cpu_map. */
6419	for_each_cpu(i, cpu_map) {
6420		struct sched_domain_topology_level *tl;
6421
6422		sd = NULL;
6423		for_each_sd_topology(tl) {
6424			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
6425			if (tl == sched_domain_topology)
6426				*per_cpu_ptr(d.sd, i) = sd;
6427			if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
6428				sd->flags |= SD_OVERLAP;
6429			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
6430				break;
6431		}
6432	}
6433
6434	/* Build the groups for the domains */
6435	for_each_cpu(i, cpu_map) {
6436		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6437			sd->span_weight = cpumask_weight(sched_domain_span(sd));
6438			if (sd->flags & SD_OVERLAP) {
6439				if (build_overlap_sched_groups(sd, i))
6440					goto error;
6441			} else {
6442				if (build_sched_groups(sd, i))
6443					goto error;
6444			}
6445		}
6446	}
6447
6448	/* Calculate CPU power for physical packages and nodes */
6449	for (i = nr_cpumask_bits-1; i >= 0; i--) {
6450		if (!cpumask_test_cpu(i, cpu_map))
6451			continue;
6452
6453		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6454			claim_allocations(i, sd);
6455			init_sched_groups_power(i, sd);
6456		}
6457	}
6458
6459	/* Attach the domains */
6460	rcu_read_lock();
6461	for_each_cpu(i, cpu_map) {
6462		sd = *per_cpu_ptr(d.sd, i);
6463		cpu_attach_domain(sd, d.rd, i);
6464	}
6465	rcu_read_unlock();
6466
6467	ret = 0;
6468error:
6469	__free_domain_allocs(&d, alloc_state, cpu_map);
6470	return ret;
6471}
6472
6473static cpumask_var_t *doms_cur;	/* current sched domains */
6474static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
6475static struct sched_domain_attr *dattr_cur;
6476				/* attribues of custom domains in 'doms_cur' */
6477
6478/*
6479 * Special case: If a kmalloc of a doms_cur partition (array of
6480 * cpumask) fails, then fallback to a single sched domain,
6481 * as determined by the single cpumask fallback_doms.
6482 */
6483static cpumask_var_t fallback_doms;
6484
6485/*
6486 * arch_update_cpu_topology lets virtualized architectures update the
6487 * cpu core maps. It is supposed to return 1 if the topology changed
6488 * or 0 if it stayed the same.
6489 */
6490int __weak arch_update_cpu_topology(void)
6491{
 
 
6492	return 0;
6493}
6494
6495cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
6496{
6497	int i;
6498	cpumask_var_t *doms;
6499
6500	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
6501	if (!doms)
6502		return NULL;
6503	for (i = 0; i < ndoms; i++) {
6504		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
6505			free_sched_domains(doms, i);
6506			return NULL;
6507		}
6508	}
6509	return doms;
6510}
6511
6512void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
6513{
6514	unsigned int i;
6515	for (i = 0; i < ndoms; i++)
6516		free_cpumask_var(doms[i]);
6517	kfree(doms);
6518}
6519
6520/*
6521 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
6522 * For now this just excludes isolated cpus, but could be used to
6523 * exclude other special cases in the future.
6524 */
6525static int init_sched_domains(const struct cpumask *cpu_map)
6526{
6527	int err;
6528
6529	arch_update_cpu_topology();
6530	ndoms_cur = 1;
6531	doms_cur = alloc_sched_domains(ndoms_cur);
6532	if (!doms_cur)
6533		doms_cur = &fallback_doms;
6534	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
6535	err = build_sched_domains(doms_cur[0], NULL);
6536	register_sched_domain_sysctl();
6537
6538	return err;
6539}
6540
6541/*
6542 * Detach sched domains from a group of cpus specified in cpu_map
6543 * These cpus will now be attached to the NULL domain
6544 */
6545static void detach_destroy_domains(const struct cpumask *cpu_map)
6546{
6547	int i;
6548
6549	rcu_read_lock();
6550	for_each_cpu(i, cpu_map)
6551		cpu_attach_domain(NULL, &def_root_domain, i);
6552	rcu_read_unlock();
6553}
6554
6555/* handle null as "default" */
6556static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
6557			struct sched_domain_attr *new, int idx_new)
6558{
6559	struct sched_domain_attr tmp;
6560
6561	/* fast path */
6562	if (!new && !cur)
6563		return 1;
6564
6565	tmp = SD_ATTR_INIT;
6566	return !memcmp(cur ? (cur + idx_cur) : &tmp,
6567			new ? (new + idx_new) : &tmp,
6568			sizeof(struct sched_domain_attr));
6569}
6570
6571/*
6572 * Partition sched domains as specified by the 'ndoms_new'
6573 * cpumasks in the array doms_new[] of cpumasks. This compares
6574 * doms_new[] to the current sched domain partitioning, doms_cur[].
6575 * It destroys each deleted domain and builds each new domain.
6576 *
6577 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
6578 * The masks don't intersect (don't overlap.) We should setup one
6579 * sched domain for each mask. CPUs not in any of the cpumasks will
6580 * not be load balanced. If the same cpumask appears both in the
6581 * current 'doms_cur' domains and in the new 'doms_new', we can leave
6582 * it as it is.
6583 *
6584 * The passed in 'doms_new' should be allocated using
6585 * alloc_sched_domains.  This routine takes ownership of it and will
6586 * free_sched_domains it when done with it. If the caller failed the
6587 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
6588 * and partition_sched_domains() will fallback to the single partition
6589 * 'fallback_doms', it also forces the domains to be rebuilt.
6590 *
6591 * If doms_new == NULL it will be replaced with cpu_online_mask.
6592 * ndoms_new == 0 is a special case for destroying existing domains,
6593 * and it will not create the default domain.
6594 *
6595 * Call with hotplug lock held
6596 */
6597void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
6598			     struct sched_domain_attr *dattr_new)
6599{
6600	int i, j, n;
6601	int new_topology;
6602
6603	mutex_lock(&sched_domains_mutex);
6604
6605	/* always unregister in case we don't destroy any domains */
6606	unregister_sched_domain_sysctl();
6607
6608	/* Let architecture update cpu core mappings. */
6609	new_topology = arch_update_cpu_topology();
6610
6611	n = doms_new ? ndoms_new : 0;
6612
6613	/* Destroy deleted domains */
6614	for (i = 0; i < ndoms_cur; i++) {
6615		for (j = 0; j < n && !new_topology; j++) {
6616			if (cpumask_equal(doms_cur[i], doms_new[j])
6617			    && dattrs_equal(dattr_cur, i, dattr_new, j))
6618				goto match1;
6619		}
6620		/* no match - a current sched domain not in new doms_new[] */
6621		detach_destroy_domains(doms_cur[i]);
6622match1:
6623		;
6624	}
6625
6626	n = ndoms_cur;
6627	if (doms_new == NULL) {
6628		n = 0;
6629		doms_new = &fallback_doms;
6630		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
6631		WARN_ON_ONCE(dattr_new);
6632	}
6633
6634	/* Build new domains */
6635	for (i = 0; i < ndoms_new; i++) {
6636		for (j = 0; j < n && !new_topology; j++) {
6637			if (cpumask_equal(doms_new[i], doms_cur[j])
6638			    && dattrs_equal(dattr_new, i, dattr_cur, j))
6639				goto match2;
6640		}
6641		/* no match - add a new doms_new */
6642		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
6643match2:
6644		;
6645	}
6646
6647	/* Remember the new sched domains */
6648	if (doms_cur != &fallback_doms)
6649		free_sched_domains(doms_cur, ndoms_cur);
6650	kfree(dattr_cur);	/* kfree(NULL) is safe */
6651	doms_cur = doms_new;
6652	dattr_cur = dattr_new;
6653	ndoms_cur = ndoms_new;
6654
6655	register_sched_domain_sysctl();
6656
6657	mutex_unlock(&sched_domains_mutex);
6658}
6659
6660static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */
6661
6662/*
6663 * Update cpusets according to cpu_active mask.  If cpusets are
6664 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
6665 * around partition_sched_domains().
6666 *
6667 * If we come here as part of a suspend/resume, don't touch cpusets because we
6668 * want to restore it back to its original state upon resume anyway.
6669 */
6670static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
6671			     void *hcpu)
6672{
6673	switch (action) {
6674	case CPU_ONLINE_FROZEN:
6675	case CPU_DOWN_FAILED_FROZEN:
6676
6677		/*
6678		 * num_cpus_frozen tracks how many CPUs are involved in suspend
6679		 * resume sequence. As long as this is not the last online
6680		 * operation in the resume sequence, just build a single sched
6681		 * domain, ignoring cpusets.
6682		 */
6683		num_cpus_frozen--;
6684		if (likely(num_cpus_frozen)) {
6685			partition_sched_domains(1, NULL, NULL);
6686			break;
6687		}
6688
6689		/*
6690		 * This is the last CPU online operation. So fall through and
6691		 * restore the original sched domains by considering the
6692		 * cpuset configurations.
6693		 */
6694
6695	case CPU_ONLINE:
6696	case CPU_DOWN_FAILED:
6697		cpuset_update_active_cpus(true);
6698		break;
6699	default:
6700		return NOTIFY_DONE;
6701	}
6702	return NOTIFY_OK;
6703}
 
6704
6705static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
6706			       void *hcpu)
6707{
6708	switch (action) {
6709	case CPU_DOWN_PREPARE:
6710		cpuset_update_active_cpus(false);
6711		break;
6712	case CPU_DOWN_PREPARE_FROZEN:
6713		num_cpus_frozen++;
6714		partition_sched_domains(1, NULL, NULL);
6715		break;
6716	default:
6717		return NOTIFY_DONE;
6718	}
6719	return NOTIFY_OK;
6720}
 
6721
6722void __init sched_init_smp(void)
6723{
6724	cpumask_var_t non_isolated_cpus;
6725
6726	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
6727	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
6728
6729	sched_init_numa();
6730
6731	/*
6732	 * There's no userspace yet to cause hotplug operations; hence all the
6733	 * cpu masks are stable and all blatant races in the below code cannot
6734	 * happen.
6735	 */
6736	mutex_lock(&sched_domains_mutex);
6737	init_sched_domains(cpu_active_mask);
6738	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
6739	if (cpumask_empty(non_isolated_cpus))
6740		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
6741	mutex_unlock(&sched_domains_mutex);
6742
6743	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
6744	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
6745	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
6746
6747	init_hrtick();
6748
6749	/* Move init over to a non-isolated CPU */
6750	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
6751		BUG();
6752	sched_init_granularity();
6753	free_cpumask_var(non_isolated_cpus);
6754
6755	init_sched_rt_class();
6756	init_sched_dl_class();
 
 
6757}
 
 
 
 
 
 
 
 
6758#else
6759void __init sched_init_smp(void)
6760{
6761	sched_init_granularity();
6762}
6763#endif /* CONFIG_SMP */
6764
6765const_debug unsigned int sysctl_timer_migration = 1;
6766
6767int in_sched_functions(unsigned long addr)
6768{
6769	return in_lock_functions(addr) ||
6770		(addr >= (unsigned long)__sched_text_start
6771		&& addr < (unsigned long)__sched_text_end);
6772}
6773
6774#ifdef CONFIG_CGROUP_SCHED
6775/*
6776 * Default task group.
6777 * Every task in system belongs to this group at bootup.
6778 */
6779struct task_group root_task_group;
6780LIST_HEAD(task_groups);
 
 
 
6781#endif
6782
6783DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
 
6784
6785void __init sched_init(void)
6786{
6787	int i, j;
6788	unsigned long alloc_size = 0, ptr;
 
 
 
 
 
 
 
 
 
 
6789
6790#ifdef CONFIG_FAIR_GROUP_SCHED
6791	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6792#endif
6793#ifdef CONFIG_RT_GROUP_SCHED
6794	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6795#endif
6796#ifdef CONFIG_CPUMASK_OFFSTACK
6797	alloc_size += num_possible_cpus() * cpumask_size();
6798#endif
6799	if (alloc_size) {
6800		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
6801
6802#ifdef CONFIG_FAIR_GROUP_SCHED
6803		root_task_group.se = (struct sched_entity **)ptr;
6804		ptr += nr_cpu_ids * sizeof(void **);
6805
6806		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6807		ptr += nr_cpu_ids * sizeof(void **);
6808
 
 
6809#endif /* CONFIG_FAIR_GROUP_SCHED */
6810#ifdef CONFIG_RT_GROUP_SCHED
6811		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
6812		ptr += nr_cpu_ids * sizeof(void **);
6813
6814		root_task_group.rt_rq = (struct rt_rq **)ptr;
6815		ptr += nr_cpu_ids * sizeof(void **);
6816
6817#endif /* CONFIG_RT_GROUP_SCHED */
 
6818#ifdef CONFIG_CPUMASK_OFFSTACK
6819		for_each_possible_cpu(i) {
6820			per_cpu(load_balance_mask, i) = (void *)ptr;
6821			ptr += cpumask_size();
6822		}
6823#endif /* CONFIG_CPUMASK_OFFSTACK */
6824	}
 
6825
6826	init_rt_bandwidth(&def_rt_bandwidth,
6827			global_rt_period(), global_rt_runtime());
6828	init_dl_bandwidth(&def_dl_bandwidth,
6829			global_rt_period(), global_rt_runtime());
6830
6831#ifdef CONFIG_SMP
6832	init_defrootdomain();
6833#endif
6834
6835#ifdef CONFIG_RT_GROUP_SCHED
6836	init_rt_bandwidth(&root_task_group.rt_bandwidth,
6837			global_rt_period(), global_rt_runtime());
6838#endif /* CONFIG_RT_GROUP_SCHED */
6839
6840#ifdef CONFIG_CGROUP_SCHED
 
 
6841	list_add(&root_task_group.list, &task_groups);
6842	INIT_LIST_HEAD(&root_task_group.children);
6843	INIT_LIST_HEAD(&root_task_group.siblings);
6844	autogroup_init(&init_task);
6845
6846#endif /* CONFIG_CGROUP_SCHED */
6847
6848	for_each_possible_cpu(i) {
6849		struct rq *rq;
6850
6851		rq = cpu_rq(i);
6852		raw_spin_lock_init(&rq->lock);
6853		rq->nr_running = 0;
6854		rq->calc_load_active = 0;
6855		rq->calc_load_update = jiffies + LOAD_FREQ;
6856		init_cfs_rq(&rq->cfs);
6857		init_rt_rq(&rq->rt, rq);
6858		init_dl_rq(&rq->dl, rq);
6859#ifdef CONFIG_FAIR_GROUP_SCHED
6860		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
6861		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
 
6862		/*
6863		 * How much cpu bandwidth does root_task_group get?
6864		 *
6865		 * In case of task-groups formed thr' the cgroup filesystem, it
6866		 * gets 100% of the cpu resources in the system. This overall
6867		 * system cpu resource is divided among the tasks of
6868		 * root_task_group and its child task-groups in a fair manner,
6869		 * based on each entity's (task or task-group's) weight
6870		 * (se->load.weight).
6871		 *
6872		 * In other words, if root_task_group has 10 tasks of weight
6873		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
6874		 * then A0's share of the cpu resource is:
6875		 *
6876		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
6877		 *
6878		 * We achieve this by letting root_task_group's tasks sit
6879		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
6880		 */
6881		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
6882		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
6883#endif /* CONFIG_FAIR_GROUP_SCHED */
6884
6885		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
6886#ifdef CONFIG_RT_GROUP_SCHED
6887		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
6888#endif
6889
6890		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
6891			rq->cpu_load[j] = 0;
6892
6893		rq->last_load_update_tick = jiffies;
6894
6895#ifdef CONFIG_SMP
6896		rq->sd = NULL;
6897		rq->rd = NULL;
6898		rq->cpu_power = SCHED_POWER_SCALE;
6899		rq->post_schedule = 0;
6900		rq->active_balance = 0;
6901		rq->next_balance = jiffies;
6902		rq->push_cpu = 0;
6903		rq->cpu = i;
6904		rq->online = 0;
6905		rq->idle_stamp = 0;
6906		rq->avg_idle = 2*sysctl_sched_migration_cost;
6907		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
6908
6909		INIT_LIST_HEAD(&rq->cfs_tasks);
6910
6911		rq_attach_root(rq, &def_root_domain);
6912#ifdef CONFIG_NO_HZ_COMMON
6913		rq->nohz_flags = 0;
6914#endif
6915#ifdef CONFIG_NO_HZ_FULL
6916		rq->last_sched_tick = 0;
6917#endif
6918#endif
6919		init_rq_hrtick(rq);
 
6920		atomic_set(&rq->nr_iowait, 0);
6921	}
6922
6923	set_load_weight(&init_task);
6924
6925#ifdef CONFIG_PREEMPT_NOTIFIERS
6926	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
6927#endif
6928
6929	/*
6930	 * The boot idle thread does lazy MMU switching as well:
6931	 */
6932	atomic_inc(&init_mm.mm_count);
6933	enter_lazy_tlb(&init_mm, current);
6934
6935	/*
6936	 * Make us the idle thread. Technically, schedule() should not be
6937	 * called from this thread, however somewhere below it might be,
6938	 * but because we are the idle thread, we just pick up running again
6939	 * when this runqueue becomes "idle".
6940	 */
6941	init_idle(current, smp_processor_id());
6942
6943	calc_load_update = jiffies + LOAD_FREQ;
6944
6945	/*
6946	 * During early bootup we pretend to be a normal task:
6947	 */
6948	current->sched_class = &fair_sched_class;
6949
6950#ifdef CONFIG_SMP
6951	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
6952	/* May be allocated at isolcpus cmdline parse time */
6953	if (cpu_isolated_map == NULL)
6954		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
6955	idle_thread_set_boot_cpu();
6956#endif
6957	init_sched_fair_class();
6958
 
 
 
 
 
 
6959	scheduler_running = 1;
6960}
6961
6962#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
6963static inline int preempt_count_equals(int preempt_offset)
6964{
6965	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
6966
6967	return (nested == preempt_offset);
6968}
6969
6970void __might_sleep(const char *file, int line, int preempt_offset)
6971{
6972	static unsigned long prev_jiffy;	/* ratelimiting */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6973
6974	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
6975	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
6976	     !is_idle_task(current)) ||
6977	    system_state != SYSTEM_RUNNING || oops_in_progress)
 
6978		return;
 
6979	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
6980		return;
6981	prev_jiffy = jiffies;
6982
 
 
 
6983	printk(KERN_ERR
6984		"BUG: sleeping function called from invalid context at %s:%d\n",
6985			file, line);
6986	printk(KERN_ERR
6987		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
6988			in_atomic(), irqs_disabled(),
6989			current->pid, current->comm);
6990
 
 
 
6991	debug_show_held_locks(current);
6992	if (irqs_disabled())
6993		print_irqtrace_events(current);
6994#ifdef CONFIG_DEBUG_PREEMPT
6995	if (!preempt_count_equals(preempt_offset)) {
6996		pr_err("Preemption disabled at:");
6997		print_ip_sym(current->preempt_disable_ip);
6998		pr_cont("\n");
6999	}
7000#endif
7001	dump_stack();
 
7002}
7003EXPORT_SYMBOL(__might_sleep);
7004#endif
7005
7006#ifdef CONFIG_MAGIC_SYSRQ
7007static void normalize_task(struct rq *rq, struct task_struct *p)
7008{
7009	const struct sched_class *prev_class = p->sched_class;
7010	struct sched_attr attr = {
7011		.sched_policy = SCHED_NORMAL,
7012	};
7013	int old_prio = p->prio;
7014	int on_rq;
7015
7016	on_rq = p->on_rq;
7017	if (on_rq)
7018		dequeue_task(rq, p, 0);
7019	__setscheduler(rq, p, &attr);
7020	if (on_rq) {
7021		enqueue_task(rq, p, 0);
7022		resched_task(rq->curr);
7023	}
7024
7025	check_class_changed(rq, p, prev_class, old_prio);
 
 
 
 
 
 
 
 
 
 
 
7026}
 
 
7027
 
7028void normalize_rt_tasks(void)
7029{
7030	struct task_struct *g, *p;
7031	unsigned long flags;
7032	struct rq *rq;
 
7033
7034	read_lock_irqsave(&tasklist_lock, flags);
7035	do_each_thread(g, p) {
7036		/*
7037		 * Only normalize user tasks:
7038		 */
7039		if (!p->mm)
7040			continue;
7041
7042		p->se.exec_start		= 0;
7043#ifdef CONFIG_SCHEDSTATS
7044		p->se.statistics.wait_start	= 0;
7045		p->se.statistics.sleep_start	= 0;
7046		p->se.statistics.block_start	= 0;
7047#endif
7048
7049		if (!dl_task(p) && !rt_task(p)) {
7050			/*
7051			 * Renice negative nice level userspace
7052			 * tasks back to 0:
7053			 */
7054			if (task_nice(p) < 0 && p->mm)
7055				set_user_nice(p, 0);
7056			continue;
7057		}
7058
7059		raw_spin_lock(&p->pi_lock);
7060		rq = __task_rq_lock(p);
7061
7062		normalize_task(rq, p);
7063
7064		__task_rq_unlock(rq);
7065		raw_spin_unlock(&p->pi_lock);
7066	} while_each_thread(g, p);
7067
7068	read_unlock_irqrestore(&tasklist_lock, flags);
7069}
7070
7071#endif /* CONFIG_MAGIC_SYSRQ */
7072
7073#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7074/*
7075 * These functions are only useful for the IA64 MCA handling, or kdb.
7076 *
7077 * They can only be called when the whole system has been
7078 * stopped - every CPU needs to be quiescent, and no scheduling
7079 * activity can take place. Using them for anything else would
7080 * be a serious bug, and as a result, they aren't even visible
7081 * under any other configuration.
7082 */
7083
7084/**
7085 * curr_task - return the current task for a given cpu.
7086 * @cpu: the processor in question.
7087 *
7088 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7089 *
7090 * Return: The current task for @cpu.
7091 */
7092struct task_struct *curr_task(int cpu)
7093{
7094	return cpu_curr(cpu);
7095}
7096
7097#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
7098
7099#ifdef CONFIG_IA64
7100/**
7101 * set_curr_task - set the current task for a given cpu.
7102 * @cpu: the processor in question.
7103 * @p: the task pointer to set.
7104 *
7105 * Description: This function must only be used when non-maskable interrupts
7106 * are serviced on a separate stack. It allows the architecture to switch the
7107 * notion of the current task on a cpu in a non-blocking manner. This function
7108 * must be called with all CPU's synchronized, and interrupts disabled, the
7109 * and caller must save the original value of the current task (see
7110 * curr_task() above) and restore that value before reenabling interrupts and
7111 * re-starting the system.
7112 *
7113 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7114 */
7115void set_curr_task(int cpu, struct task_struct *p)
7116{
7117	cpu_curr(cpu) = p;
7118}
7119
7120#endif
7121
7122#ifdef CONFIG_CGROUP_SCHED
7123/* task_group_lock serializes the addition/removal of task groups */
7124static DEFINE_SPINLOCK(task_group_lock);
7125
7126static void free_sched_group(struct task_group *tg)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7127{
7128	free_fair_sched_group(tg);
7129	free_rt_sched_group(tg);
7130	autogroup_free(tg);
7131	kfree(tg);
7132}
7133
7134/* allocate runqueue etc for a new task group */
7135struct task_group *sched_create_group(struct task_group *parent)
7136{
7137	struct task_group *tg;
7138
7139	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
7140	if (!tg)
7141		return ERR_PTR(-ENOMEM);
7142
7143	if (!alloc_fair_sched_group(tg, parent))
7144		goto err;
7145
7146	if (!alloc_rt_sched_group(tg, parent))
7147		goto err;
7148
 
 
7149	return tg;
7150
7151err:
7152	free_sched_group(tg);
7153	return ERR_PTR(-ENOMEM);
7154}
7155
7156void sched_online_group(struct task_group *tg, struct task_group *parent)
7157{
7158	unsigned long flags;
7159
7160	spin_lock_irqsave(&task_group_lock, flags);
7161	list_add_rcu(&tg->list, &task_groups);
7162
7163	WARN_ON(!parent); /* root should already exist */
 
7164
7165	tg->parent = parent;
7166	INIT_LIST_HEAD(&tg->children);
7167	list_add_rcu(&tg->siblings, &parent->children);
7168	spin_unlock_irqrestore(&task_group_lock, flags);
 
 
7169}
7170
7171/* rcu callback to free various structures associated with a task group */
7172static void free_sched_group_rcu(struct rcu_head *rhp)
7173{
7174	/* now it should be safe to free those cfs_rqs */
7175	free_sched_group(container_of(rhp, struct task_group, rcu));
7176}
7177
7178/* Destroy runqueue etc associated with a task group */
7179void sched_destroy_group(struct task_group *tg)
7180{
7181	/* wait for possible concurrent references to cfs_rqs complete */
7182	call_rcu(&tg->rcu, free_sched_group_rcu);
7183}
7184
7185void sched_offline_group(struct task_group *tg)
7186{
7187	unsigned long flags;
7188	int i;
7189
7190	/* end participation in shares distribution */
7191	for_each_possible_cpu(i)
7192		unregister_fair_sched_group(tg, i);
7193
7194	spin_lock_irqsave(&task_group_lock, flags);
7195	list_del_rcu(&tg->list);
7196	list_del_rcu(&tg->siblings);
7197	spin_unlock_irqrestore(&task_group_lock, flags);
7198}
7199
7200/* change task's runqueue when it moves between groups.
7201 *	The caller of this function should have put the task in its new group
7202 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
7203 *	reflect its new group.
7204 */
7205void sched_move_task(struct task_struct *tsk)
7206{
7207	struct task_group *tg;
7208	int on_rq, running;
7209	unsigned long flags;
7210	struct rq *rq;
7211
7212	rq = task_rq_lock(tsk, &flags);
7213
7214	running = task_current(rq, tsk);
7215	on_rq = tsk->on_rq;
7216
7217	if (on_rq)
7218		dequeue_task(rq, tsk, 0);
7219	if (unlikely(running))
7220		tsk->sched_class->put_prev_task(rq, tsk);
7221
7222	tg = container_of(task_css_check(tsk, cpu_cgrp_id,
7223				lockdep_is_held(&tsk->sighand->siglock)),
7224			  struct task_group, css);
7225	tg = autogroup_task_group(tsk, tg);
7226	tsk->sched_task_group = tg;
7227
7228#ifdef CONFIG_FAIR_GROUP_SCHED
7229	if (tsk->sched_class->task_move_group)
7230		tsk->sched_class->task_move_group(tsk, on_rq);
7231	else
7232#endif
7233		set_task_rq(tsk, task_cpu(tsk));
7234
7235	if (unlikely(running))
7236		tsk->sched_class->set_curr_task(rq);
7237	if (on_rq)
7238		enqueue_task(rq, tsk, 0);
7239
7240	task_rq_unlock(rq, tsk, &flags);
7241}
7242#endif /* CONFIG_CGROUP_SCHED */
7243
7244#ifdef CONFIG_RT_GROUP_SCHED
7245/*
7246 * Ensure that the real time constraints are schedulable.
 
 
 
 
7247 */
7248static DEFINE_MUTEX(rt_constraints_mutex);
7249
7250/* Must be called with tasklist_lock held */
7251static inline int tg_has_rt_tasks(struct task_group *tg)
7252{
7253	struct task_struct *g, *p;
7254
7255	do_each_thread(g, p) {
7256		if (rt_task(p) && task_rq(p)->rt.tg == tg)
7257			return 1;
7258	} while_each_thread(g, p);
7259
7260	return 0;
7261}
7262
7263struct rt_schedulable_data {
7264	struct task_group *tg;
7265	u64 rt_period;
7266	u64 rt_runtime;
7267};
7268
7269static int tg_rt_schedulable(struct task_group *tg, void *data)
7270{
7271	struct rt_schedulable_data *d = data;
7272	struct task_group *child;
7273	unsigned long total, sum = 0;
7274	u64 period, runtime;
7275
7276	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7277	runtime = tg->rt_bandwidth.rt_runtime;
7278
7279	if (tg == d->tg) {
7280		period = d->rt_period;
7281		runtime = d->rt_runtime;
7282	}
7283
7284	/*
7285	 * Cannot have more runtime than the period.
7286	 */
7287	if (runtime > period && runtime != RUNTIME_INF)
7288		return -EINVAL;
7289
7290	/*
7291	 * Ensure we don't starve existing RT tasks.
7292	 */
7293	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
7294		return -EBUSY;
7295
7296	total = to_ratio(period, runtime);
 
7297
7298	/*
7299	 * Nobody can have more than the global setting allows.
7300	 */
7301	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
7302		return -EINVAL;
7303
7304	/*
7305	 * The sum of our children's runtime should not exceed our own.
7306	 */
7307	list_for_each_entry_rcu(child, &tg->children, siblings) {
7308		period = ktime_to_ns(child->rt_bandwidth.rt_period);
7309		runtime = child->rt_bandwidth.rt_runtime;
7310
7311		if (child == d->tg) {
7312			period = d->rt_period;
7313			runtime = d->rt_runtime;
7314		}
7315
7316		sum += to_ratio(period, runtime);
 
 
 
 
 
 
 
 
 
7317	}
7318
7319	if (sum > total)
7320		return -EINVAL;
7321
7322	return 0;
7323}
7324
7325static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
7326{
7327	int ret;
7328
7329	struct rt_schedulable_data data = {
7330		.tg = tg,
7331		.rt_period = period,
7332		.rt_runtime = runtime,
7333	};
7334
7335	rcu_read_lock();
7336	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
7337	rcu_read_unlock();
7338
7339	return ret;
7340}
7341
7342static int tg_set_rt_bandwidth(struct task_group *tg,
7343		u64 rt_period, u64 rt_runtime)
7344{
7345	int i, err = 0;
7346
7347	mutex_lock(&rt_constraints_mutex);
7348	read_lock(&tasklist_lock);
7349	err = __rt_schedulable(tg, rt_period, rt_runtime);
7350	if (err)
7351		goto unlock;
7352
7353	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
7354	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
7355	tg->rt_bandwidth.rt_runtime = rt_runtime;
7356
7357	for_each_possible_cpu(i) {
7358		struct rt_rq *rt_rq = tg->rt_rq[i];
7359
7360		raw_spin_lock(&rt_rq->rt_runtime_lock);
7361		rt_rq->rt_runtime = rt_runtime;
7362		raw_spin_unlock(&rt_rq->rt_runtime_lock);
7363	}
7364	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
7365unlock:
7366	read_unlock(&tasklist_lock);
7367	mutex_unlock(&rt_constraints_mutex);
7368
7369	return err;
7370}
7371
7372static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
7373{
7374	u64 rt_runtime, rt_period;
7375
7376	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7377	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
7378	if (rt_runtime_us < 0)
7379		rt_runtime = RUNTIME_INF;
7380
7381	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7382}
7383
7384static long sched_group_rt_runtime(struct task_group *tg)
7385{
7386	u64 rt_runtime_us;
7387
7388	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
7389		return -1;
7390
7391	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
7392	do_div(rt_runtime_us, NSEC_PER_USEC);
7393	return rt_runtime_us;
7394}
7395
7396static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
 
7397{
7398	u64 rt_runtime, rt_period;
7399
7400	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
7401	rt_runtime = tg->rt_bandwidth.rt_runtime;
7402
7403	if (rt_period == 0)
7404		return -EINVAL;
7405
7406	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7407}
7408
7409static long sched_group_rt_period(struct task_group *tg)
7410{
7411	u64 rt_period_us;
 
7412
7413	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
7414	do_div(rt_period_us, NSEC_PER_USEC);
7415	return rt_period_us;
7416}
7417#endif /* CONFIG_RT_GROUP_SCHED */
7418
7419#ifdef CONFIG_RT_GROUP_SCHED
7420static int sched_rt_global_constraints(void)
7421{
7422	int ret = 0;
7423
7424	mutex_lock(&rt_constraints_mutex);
7425	read_lock(&tasklist_lock);
7426	ret = __rt_schedulable(NULL, 0, 0);
7427	read_unlock(&tasklist_lock);
7428	mutex_unlock(&rt_constraints_mutex);
7429
7430	return ret;
7431}
7432
7433static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
7434{
7435	/* Don't accept realtime tasks when there is no way for them to run */
7436	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
7437		return 0;
7438
7439	return 1;
 
 
 
7440}
7441
7442#else /* !CONFIG_RT_GROUP_SCHED */
7443static int sched_rt_global_constraints(void)
 
 
 
7444{
7445	unsigned long flags;
7446	int i, ret = 0;
7447
7448	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
7449	for_each_possible_cpu(i) {
7450		struct rt_rq *rt_rq = &cpu_rq(i)->rt;
7451
7452		raw_spin_lock(&rt_rq->rt_runtime_lock);
7453		rt_rq->rt_runtime = global_rt_runtime();
7454		raw_spin_unlock(&rt_rq->rt_runtime_lock);
7455	}
7456	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
7457
7458	return ret;
7459}
7460#endif /* CONFIG_RT_GROUP_SCHED */
7461
7462static int sched_dl_global_constraints(void)
7463{
7464	u64 runtime = global_rt_runtime();
7465	u64 period = global_rt_period();
7466	u64 new_bw = to_ratio(period, runtime);
7467	int cpu, ret = 0;
7468	unsigned long flags;
7469
7470	/*
7471	 * Here we want to check the bandwidth not being set to some
7472	 * value smaller than the currently allocated bandwidth in
7473	 * any of the root_domains.
7474	 *
7475	 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
7476	 * cycling on root_domains... Discussion on different/better
7477	 * solutions is welcome!
7478	 */
7479	for_each_possible_cpu(cpu) {
7480		struct dl_bw *dl_b = dl_bw_of(cpu);
7481
7482		raw_spin_lock_irqsave(&dl_b->lock, flags);
7483		if (new_bw < dl_b->total_bw)
7484			ret = -EBUSY;
7485		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
 
 
7486
7487		if (ret)
7488			break;
7489	}
7490
7491	return ret;
7492}
7493
7494static void sched_dl_do_global(void)
7495{
7496	u64 new_bw = -1;
7497	int cpu;
7498	unsigned long flags;
7499
7500	def_dl_bandwidth.dl_period = global_rt_period();
7501	def_dl_bandwidth.dl_runtime = global_rt_runtime();
7502
7503	if (global_rt_runtime() != RUNTIME_INF)
7504		new_bw = to_ratio(global_rt_period(), global_rt_runtime());
7505
7506	/*
7507	 * FIXME: As above...
7508	 */
7509	for_each_possible_cpu(cpu) {
7510		struct dl_bw *dl_b = dl_bw_of(cpu);
7511
7512		raw_spin_lock_irqsave(&dl_b->lock, flags);
7513		dl_b->bw = new_bw;
7514		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
7515	}
7516}
7517
7518static int sched_rt_global_validate(void)
 
7519{
7520	if (sysctl_sched_rt_period <= 0)
7521		return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7522
7523	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
7524		(sysctl_sched_rt_runtime > sysctl_sched_rt_period))
7525		return -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
7526
7527	return 0;
 
 
7528}
7529
7530static void sched_rt_do_global(void)
7531{
7532	def_rt_bandwidth.rt_runtime = global_rt_runtime();
7533	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
7534}
 
 
7535
7536int sched_rt_handler(struct ctl_table *table, int write,
7537		void __user *buffer, size_t *lenp,
7538		loff_t *ppos)
7539{
7540	int old_period, old_runtime;
7541	static DEFINE_MUTEX(mutex);
7542	int ret;
 
7543
7544	mutex_lock(&mutex);
7545	old_period = sysctl_sched_rt_period;
7546	old_runtime = sysctl_sched_rt_runtime;
7547
7548	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7549
7550	if (!ret && write) {
7551		ret = sched_rt_global_validate();
7552		if (ret)
7553			goto undo;
7554
7555		ret = sched_rt_global_constraints();
7556		if (ret)
7557			goto undo;
7558
7559		ret = sched_dl_global_constraints();
7560		if (ret)
7561			goto undo;
 
 
 
 
 
 
 
7562
7563		sched_rt_do_global();
7564		sched_dl_do_global();
7565	}
7566	if (0) {
7567undo:
7568		sysctl_sched_rt_period = old_period;
7569		sysctl_sched_rt_runtime = old_runtime;
7570	}
7571	mutex_unlock(&mutex);
7572
7573	return ret;
7574}
7575
7576int sched_rr_handler(struct ctl_table *table, int write,
7577		void __user *buffer, size_t *lenp,
7578		loff_t *ppos)
7579{
7580	int ret;
7581	static DEFINE_MUTEX(mutex);
7582
7583	mutex_lock(&mutex);
7584	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7585	/* make sure that internally we keep jiffies */
7586	/* also, writing zero resets timeslice to default */
7587	if (!ret && write) {
7588		sched_rr_timeslice = sched_rr_timeslice <= 0 ?
7589			RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
7590	}
7591	mutex_unlock(&mutex);
7592	return ret;
7593}
7594
7595#ifdef CONFIG_CGROUP_SCHED
 
 
7596
7597static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
7598{
7599	return css ? container_of(css, struct task_group, css) : NULL;
7600}
7601
7602static struct cgroup_subsys_state *
7603cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
7604{
7605	struct task_group *parent = css_tg(parent_css);
7606	struct task_group *tg;
7607
7608	if (!parent) {
7609		/* This is early initialization for the top cgroup */
7610		return &root_task_group.css;
7611	}
7612
7613	tg = sched_create_group(parent);
7614	if (IS_ERR(tg))
7615		return ERR_PTR(-ENOMEM);
 
 
7616
7617	return &tg->css;
7618}
7619
7620static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
7621{
7622	struct task_group *tg = css_tg(css);
7623	struct task_group *parent = css_tg(css_parent(css));
7624
7625	if (parent)
7626		sched_online_group(tg, parent);
7627	return 0;
7628}
7629
7630static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
 
 
7631{
7632	struct task_group *tg = css_tg(css);
7633
7634	sched_destroy_group(tg);
7635}
7636
7637static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
 
 
7638{
7639	struct task_group *tg = css_tg(css);
7640
7641	sched_offline_group(tg);
7642}
7643
7644static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
7645				 struct cgroup_taskset *tset)
7646{
7647	struct task_struct *task;
 
 
 
7648
7649	cgroup_taskset_for_each(task, tset) {
7650#ifdef CONFIG_RT_GROUP_SCHED
7651		if (!sched_rt_can_attach(css_tg(css), task))
7652			return -EINVAL;
7653#else
7654		/* We don't support RT-tasks being in separate groups */
7655		if (task->sched_class != &fair_sched_class)
7656			return -EINVAL;
7657#endif
7658	}
7659	return 0;
 
 
 
7660}
7661
7662static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
7663			      struct cgroup_taskset *tset)
7664{
7665	struct task_struct *task;
7666
7667	cgroup_taskset_for_each(task, tset)
7668		sched_move_task(task);
7669}
7670
7671static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
7672			    struct cgroup_subsys_state *old_css,
7673			    struct task_struct *task)
7674{
7675	/*
7676	 * cgroup_exit() is called in the copy_process() failure path.
7677	 * Ignore this case since the task hasn't ran yet, this avoids
7678	 * trying to poke a half freed task state from generic code.
7679	 */
7680	if (!(task->flags & PF_EXITING))
7681		return;
7682
7683	sched_move_task(task);
7684}
 
7685
7686#ifdef CONFIG_FAIR_GROUP_SCHED
7687static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
7688				struct cftype *cftype, u64 shareval)
7689{
 
 
7690	return sched_group_set_shares(css_tg(css), scale_load(shareval));
7691}
7692
7693static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
7694			       struct cftype *cft)
7695{
7696	struct task_group *tg = css_tg(css);
7697
7698	return (u64) scale_load_down(tg->shares);
7699}
7700
7701#ifdef CONFIG_CFS_BANDWIDTH
7702static DEFINE_MUTEX(cfs_constraints_mutex);
7703
7704const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
7705const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
 
 
7706
7707static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
7708
7709static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
7710{
7711	int i, ret = 0, runtime_enabled, runtime_was_enabled;
7712	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7713
7714	if (tg == &root_task_group)
7715		return -EINVAL;
7716
7717	/*
7718	 * Ensure we have at some amount of bandwidth every period.  This is
7719	 * to prevent reaching a state of large arrears when throttled via
7720	 * entity_tick() resulting in prolonged exit starvation.
7721	 */
7722	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
7723		return -EINVAL;
7724
7725	/*
7726	 * Likewise, bound things on the otherside by preventing insane quota
7727	 * periods.  This also allows us to normalize in computing quota
7728	 * feasibility.
7729	 */
7730	if (period > max_cfs_quota_period)
7731		return -EINVAL;
7732
 
 
 
 
 
 
 
 
 
 
 
7733	mutex_lock(&cfs_constraints_mutex);
7734	ret = __cfs_schedulable(tg, period, quota);
7735	if (ret)
7736		goto out_unlock;
7737
7738	runtime_enabled = quota != RUNTIME_INF;
7739	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
7740	/*
7741	 * If we need to toggle cfs_bandwidth_used, off->on must occur
7742	 * before making related changes, and on->off must occur afterwards
7743	 */
7744	if (runtime_enabled && !runtime_was_enabled)
7745		cfs_bandwidth_usage_inc();
7746	raw_spin_lock_irq(&cfs_b->lock);
7747	cfs_b->period = ns_to_ktime(period);
7748	cfs_b->quota = quota;
7749
7750	__refill_cfs_bandwidth_runtime(cfs_b);
7751	/* restart the period timer (if active) to handle new period expiry */
7752	if (runtime_enabled && cfs_b->timer_active) {
7753		/* force a reprogram */
7754		__start_cfs_bandwidth(cfs_b, true);
7755	}
7756	raw_spin_unlock_irq(&cfs_b->lock);
7757
7758	for_each_possible_cpu(i) {
7759		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
7760		struct rq *rq = cfs_rq->rq;
 
7761
7762		raw_spin_lock_irq(&rq->lock);
7763		cfs_rq->runtime_enabled = runtime_enabled;
7764		cfs_rq->runtime_remaining = 0;
7765
7766		if (cfs_rq->throttled)
7767			unthrottle_cfs_rq(cfs_rq);
7768		raw_spin_unlock_irq(&rq->lock);
7769	}
7770	if (runtime_was_enabled && !runtime_enabled)
7771		cfs_bandwidth_usage_dec();
7772out_unlock:
7773	mutex_unlock(&cfs_constraints_mutex);
 
7774
7775	return ret;
7776}
7777
7778int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
7779{
7780	u64 quota, period;
7781
7782	period = ktime_to_ns(tg->cfs_bandwidth.period);
7783	if (cfs_quota_us < 0)
7784		quota = RUNTIME_INF;
7785	else
7786		quota = (u64)cfs_quota_us * NSEC_PER_USEC;
 
 
7787
7788	return tg_set_cfs_bandwidth(tg, period, quota);
7789}
7790
7791long tg_get_cfs_quota(struct task_group *tg)
7792{
7793	u64 quota_us;
7794
7795	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
7796		return -1;
7797
7798	quota_us = tg->cfs_bandwidth.quota;
7799	do_div(quota_us, NSEC_PER_USEC);
7800
7801	return quota_us;
7802}
7803
7804int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
7805{
7806	u64 quota, period;
7807
 
 
 
7808	period = (u64)cfs_period_us * NSEC_PER_USEC;
7809	quota = tg->cfs_bandwidth.quota;
7810
7811	return tg_set_cfs_bandwidth(tg, period, quota);
7812}
7813
7814long tg_get_cfs_period(struct task_group *tg)
7815{
7816	u64 cfs_period_us;
7817
7818	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
7819	do_div(cfs_period_us, NSEC_PER_USEC);
7820
7821	return cfs_period_us;
7822}
7823
7824static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
7825				  struct cftype *cft)
7826{
7827	return tg_get_cfs_quota(css_tg(css));
7828}
7829
7830static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
7831				   struct cftype *cftype, s64 cfs_quota_us)
7832{
7833	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
7834}
7835
7836static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
7837				   struct cftype *cft)
7838{
7839	return tg_get_cfs_period(css_tg(css));
7840}
7841
7842static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
7843				    struct cftype *cftype, u64 cfs_period_us)
7844{
7845	return tg_set_cfs_period(css_tg(css), cfs_period_us);
7846}
7847
7848struct cfs_schedulable_data {
7849	struct task_group *tg;
7850	u64 period, quota;
7851};
7852
7853/*
7854 * normalize group quota/period to be quota/max_period
7855 * note: units are usecs
7856 */
7857static u64 normalize_cfs_quota(struct task_group *tg,
7858			       struct cfs_schedulable_data *d)
7859{
7860	u64 quota, period;
7861
7862	if (tg == d->tg) {
7863		period = d->period;
7864		quota = d->quota;
7865	} else {
7866		period = tg_get_cfs_period(tg);
7867		quota = tg_get_cfs_quota(tg);
7868	}
7869
7870	/* note: these should typically be equivalent */
7871	if (quota == RUNTIME_INF || quota == -1)
7872		return RUNTIME_INF;
7873
7874	return to_ratio(period, quota);
7875}
7876
7877static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
7878{
7879	struct cfs_schedulable_data *d = data;
7880	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7881	s64 quota = 0, parent_quota = -1;
7882
7883	if (!tg->parent) {
7884		quota = RUNTIME_INF;
7885	} else {
7886		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
7887
7888		quota = normalize_cfs_quota(tg, d);
7889		parent_quota = parent_b->hierarchal_quota;
7890
7891		/*
7892		 * ensure max(child_quota) <= parent_quota, inherit when no
7893		 * limit is set
 
7894		 */
7895		if (quota == RUNTIME_INF)
7896			quota = parent_quota;
7897		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
7898			return -EINVAL;
 
 
 
 
7899	}
7900	cfs_b->hierarchal_quota = quota;
7901
7902	return 0;
7903}
7904
7905static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
7906{
7907	int ret;
7908	struct cfs_schedulable_data data = {
7909		.tg = tg,
7910		.period = period,
7911		.quota = quota,
7912	};
7913
7914	if (quota != RUNTIME_INF) {
7915		do_div(data.period, NSEC_PER_USEC);
7916		do_div(data.quota, NSEC_PER_USEC);
7917	}
7918
7919	rcu_read_lock();
7920	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
7921	rcu_read_unlock();
7922
7923	return ret;
7924}
7925
7926static int cpu_stats_show(struct seq_file *sf, void *v)
7927{
7928	struct task_group *tg = css_tg(seq_css(sf));
7929	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7930
7931	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
7932	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
7933	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
7934
 
 
 
 
 
 
 
 
 
 
7935	return 0;
7936}
7937#endif /* CONFIG_CFS_BANDWIDTH */
7938#endif /* CONFIG_FAIR_GROUP_SCHED */
7939
7940#ifdef CONFIG_RT_GROUP_SCHED
7941static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
7942				struct cftype *cft, s64 val)
7943{
7944	return sched_group_set_rt_runtime(css_tg(css), val);
7945}
7946
7947static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
7948			       struct cftype *cft)
7949{
7950	return sched_group_rt_runtime(css_tg(css));
7951}
7952
7953static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
7954				    struct cftype *cftype, u64 rt_period_us)
7955{
7956	return sched_group_set_rt_period(css_tg(css), rt_period_us);
7957}
7958
7959static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
7960				   struct cftype *cft)
7961{
7962	return sched_group_rt_period(css_tg(css));
7963}
7964#endif /* CONFIG_RT_GROUP_SCHED */
7965
7966static struct cftype cpu_files[] = {
7967#ifdef CONFIG_FAIR_GROUP_SCHED
7968	{
7969		.name = "shares",
7970		.read_u64 = cpu_shares_read_u64,
7971		.write_u64 = cpu_shares_write_u64,
7972	},
7973#endif
7974#ifdef CONFIG_CFS_BANDWIDTH
7975	{
7976		.name = "cfs_quota_us",
7977		.read_s64 = cpu_cfs_quota_read_s64,
7978		.write_s64 = cpu_cfs_quota_write_s64,
7979	},
7980	{
7981		.name = "cfs_period_us",
7982		.read_u64 = cpu_cfs_period_read_u64,
7983		.write_u64 = cpu_cfs_period_write_u64,
7984	},
7985	{
7986		.name = "stat",
7987		.seq_show = cpu_stats_show,
7988	},
7989#endif
7990#ifdef CONFIG_RT_GROUP_SCHED
7991	{
7992		.name = "rt_runtime_us",
7993		.read_s64 = cpu_rt_runtime_read,
7994		.write_s64 = cpu_rt_runtime_write,
7995	},
7996	{
7997		.name = "rt_period_us",
7998		.read_u64 = cpu_rt_period_read_uint,
7999		.write_u64 = cpu_rt_period_write_uint,
8000	},
8001#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8002	{ }	/* terminate */
8003};
8004
8005struct cgroup_subsys cpu_cgrp_subsys = {
8006	.css_alloc	= cpu_cgroup_css_alloc,
8007	.css_free	= cpu_cgroup_css_free,
8008	.css_online	= cpu_cgroup_css_online,
8009	.css_offline	= cpu_cgroup_css_offline,
 
 
 
8010	.can_attach	= cpu_cgroup_can_attach,
8011	.attach		= cpu_cgroup_attach,
8012	.exit		= cpu_cgroup_exit,
8013	.base_cftypes	= cpu_files,
8014	.early_init	= 1,
 
8015};
8016
8017#endif	/* CONFIG_CGROUP_SCHED */
8018
8019void dump_cpu_task(int cpu)
8020{
8021	pr_info("Task dump for CPU %d:\n", cpu);
8022	sched_show_task(cpu_curr(cpu));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8023}