1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   4 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   5 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
   6 *
   7 *  NOHZ implementation for low and high resolution timers
   8 *
   9 *  Started by: Thomas Gleixner and Ingo Molnar
  10 */
  11#include <linux/cpu.h>
  12#include <linux/err.h>
  13#include <linux/hrtimer.h>
  14#include <linux/interrupt.h>
  15#include <linux/kernel_stat.h>
  16#include <linux/percpu.h>
  17#include <linux/nmi.h>
  18#include <linux/profile.h>
  19#include <linux/sched/signal.h>
  20#include <linux/sched/clock.h>
  21#include <linux/sched/stat.h>
  22#include <linux/sched/nohz.h>
  23#include <linux/sched/loadavg.h>
  24#include <linux/module.h>
  25#include <linux/irq_work.h>
  26#include <linux/posix-timers.h>
  27#include <linux/context_tracking.h>
  28#include <linux/mm.h>
  29
  30#include <asm/irq_regs.h>
  31
  32#include "tick-internal.h"
  33
  34#include <trace/events/timer.h>
  35
  36/*
  37 * Per-CPU nohz control structure
  38 */
  39static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  40
  41struct tick_sched *tick_get_tick_sched(int cpu)
  42{
  43	return &per_cpu(tick_cpu_sched, cpu);
  44}
  45
  46#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
  47/*
  48 * The time when the last jiffy update happened. Write access must hold
  49 * jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a
  50 * consistent view of jiffies and last_jiffies_update.
  51 */
  52static ktime_t last_jiffies_update;
  53
  54/*
  55 * Must be called with interrupts disabled !
  56 */
  57static void tick_do_update_jiffies64(ktime_t now)
  58{
  59	unsigned long ticks = 1;
  60	ktime_t delta, nextp;
  61
  62	/*
  63	 * 64-bit can do a quick check without holding the jiffies lock and
  64	 * without looking at the sequence count. The smp_load_acquire()
  65	 * pairs with the update done later in this function.
  66	 *
  67	 * 32-bit cannot do that because the store of 'tick_next_period'
  68	 * consists of two 32-bit stores, and the first store could be
  69	 * moved by the CPU to a random point in the future.
  70	 */
  71	if (IS_ENABLED(CONFIG_64BIT)) {
  72		if (ktime_before(now, smp_load_acquire(&tick_next_period)))
  73			return;
  74	} else {
  75		unsigned int seq;
  76
  77		/*
  78		 * Avoid contention on 'jiffies_lock' and protect the quick
  79		 * check with the sequence count.
  80		 */
  81		do {
  82			seq = read_seqcount_begin(&jiffies_seq);
  83			nextp = tick_next_period;
  84		} while (read_seqcount_retry(&jiffies_seq, seq));
  85
  86		if (ktime_before(now, nextp))
  87			return;
  88	}
  89
  90	/* Quick check failed, i.e. update is required. */
  91	raw_spin_lock(&jiffies_lock);
  92	/*
  93	 * Re-evaluate with the lock held. Another CPU might have done the
  94	 * update already.
  95	 */
  96	if (ktime_before(now, tick_next_period)) {
  97		raw_spin_unlock(&jiffies_lock);
  98		return;
  99	}
 100
 101	write_seqcount_begin(&jiffies_seq);
 102
 103	delta = ktime_sub(now, tick_next_period);
 104	if (unlikely(delta >= TICK_NSEC)) {
 105		/* Slow path for long idle sleep times */
 106		s64 incr = TICK_NSEC;
 107
 108		ticks += ktime_divns(delta, incr);
 109
 110		last_jiffies_update = ktime_add_ns(last_jiffies_update,
 111						   incr * ticks);
 112	} else {
 113		last_jiffies_update = ktime_add_ns(last_jiffies_update,
 114						   TICK_NSEC);
 115	}
 116
 117	/* Advance jiffies to complete the 'jiffies_seq' protected job */
 118	jiffies_64 += ticks;
 119
 120	/* Keep the tick_next_period variable up to date */
 
 
 121	nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC);
 122
 123	if (IS_ENABLED(CONFIG_64BIT)) {
 124		/*
 125		 * Pairs with smp_load_acquire() in the lockless quick
 126		 * check above, and ensures that the update to 'jiffies_64' is
 127		 * not reordered vs. the store to 'tick_next_period', neither
 128		 * by the compiler nor by the CPU.
 129		 */
 130		smp_store_release(&tick_next_period, nextp);
 131	} else {
 132		/*
 133		 * A plain store is good enough on 32-bit, as the quick check
 134		 * above is protected by the sequence count.
 135		 */
 136		tick_next_period = nextp;
 137	}
 138
 139	/*
 140	 * Release the sequence count. calc_global_load() below is not
 141	 * protected by it, but 'jiffies_lock' needs to be held to prevent
 142	 * concurrent invocations.
 143	 */
 144	write_seqcount_end(&jiffies_seq);
 145
 146	calc_global_load();
 147
 148	raw_spin_unlock(&jiffies_lock);
 149	update_wall_time();
 150}
 151
 152/*
 153 * Initialize and return retrieve the jiffies update.
 154 */
 155static ktime_t tick_init_jiffy_update(void)
 156{
 157	ktime_t period;
 158
 159	raw_spin_lock(&jiffies_lock);
 160	write_seqcount_begin(&jiffies_seq);
 161
 162	/* Have we started the jiffies update yet ? */
 163	if (last_jiffies_update == 0) {
 164		u32 rem;
 165
 166		/*
 167		 * Ensure that the tick is aligned to a multiple of
 168		 * TICK_NSEC.
 169		 */
 170		div_u64_rem(tick_next_period, TICK_NSEC, &rem);
 171		if (rem)
 172			tick_next_period += TICK_NSEC - rem;
 173
 174		last_jiffies_update = tick_next_period;
 175	}
 176	period = last_jiffies_update;
 177
 178	write_seqcount_end(&jiffies_seq);
 179	raw_spin_unlock(&jiffies_lock);
 180
 181	return period;
 182}
 183
 184#define MAX_STALLED_JIFFIES 5
 185
 186static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
 187{
 188	int cpu = smp_processor_id();
 189
 190#ifdef CONFIG_NO_HZ_COMMON
 191	/*
 192	 * Check if the do_timer duty was dropped. We don't care about
 193	 * concurrency: This happens only when the CPU in charge went
 194	 * into a long sleep. If two CPUs happen to assign themselves to
 195	 * this duty, then the jiffies update is still serialized by
 196	 * 'jiffies_lock'.
 197	 *
 198	 * If nohz_full is enabled, this should not happen because the
 199	 * 'tick_do_timer_cpu' CPU never relinquishes.
 200	 */
 201	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
 202#ifdef CONFIG_NO_HZ_FULL
 203		WARN_ON_ONCE(tick_nohz_full_running);
 204#endif
 205		tick_do_timer_cpu = cpu;
 206	}
 207#endif
 208
 209	/* Check if jiffies need an update */
 210	if (tick_do_timer_cpu == cpu)
 211		tick_do_update_jiffies64(now);
 212
 213	/*
 214	 * If the jiffies update stalled for too long (timekeeper in stop_machine()
 215	 * or VMEXIT'ed for several msecs), force an update.
 216	 */
 217	if (ts->last_tick_jiffies != jiffies) {
 218		ts->stalled_jiffies = 0;
 219		ts->last_tick_jiffies = READ_ONCE(jiffies);
 220	} else {
 221		if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {
 222			tick_do_update_jiffies64(now);
 223			ts->stalled_jiffies = 0;
 224			ts->last_tick_jiffies = READ_ONCE(jiffies);
 225		}
 226	}
 227
 228	if (ts->inidle)
 229		ts->got_idle_tick = 1;
 230}
 231
 232static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 233{
 234#ifdef CONFIG_NO_HZ_COMMON
 235	/*
 236	 * When we are idle and the tick is stopped, we have to touch
 237	 * the watchdog as we might not schedule for a really long
 238	 * time. This happens on completely idle SMP systems while
 239	 * waiting on the login prompt. We also increment the "start of
 240	 * idle" jiffy stamp so the idle accounting adjustment we do
 241	 * when we go busy again does not account too many ticks.
 242	 */
 243	if (ts->tick_stopped) {
 244		touch_softlockup_watchdog_sched();
 245		if (is_idle_task(current))
 246			ts->idle_jiffies++;
 247		/*
 248		 * In case the current tick fired too early past its expected
 249		 * expiration, make sure we don't bypass the next clock reprogramming
 250		 * to the same deadline.
 251		 */
 252		ts->next_tick = 0;
 253	}
 254#endif
 255	update_process_times(user_mode(regs));
 256	profile_tick(CPU_PROFILING);
 257}
 258#endif
 259
 260#ifdef CONFIG_NO_HZ_FULL
 261cpumask_var_t tick_nohz_full_mask;
 262EXPORT_SYMBOL_GPL(tick_nohz_full_mask);
 263bool tick_nohz_full_running;
 264EXPORT_SYMBOL_GPL(tick_nohz_full_running);
 265static atomic_t tick_dep_mask;
 266
 267static bool check_tick_dependency(atomic_t *dep)
 268{
 269	int val = atomic_read(dep);
 270
 271	if (val & TICK_DEP_MASK_POSIX_TIMER) {
 272		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
 273		return true;
 274	}
 275
 276	if (val & TICK_DEP_MASK_PERF_EVENTS) {
 277		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
 278		return true;
 279	}
 280
 281	if (val & TICK_DEP_MASK_SCHED) {
 282		trace_tick_stop(0, TICK_DEP_MASK_SCHED);
 283		return true;
 284	}
 285
 286	if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
 287		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
 288		return true;
 289	}
 290
 291	if (val & TICK_DEP_MASK_RCU) {
 292		trace_tick_stop(0, TICK_DEP_MASK_RCU);
 293		return true;
 294	}
 295
 296	if (val & TICK_DEP_MASK_RCU_EXP) {
 297		trace_tick_stop(0, TICK_DEP_MASK_RCU_EXP);
 298		return true;
 299	}
 300
 301	return false;
 302}
 303
 304static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
 305{
 306	lockdep_assert_irqs_disabled();
 307
 308	if (unlikely(!cpu_online(cpu)))
 309		return false;
 310
 311	if (check_tick_dependency(&tick_dep_mask))
 312		return false;
 313
 314	if (check_tick_dependency(&ts->tick_dep_mask))
 315		return false;
 316
 317	if (check_tick_dependency(&current->tick_dep_mask))
 318		return false;
 319
 320	if (check_tick_dependency(&current->signal->tick_dep_mask))
 321		return false;
 322
 323	return true;
 324}
 325
 326static void nohz_full_kick_func(struct irq_work *work)
 327{
 328	/* Empty, the tick restart happens on tick_nohz_irq_exit() */
 329}
 330
 331static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) =
 332	IRQ_WORK_INIT_HARD(nohz_full_kick_func);
 333
 334/*
 335 * Kick this CPU if it's full dynticks in order to force it to
 336 * re-evaluate its dependency on the tick and restart it if necessary.
 337 * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
 338 * is NMI safe.
 339 */
 340static void tick_nohz_full_kick(void)
 341{
 342	if (!tick_nohz_full_cpu(smp_processor_id()))
 343		return;
 344
 345	irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
 346}
 347
 348/*
 349 * Kick the CPU if it's full dynticks in order to force it to
 350 * re-evaluate its dependency on the tick and restart it if necessary.
 351 */
 352void tick_nohz_full_kick_cpu(int cpu)
 353{
 354	if (!tick_nohz_full_cpu(cpu))
 355		return;
 356
 357	irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
 358}
 359
 360static void tick_nohz_kick_task(struct task_struct *tsk)
 361{
 362	int cpu;
 363
 364	/*
 365	 * If the task is not running, run_posix_cpu_timers()
 366	 * has nothing to elapse, and an IPI can then be optimized out.
 367	 *
 368	 * activate_task()                      STORE p->tick_dep_mask
 369	 *   STORE p->on_rq
 370	 * __schedule() (switch to task 'p')    smp_mb() (atomic_fetch_or())
 371	 *   LOCK rq->lock                      LOAD p->on_rq
 372	 *   smp_mb__after_spin_lock()
 373	 *   tick_nohz_task_switch()
 374	 *     LOAD p->tick_dep_mask
 375	 */
 376	if (!sched_task_on_rq(tsk))
 377		return;
 378
 379	/*
 380	 * If the task concurrently migrates to another CPU,
 381	 * we guarantee it sees the new tick dependency upon
 382	 * schedule.
 383	 *
 384	 * set_task_cpu(p, cpu);
 385	 *   STORE p->cpu = @cpu
 386	 * __schedule() (switch to task 'p')
 387	 *   LOCK rq->lock
 388	 *   smp_mb__after_spin_lock()          STORE p->tick_dep_mask
 389	 *   tick_nohz_task_switch()            smp_mb() (atomic_fetch_or())
 390	 *      LOAD p->tick_dep_mask           LOAD p->cpu
 391	 */
 392	cpu = task_cpu(tsk);
 393
 394	preempt_disable();
 395	if (cpu_online(cpu))
 396		tick_nohz_full_kick_cpu(cpu);
 397	preempt_enable();
 398}
 399
 400/*
 401 * Kick all full dynticks CPUs in order to force these to re-evaluate
 402 * their dependency on the tick and restart it if necessary.
 403 */
 404static void tick_nohz_full_kick_all(void)
 405{
 406	int cpu;
 407
 408	if (!tick_nohz_full_running)
 409		return;
 410
 411	preempt_disable();
 412	for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
 413		tick_nohz_full_kick_cpu(cpu);
 414	preempt_enable();
 415}
 416
 417static void tick_nohz_dep_set_all(atomic_t *dep,
 418				  enum tick_dep_bits bit)
 419{
 420	int prev;
 421
 422	prev = atomic_fetch_or(BIT(bit), dep);
 423	if (!prev)
 424		tick_nohz_full_kick_all();
 425}
 426
 427/*
 428 * Set a global tick dependency. Used by perf events that rely on freq and
 429 * unstable clocks.
 430 */
 431void tick_nohz_dep_set(enum tick_dep_bits bit)
 432{
 433	tick_nohz_dep_set_all(&tick_dep_mask, bit);
 434}
 435
 436void tick_nohz_dep_clear(enum tick_dep_bits bit)
 437{
 438	atomic_andnot(BIT(bit), &tick_dep_mask);
 439}
 440
 441/*
 442 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
 443 * manage event-throttling.
 444 */
 445void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
 446{
 447	int prev;
 448	struct tick_sched *ts;
 449
 450	ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 451
 452	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
 453	if (!prev) {
 454		preempt_disable();
 455		/* Perf needs local kick that is NMI safe */
 456		if (cpu == smp_processor_id()) {
 457			tick_nohz_full_kick();
 458		} else {
 459			/* Remote IRQ work not NMI-safe */
 460			if (!WARN_ON_ONCE(in_nmi()))
 461				tick_nohz_full_kick_cpu(cpu);
 462		}
 463		preempt_enable();
 464	}
 465}
 466EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
 467
 468void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
 469{
 470	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 471
 472	atomic_andnot(BIT(bit), &ts->tick_dep_mask);
 473}
 474EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
 475
 476/*
 477 * Set a per-task tick dependency. RCU needs this. Also posix CPU timers
 478 * in order to elapse per task timers.
 479 */
 480void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
 481{
 482	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask))
 483		tick_nohz_kick_task(tsk);
 484}
 485EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
 486
 487void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
 488{
 489	atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
 490}
 491EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
 492
 493/*
 494 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
 495 * per process timers.
 496 */
 497void tick_nohz_dep_set_signal(struct task_struct *tsk,
 498			      enum tick_dep_bits bit)
 499{
 500	int prev;
 501	struct signal_struct *sig = tsk->signal;
 502
 503	prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask);
 504	if (!prev) {
 505		struct task_struct *t;
 506
 507		lockdep_assert_held(&tsk->sighand->siglock);
 508		__for_each_thread(sig, t)
 509			tick_nohz_kick_task(t);
 510	}
 511}
 512
 513void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
 514{
 515	atomic_andnot(BIT(bit), &sig->tick_dep_mask);
 516}
 517
 518/*
 519 * Re-evaluate the need for the tick as we switch the current task.
 520 * It might need the tick due to per task/process properties:
 521 * perf events, posix CPU timers, ...
 522 */
 523void __tick_nohz_task_switch(void)
 524{
 525	struct tick_sched *ts;
 526
 527	if (!tick_nohz_full_cpu(smp_processor_id()))
 528		return;
 529
 530	ts = this_cpu_ptr(&tick_cpu_sched);
 531
 532	if (ts->tick_stopped) {
 533		if (atomic_read(&current->tick_dep_mask) ||
 534		    atomic_read(&current->signal->tick_dep_mask))
 535			tick_nohz_full_kick();
 536	}
 537}
 538
 539/* Get the boot-time nohz CPU list from the kernel parameters. */
 540void __init tick_nohz_full_setup(cpumask_var_t cpumask)
 541{
 542	alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
 543	cpumask_copy(tick_nohz_full_mask, cpumask);
 544	tick_nohz_full_running = true;
 545}
 
 546
 547bool tick_nohz_cpu_hotpluggable(unsigned int cpu)
 548{
 549	/*
 550	 * The 'tick_do_timer_cpu' CPU handles housekeeping duty (unbound
 551	 * timers, workqueues, timekeeping, ...) on behalf of full dynticks
 552	 * CPUs. It must remain online when nohz full is enabled.
 553	 */
 554	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
 555		return false;
 556	return true;
 557}
 558
 559static int tick_nohz_cpu_down(unsigned int cpu)
 560{
 561	return tick_nohz_cpu_hotpluggable(cpu) ? 0 : -EBUSY;
 562}
 563
 564void __init tick_nohz_init(void)
 565{
 566	int cpu, ret;
 567
 568	if (!tick_nohz_full_running)
 569		return;
 570
 571	/*
 572	 * Full dynticks uses IRQ work to drive the tick rescheduling on safe
 573	 * locking contexts. But then we need IRQ work to raise its own
 574	 * interrupts to avoid circular dependency on the tick.
 575	 */
 576	if (!arch_irq_work_has_interrupt()) {
 577		pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support IRQ work self-IPIs\n");
 578		cpumask_clear(tick_nohz_full_mask);
 579		tick_nohz_full_running = false;
 580		return;
 581	}
 582
 583	if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
 584			!IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
 585		cpu = smp_processor_id();
 586
 587		if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
 588			pr_warn("NO_HZ: Clearing %d from nohz_full range "
 589				"for timekeeping\n", cpu);
 590			cpumask_clear_cpu(cpu, tick_nohz_full_mask);
 591		}
 592	}
 593
 594	for_each_cpu(cpu, tick_nohz_full_mask)
 595		ct_cpu_track_user(cpu);
 596
 597	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 598					"kernel/nohz:predown", NULL,
 599					tick_nohz_cpu_down);
 600	WARN_ON(ret < 0);
 601	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 602		cpumask_pr_args(tick_nohz_full_mask));
 603}
 604#endif
 605
 606/*
 607 * NOHZ - aka dynamic tick functionality
 608 */
 609#ifdef CONFIG_NO_HZ_COMMON
 610/*
 611 * NO HZ enabled ?
 612 */
 613bool tick_nohz_enabled __read_mostly  = true;
 614unsigned long tick_nohz_active  __read_mostly;
 615/*
 616 * Enable / Disable tickless mode
 617 */
 618static int __init setup_tick_nohz(char *str)
 619{
 620	return (kstrtobool(str, &tick_nohz_enabled) == 0);
 621}
 622
 623__setup("nohz=", setup_tick_nohz);
 624
 625bool tick_nohz_tick_stopped(void)
 626{
 627	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 628
 629	return ts->tick_stopped;
 630}
 631
 632bool tick_nohz_tick_stopped_cpu(int cpu)
 633{
 634	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 635
 636	return ts->tick_stopped;
 637}
 638
 639/**
 640 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 641 *
 642 * Called from interrupt entry when the CPU was idle
 643 *
 644 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 645 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 646 * value. We do this unconditionally on any CPU, as we don't know whether the
 647 * CPU, which has the update task assigned, is in a long sleep.
 648 */
 649static void tick_nohz_update_jiffies(ktime_t now)
 650{
 651	unsigned long flags;
 652
 653	__this_cpu_write(tick_cpu_sched.idle_waketime, now);
 654
 655	local_irq_save(flags);
 656	tick_do_update_jiffies64(now);
 657	local_irq_restore(flags);
 658
 659	touch_softlockup_watchdog_sched();
 660}
 661
 662static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
 
 
 
 
 663{
 664	ktime_t delta;
 665
 666	if (WARN_ON_ONCE(!ts->idle_active))
 667		return;
 
 
 
 
 
 
 668
 669	delta = ktime_sub(now, ts->idle_entrytime);
 
 670
 671	write_seqcount_begin(&ts->idle_sleeptime_seq);
 672	if (nr_iowait_cpu(smp_processor_id()) > 0)
 673		ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
 674	else
 675		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 676
 677	ts->idle_entrytime = now;
 
 
 678	ts->idle_active = 0;
 679	write_seqcount_end(&ts->idle_sleeptime_seq);
 680
 681	sched_clock_idle_wakeup_event();
 682}
 683
 684static void tick_nohz_start_idle(struct tick_sched *ts)
 685{
 686	write_seqcount_begin(&ts->idle_sleeptime_seq);
 687	ts->idle_entrytime = ktime_get();
 688	ts->idle_active = 1;
 689	write_seqcount_end(&ts->idle_sleeptime_seq);
 690
 691	sched_clock_idle_sleep_event();
 692}
 693
 694static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,
 695				 bool compute_delta, u64 *last_update_time)
 696{
 697	ktime_t now, idle;
 698	unsigned int seq;
 699
 700	if (!tick_nohz_active)
 701		return -1;
 702
 703	now = ktime_get();
 704	if (last_update_time)
 705		*last_update_time = ktime_to_us(now);
 706
 707	do {
 708		seq = read_seqcount_begin(&ts->idle_sleeptime_seq);
 709
 710		if (ts->idle_active && compute_delta) {
 711			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 712
 713			idle = ktime_add(*sleeptime, delta);
 714		} else {
 715			idle = *sleeptime;
 716		}
 717	} while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));
 718
 719	return ktime_to_us(idle);
 720
 721}
 722
 723/**
 724 * get_cpu_idle_time_us - get the total idle time of a CPU
 725 * @cpu: CPU number to query
 726 * @last_update_time: variable to store update time in. Do not update
 727 * counters if NULL.
 728 *
 729 * Return the cumulative idle time (since boot) for a given
 730 * CPU, in microseconds. Note that this is partially broken due to
 731 * the counter of iowait tasks that can be remotely updated without
 732 * any synchronization. Therefore it is possible to observe backward
 733 * values within two consecutive reads.
 734 *
 735 * This time is measured via accounting rather than sampling,
 736 * and is as accurate as ktime_get() is.
 737 *
 738 * This function returns -1 if NOHZ is not enabled.
 739 */
 740u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 741{
 742	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 743
 744	return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,
 745				     !nr_iowait_cpu(cpu), last_update_time);
 746}
 747EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 748
 749/**
 750 * get_cpu_iowait_time_us - get the total iowait time of a CPU
 751 * @cpu: CPU number to query
 752 * @last_update_time: variable to store update time in. Do not update
 753 * counters if NULL.
 754 *
 755 * Return the cumulative iowait time (since boot) for a given
 756 * CPU, in microseconds. Note this is partially broken due to
 757 * the counter of iowait tasks that can be remotely updated without
 758 * any synchronization. Therefore it is possible to observe backward
 759 * values within two consecutive reads.
 760 *
 761 * This time is measured via accounting rather than sampling,
 762 * and is as accurate as ktime_get() is.
 763 *
 764 * This function returns -1 if NOHZ is not enabled.
 765 */
 766u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 767{
 768	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 
 
 
 
 769
 770	return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,
 771				     nr_iowait_cpu(cpu), last_update_time);
 
 
 
 
 
 
 
 
 
 
 
 
 
 772}
 773EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 774
 775static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 776{
 777	hrtimer_cancel(&ts->sched_timer);
 778	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 779
 780	/* Forward the time to expire in the future */
 781	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
 782
 783	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 784		hrtimer_start_expires(&ts->sched_timer,
 785				      HRTIMER_MODE_ABS_PINNED_HARD);
 786	} else {
 787		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
 788	}
 789
 790	/*
 791	 * Reset to make sure the next tick stop doesn't get fooled by past
 792	 * cached clock deadline.
 793	 */
 794	ts->next_tick = 0;
 795}
 796
 797static inline bool local_timer_softirq_pending(void)
 798{
 799	return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
 800}
 801
 802static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 803{
 804	u64 basemono, next_tick, delta, expires;
 805	unsigned long basejiff;
 806	unsigned int seq;
 807
 808	/* Read jiffies and the time when jiffies were updated last */
 809	do {
 810		seq = read_seqcount_begin(&jiffies_seq);
 811		basemono = last_jiffies_update;
 812		basejiff = jiffies;
 813	} while (read_seqcount_retry(&jiffies_seq, seq));
 814	ts->last_jiffies = basejiff;
 815	ts->timer_expires_base = basemono;
 816
 817	/*
 818	 * Keep the periodic tick, when RCU, architecture or irq_work
 819	 * requests it.
 820	 * Aside of that, check whether the local timer softirq is
 821	 * pending. If so, its a bad idea to call get_next_timer_interrupt(),
 822	 * because there is an already expired timer, so it will request
 823	 * immediate expiry, which rearms the hardware timer with a
 824	 * minimal delta, which brings us back to this place
 825	 * immediately. Lather, rinse and repeat...
 826	 */
 827	if (rcu_needs_cpu() || arch_needs_cpu() ||
 828	    irq_work_needs_cpu() || local_timer_softirq_pending()) {
 829		next_tick = basemono + TICK_NSEC;
 830	} else {
 831		/*
 832		 * Get the next pending timer. If high resolution
 833		 * timers are enabled this only takes the timer wheel
 834		 * timers into account. If high resolution timers are
 835		 * disabled this also looks at the next expiring
 836		 * hrtimer.
 837		 */
 838		next_tick = get_next_timer_interrupt(basejiff, basemono);
 839		ts->next_timer = next_tick;
 
 
 840	}
 841
 842	/* Make sure next_tick is never before basemono! */
 843	if (WARN_ON_ONCE(basemono > next_tick))
 844		next_tick = basemono;
 845
 846	/*
 847	 * If the tick is due in the next period, keep it ticking or
 848	 * force prod the timer.
 849	 */
 850	delta = next_tick - basemono;
 851	if (delta <= (u64)TICK_NSEC) {
 852		/*
 853		 * Tell the timer code that the base is not idle, i.e. undo
 854		 * the effect of get_next_timer_interrupt():
 855		 */
 856		timer_clear_idle();
 857		/*
 858		 * We've not stopped the tick yet, and there's a timer in the
 859		 * next period, so no point in stopping it either, bail.
 860		 */
 861		if (!ts->tick_stopped) {
 862			ts->timer_expires = 0;
 863			goto out;
 864		}
 865	}
 866
 867	/*
 868	 * If this CPU is the one which had the do_timer() duty last, we limit
 869	 * the sleep time to the timekeeping 'max_deferment' value.
 870	 * Otherwise we can sleep as long as we want.
 871	 */
 872	delta = timekeeping_max_deferment();
 873	if (cpu != tick_do_timer_cpu &&
 874	    (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
 875		delta = KTIME_MAX;
 876
 877	/* Calculate the next expiry time */
 878	if (delta < (KTIME_MAX - basemono))
 879		expires = basemono + delta;
 880	else
 881		expires = KTIME_MAX;
 882
 883	ts->timer_expires = min_t(u64, expires, next_tick);
 884
 885out:
 886	return ts->timer_expires;
 887}
 888
 889static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
 890{
 891	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 892	u64 basemono = ts->timer_expires_base;
 893	u64 expires = ts->timer_expires;
 
 894
 895	/* Make sure we won't be trying to stop it twice in a row. */
 896	ts->timer_expires_base = 0;
 897
 898	/*
 899	 * If this CPU is the one which updates jiffies, then give up
 900	 * the assignment and let it be taken by the CPU which runs
 901	 * the tick timer next, which might be this CPU as well. If we
 902	 * don't drop this here, the jiffies might be stale and
 903	 * do_timer() never gets invoked. Keep track of the fact that it
 904	 * was the one which had the do_timer() duty last.
 905	 */
 906	if (cpu == tick_do_timer_cpu) {
 907		tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 908		ts->do_timer_last = 1;
 909	} else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
 910		ts->do_timer_last = 0;
 911	}
 912
 913	/* Skip reprogram of event if it's not changed */
 914	if (ts->tick_stopped && (expires == ts->next_tick)) {
 915		/* Sanity check: make sure clockevent is actually programmed */
 916		if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
 917			return;
 918
 919		WARN_ON_ONCE(1);
 920		printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
 921			    basemono, ts->next_tick, dev->next_event,
 922			    hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
 923	}
 924
 925	/*
 926	 * tick_nohz_stop_tick() can be called several times before
 927	 * tick_nohz_restart_sched_tick() is called. This happens when
 928	 * interrupts arrive which do not cause a reschedule. In the first
 929	 * call we save the current tick time, so we can restart the
 930	 * scheduler tick in tick_nohz_restart_sched_tick().
 931	 */
 932	if (!ts->tick_stopped) {
 933		calc_load_nohz_start();
 934		quiet_vmstat();
 935
 936		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
 937		ts->tick_stopped = 1;
 938		trace_tick_stop(1, TICK_DEP_MASK_NONE);
 939	}
 940
 941	ts->next_tick = expires;
 942
 943	/*
 944	 * If the expiration time == KTIME_MAX, then we simply stop
 945	 * the tick timer.
 946	 */
 947	if (unlikely(expires == KTIME_MAX)) {
 948		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 949			hrtimer_cancel(&ts->sched_timer);
 950		else
 951			tick_program_event(KTIME_MAX, 1);
 952		return;
 953	}
 954
 955	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 956		hrtimer_start(&ts->sched_timer, expires,
 957			      HRTIMER_MODE_ABS_PINNED_HARD);
 958	} else {
 959		hrtimer_set_expires(&ts->sched_timer, expires);
 960		tick_program_event(expires, 1);
 961	}
 962}
 963
 964static void tick_nohz_retain_tick(struct tick_sched *ts)
 965{
 966	ts->timer_expires_base = 0;
 967}
 968
 969#ifdef CONFIG_NO_HZ_FULL
 970static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
 971{
 972	if (tick_nohz_next_event(ts, cpu))
 973		tick_nohz_stop_tick(ts, cpu);
 974	else
 975		tick_nohz_retain_tick(ts);
 976}
 977#endif /* CONFIG_NO_HZ_FULL */
 978
 979static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 980{
 981	/* Update jiffies first */
 982	tick_do_update_jiffies64(now);
 983
 984	/*
 985	 * Clear the timer idle flag, so we avoid IPIs on remote queueing and
 986	 * the clock forward checks in the enqueue path:
 987	 */
 988	timer_clear_idle();
 989
 990	calc_load_nohz_stop();
 991	touch_softlockup_watchdog_sched();
 992
 993	/* Cancel the scheduled timer and restore the tick: */
 
 994	ts->tick_stopped  = 0;
 995	tick_nohz_restart(ts, now);
 996}
 997
 998static void __tick_nohz_full_update_tick(struct tick_sched *ts,
 999					 ktime_t now)
1000{
1001#ifdef CONFIG_NO_HZ_FULL
1002	int cpu = smp_processor_id();
1003
1004	if (can_stop_full_tick(cpu, ts))
1005		tick_nohz_stop_sched_tick(ts, cpu);
1006	else if (ts->tick_stopped)
1007		tick_nohz_restart_sched_tick(ts, now);
1008#endif
1009}
1010
1011static void tick_nohz_full_update_tick(struct tick_sched *ts)
1012{
1013	if (!tick_nohz_full_cpu(smp_processor_id()))
1014		return;
1015
1016	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
1017		return;
1018
1019	__tick_nohz_full_update_tick(ts, ktime_get());
1020}
1021
1022/*
1023 * A pending softirq outside an IRQ (or softirq disabled section) context
1024 * should be waiting for ksoftirqd to handle it. Therefore we shouldn't
1025 * reach this code due to the need_resched() early check in can_stop_idle_tick().
1026 *
1027 * However if we are between CPUHP_AP_SMPBOOT_THREADS and CPU_TEARDOWN_CPU on the
1028 * cpu_down() process, softirqs can still be raised while ksoftirqd is parked,
1029 * triggering the code below, since wakep_softirqd() is ignored.
1030 *
1031 */
1032static bool report_idle_softirq(void)
1033{
1034	static int ratelimit;
1035	unsigned int pending = local_softirq_pending();
1036
1037	if (likely(!pending))
1038		return false;
1039
1040	/* Some softirqs claim to be safe against hotplug and ksoftirqd parking */
1041	if (!cpu_active(smp_processor_id())) {
1042		pending &= ~SOFTIRQ_HOTPLUG_SAFE_MASK;
1043		if (!pending)
1044			return false;
1045	}
1046
1047	if (ratelimit >= 10)
1048		return false;
1049
1050	/* On RT, softirq handling may be waiting on some lock */
1051	if (local_bh_blocked())
1052		return false;
1053
1054	pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n",
1055		pending);
1056	ratelimit++;
1057
1058	return true;
1059}
1060
1061static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
1062{
1063	/*
1064	 * If this CPU is offline and it is the one which updates
1065	 * jiffies, then give up the assignment and let it be taken by
1066	 * the CPU which runs the tick timer next. If we don't drop
1067	 * this here, the jiffies might be stale and do_timer() never
1068	 * gets invoked.
1069	 */
1070	if (unlikely(!cpu_online(cpu))) {
1071		if (cpu == tick_do_timer_cpu)
1072			tick_do_timer_cpu = TICK_DO_TIMER_NONE;
1073		/*
1074		 * Make sure the CPU doesn't get fooled by obsolete tick
1075		 * deadline if it comes back online later.
1076		 */
1077		ts->next_tick = 0;
1078		return false;
1079	}
1080
1081	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
1082		return false;
1083
1084	if (need_resched())
1085		return false;
1086
1087	if (unlikely(report_idle_softirq()))
 
 
 
 
 
 
 
 
1088		return false;
 
1089
1090	if (tick_nohz_full_enabled()) {
1091		/*
1092		 * Keep the tick alive to guarantee timekeeping progression
1093		 * if there are full dynticks CPUs around
1094		 */
1095		if (tick_do_timer_cpu == cpu)
1096			return false;
1097
1098		/* Should not happen for nohz-full */
1099		if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
1100			return false;
1101	}
1102
1103	return true;
1104}
1105
1106/**
1107 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
1108 *
1109 * When the next event is more than a tick into the future, stop the idle tick
1110 */
1111void tick_nohz_idle_stop_tick(void)
1112{
1113	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1114	int cpu = smp_processor_id();
1115	ktime_t expires;
 
1116
1117	/*
1118	 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
1119	 * tick timer expiration time is known already.
1120	 */
1121	if (ts->timer_expires_base)
1122		expires = ts->timer_expires;
1123	else if (can_stop_idle_tick(cpu, ts))
1124		expires = tick_nohz_next_event(ts, cpu);
1125	else
1126		return;
1127
1128	ts->idle_calls++;
1129
1130	if (expires > 0LL) {
1131		int was_stopped = ts->tick_stopped;
1132
1133		tick_nohz_stop_tick(ts, cpu);
1134
1135		ts->idle_sleeps++;
1136		ts->idle_expires = expires;
1137
1138		if (!was_stopped && ts->tick_stopped) {
1139			ts->idle_jiffies = ts->last_jiffies;
1140			nohz_balance_enter_idle(cpu);
1141		}
1142	} else {
1143		tick_nohz_retain_tick(ts);
1144	}
1145}
1146
 
 
 
 
 
 
 
 
 
 
1147void tick_nohz_idle_retain_tick(void)
1148{
1149	tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
1150	/*
1151	 * Undo the effect of get_next_timer_interrupt() called from
1152	 * tick_nohz_next_event().
1153	 */
1154	timer_clear_idle();
1155}
1156
1157/**
1158 * tick_nohz_idle_enter - prepare for entering idle on the current CPU
1159 *
1160 * Called when we start the idle loop.
1161 */
1162void tick_nohz_idle_enter(void)
1163{
1164	struct tick_sched *ts;
1165
1166	lockdep_assert_irqs_enabled();
1167
1168	local_irq_disable();
1169
1170	ts = this_cpu_ptr(&tick_cpu_sched);
1171
1172	WARN_ON_ONCE(ts->timer_expires_base);
1173
1174	ts->inidle = 1;
1175	tick_nohz_start_idle(ts);
1176
1177	local_irq_enable();
1178}
1179
1180/**
1181 * tick_nohz_irq_exit - Notify the tick about IRQ exit
1182 *
1183 * A timer may have been added/modified/deleted either by the current IRQ,
1184 * or by another place using this IRQ as a notification. This IRQ may have
1185 * also updated the RCU callback list. These events may require a
1186 * re-evaluation of the next tick. Depending on the context:
1187 *
1188 * 1) If the CPU is idle and no resched is pending, just proceed with idle
1189 *    time accounting. The next tick will be re-evaluated on the next idle
1190 *    loop iteration.
1191 *
1192 * 2) If the CPU is nohz_full:
1193 *
1194 *    2.1) If there is any tick dependency, restart the tick if stopped.
1195 *
1196 *    2.2) If there is no tick dependency, (re-)evaluate the next tick and
1197 *         stop/update it accordingly.
1198 */
1199void tick_nohz_irq_exit(void)
1200{
1201	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1202
1203	if (ts->inidle)
1204		tick_nohz_start_idle(ts);
1205	else
1206		tick_nohz_full_update_tick(ts);
1207}
1208
1209/**
1210 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
1211 */
1212bool tick_nohz_idle_got_tick(void)
1213{
1214	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1215
1216	if (ts->got_idle_tick) {
1217		ts->got_idle_tick = 0;
1218		return true;
1219	}
1220	return false;
1221}
1222
1223/**
1224 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
1225 * or the tick, whichever expires first. Note that, if the tick has been
1226 * stopped, it returns the next hrtimer.
1227 *
1228 * Called from power state control code with interrupts disabled
1229 */
1230ktime_t tick_nohz_get_next_hrtimer(void)
1231{
1232	return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
1233}
1234
1235/**
1236 * tick_nohz_get_sleep_length - return the expected length of the current sleep
1237 * @delta_next: duration until the next event if the tick cannot be stopped
1238 *
1239 * Called from power state control code with interrupts disabled.
1240 *
1241 * The return value of this function and/or the value returned by it through the
1242 * @delta_next pointer can be negative which must be taken into account by its
1243 * callers.
1244 */
1245ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
1246{
1247	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
1248	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1249	int cpu = smp_processor_id();
1250	/*
1251	 * The idle entry time is expected to be a sufficient approximation of
1252	 * the current time at this point.
1253	 */
1254	ktime_t now = ts->idle_entrytime;
1255	ktime_t next_event;
1256
1257	WARN_ON_ONCE(!ts->inidle);
1258
1259	*delta_next = ktime_sub(dev->next_event, now);
1260
1261	if (!can_stop_idle_tick(cpu, ts))
1262		return *delta_next;
1263
1264	next_event = tick_nohz_next_event(ts, cpu);
1265	if (!next_event)
1266		return *delta_next;
1267
1268	/*
1269	 * If the next highres timer to expire is earlier than 'next_event', the
1270	 * idle governor needs to know that.
1271	 */
1272	next_event = min_t(u64, next_event,
1273			   hrtimer_next_event_without(&ts->sched_timer));
1274
1275	return ktime_sub(next_event, now);
1276}
1277
1278/**
1279 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1280 * for a particular CPU.
1281 *
1282 * Called from the schedutil frequency scaling governor in scheduler context.
1283 */
1284unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
1285{
1286	struct tick_sched *ts = tick_get_tick_sched(cpu);
1287
1288	return ts->idle_calls;
1289}
1290
1291/**
1292 * tick_nohz_get_idle_calls - return the current idle calls counter value
1293 *
1294 * Called from the schedutil frequency scaling governor in scheduler context.
1295 */
1296unsigned long tick_nohz_get_idle_calls(void)
1297{
1298	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1299
1300	return ts->idle_calls;
1301}
1302
1303static void tick_nohz_account_idle_time(struct tick_sched *ts,
1304					ktime_t now)
1305{
1306	unsigned long ticks;
1307
1308	ts->idle_exittime = now;
1309
1310	if (vtime_accounting_enabled_this_cpu())
1311		return;
1312	/*
1313	 * We stopped the tick in idle. update_process_times() would miss the
1314	 * time we slept, as it does only a 1 tick accounting.
1315	 * Enforce that this is accounted to idle !
1316	 */
1317	ticks = jiffies - ts->idle_jiffies;
1318	/*
1319	 * We might be one off. Do not randomly account a huge number of ticks!
1320	 */
1321	if (ticks && ticks < LONG_MAX)
1322		account_idle_ticks(ticks);
1323}
1324
1325void tick_nohz_idle_restart_tick(void)
1326{
1327	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1328
1329	if (ts->tick_stopped) {
1330		ktime_t now = ktime_get();
1331		tick_nohz_restart_sched_tick(ts, now);
1332		tick_nohz_account_idle_time(ts, now);
1333	}
1334}
1335
1336static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
1337{
1338	if (tick_nohz_full_cpu(smp_processor_id()))
1339		__tick_nohz_full_update_tick(ts, now);
1340	else
1341		tick_nohz_restart_sched_tick(ts, now);
1342
1343	tick_nohz_account_idle_time(ts, now);
1344}
1345
1346/**
1347 * tick_nohz_idle_exit - Update the tick upon idle task exit
1348 *
1349 * When the idle task exits, update the tick depending on the
1350 * following situations:
1351 *
1352 * 1) If the CPU is not in nohz_full mode (most cases), then
1353 *    restart the tick.
1354 *
1355 * 2) If the CPU is in nohz_full mode (corner case):
1356 *   2.1) If the tick can be kept stopped (no tick dependencies)
1357 *        then re-evaluate the next tick and try to keep it stopped
1358 *        as long as possible.
1359 *   2.2) If the tick has dependencies, restart the tick.
1360 *
 
 
 
1361 */
1362void tick_nohz_idle_exit(void)
1363{
1364	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1365	bool idle_active, tick_stopped;
1366	ktime_t now;
1367
1368	local_irq_disable();
1369
1370	WARN_ON_ONCE(!ts->inidle);
1371	WARN_ON_ONCE(ts->timer_expires_base);
1372
1373	ts->inidle = 0;
1374	idle_active = ts->idle_active;
1375	tick_stopped = ts->tick_stopped;
1376
1377	if (idle_active || tick_stopped)
1378		now = ktime_get();
1379
1380	if (idle_active)
1381		tick_nohz_stop_idle(ts, now);
1382
1383	if (tick_stopped)
1384		tick_nohz_idle_update_tick(ts, now);
1385
1386	local_irq_enable();
1387}
1388
1389/*
1390 * In low-resolution mode, the tick handler must be implemented directly
1391 * at the clockevent level. hrtimer can't be used instead, because its
1392 * infrastructure actually relies on the tick itself as a backend in
1393 * low-resolution mode (see hrtimer_run_queues()).
1394 *
1395 * This low-resolution handler still makes use of some hrtimer APIs meanwhile
1396 * for convenience with expiration calculation and forwarding.
1397 */
1398static void tick_nohz_lowres_handler(struct clock_event_device *dev)
1399{
1400	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1401	struct pt_regs *regs = get_irq_regs();
1402	ktime_t now = ktime_get();
1403
1404	dev->next_event = KTIME_MAX;
1405
1406	tick_sched_do_timer(ts, now);
1407	tick_sched_handle(ts, regs);
1408
1409	/*
1410	 * In dynticks mode, tick reprogram is deferred:
1411	 * - to the idle task if in dynticks-idle
1412	 * - to IRQ exit if in full-dynticks.
1413	 */
1414	if (likely(!ts->tick_stopped)) {
1415		hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
1416		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1417	}
1418
 
 
1419}
1420
1421static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1422{
1423	if (!tick_nohz_enabled)
1424		return;
1425	ts->nohz_mode = mode;
1426	/* One update is enough */
1427	if (!test_and_set_bit(0, &tick_nohz_active))
1428		timers_update_nohz();
1429}
1430
1431/**
1432 * tick_nohz_switch_to_nohz - switch to NOHZ mode
1433 */
1434static void tick_nohz_switch_to_nohz(void)
1435{
1436	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1437	ktime_t next;
1438
1439	if (!tick_nohz_enabled)
1440		return;
1441
1442	if (tick_switch_to_oneshot(tick_nohz_lowres_handler))
1443		return;
1444
1445	/*
1446	 * Recycle the hrtimer in 'ts', so we can share the
1447	 * hrtimer_forward_now() function with the highres code.
1448	 */
1449	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1450	/* Get the next period */
1451	next = tick_init_jiffy_update();
1452
1453	hrtimer_set_expires(&ts->sched_timer, next);
1454	hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);
1455	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1456	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1457}
1458
1459static inline void tick_nohz_irq_enter(void)
1460{
1461	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1462	ktime_t now;
1463
1464	if (!ts->idle_active && !ts->tick_stopped)
1465		return;
1466	now = ktime_get();
1467	if (ts->idle_active)
1468		tick_nohz_stop_idle(ts, now);
1469	/*
1470	 * If all CPUs are idle we may need to update a stale jiffies value.
1471	 * Note nohz_full is a special case: a timekeeper is guaranteed to stay
1472	 * alive but it might be busy looping with interrupts disabled in some
1473	 * rare case (typically stop machine). So we must make sure we have a
1474	 * last resort.
1475	 */
1476	if (ts->tick_stopped)
1477		tick_nohz_update_jiffies(now);
1478}
1479
1480#else
1481
1482static inline void tick_nohz_switch_to_nohz(void) { }
1483static inline void tick_nohz_irq_enter(void) { }
1484static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1485
1486#endif /* CONFIG_NO_HZ_COMMON */
1487
1488/*
1489 * Called from irq_enter() to notify about the possible interruption of idle()
1490 */
1491void tick_irq_enter(void)
1492{
1493	tick_check_oneshot_broadcast_this_cpu();
1494	tick_nohz_irq_enter();
1495}
1496
1497/*
1498 * High resolution timer specific code
1499 */
1500#ifdef CONFIG_HIGH_RES_TIMERS
1501/*
1502 * We rearm the timer until we get disabled by the idle code.
1503 * Called with interrupts disabled.
1504 */
1505static enum hrtimer_restart tick_nohz_highres_handler(struct hrtimer *timer)
1506{
1507	struct tick_sched *ts =
1508		container_of(timer, struct tick_sched, sched_timer);
1509	struct pt_regs *regs = get_irq_regs();
1510	ktime_t now = ktime_get();
1511
1512	tick_sched_do_timer(ts, now);
1513
1514	/*
1515	 * Do not call when we are not in IRQ context and have
1516	 * no valid 'regs' pointer
1517	 */
1518	if (regs)
1519		tick_sched_handle(ts, regs);
1520	else
1521		ts->next_tick = 0;
1522
1523	/*
1524	 * In dynticks mode, tick reprogram is deferred:
1525	 * - to the idle task if in dynticks-idle
1526	 * - to IRQ exit if in full-dynticks.
1527	 */
1528	if (unlikely(ts->tick_stopped))
1529		return HRTIMER_NORESTART;
1530
1531	hrtimer_forward(timer, now, TICK_NSEC);
1532
1533	return HRTIMER_RESTART;
1534}
1535
1536static int sched_skew_tick;
1537
1538static int __init skew_tick(char *str)
1539{
1540	get_option(&str, &sched_skew_tick);
1541
1542	return 0;
1543}
1544early_param("skew_tick", skew_tick);
1545
1546/**
1547 * tick_setup_sched_timer - setup the tick emulation timer
1548 */
1549void tick_setup_sched_timer(void)
1550{
1551	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1552	ktime_t now = ktime_get();
1553
1554	/* Emulate tick processing via per-CPU hrtimers: */
 
 
1555	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1556	ts->sched_timer.function = tick_nohz_highres_handler;
1557
1558	/* Get the next period (per-CPU) */
1559	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1560
1561	/* Offset the tick to avert 'jiffies_lock' contention. */
1562	if (sched_skew_tick) {
1563		u64 offset = TICK_NSEC >> 1;
1564		do_div(offset, num_possible_cpus());
1565		offset *= smp_processor_id();
1566		hrtimer_add_expires_ns(&ts->sched_timer, offset);
1567	}
1568
1569	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
1570	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
1571	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1572}
1573#endif /* HIGH_RES_TIMERS */
1574
1575#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1576void tick_cancel_sched_timer(int cpu)
1577{
1578	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1579	ktime_t idle_sleeptime, iowait_sleeptime;
1580	unsigned long idle_calls, idle_sleeps;
1581
1582# ifdef CONFIG_HIGH_RES_TIMERS
1583	if (ts->sched_timer.base)
1584		hrtimer_cancel(&ts->sched_timer);
1585# endif
1586
1587	idle_sleeptime = ts->idle_sleeptime;
1588	iowait_sleeptime = ts->iowait_sleeptime;
1589	idle_calls = ts->idle_calls;
1590	idle_sleeps = ts->idle_sleeps;
1591	memset(ts, 0, sizeof(*ts));
1592	ts->idle_sleeptime = idle_sleeptime;
1593	ts->iowait_sleeptime = iowait_sleeptime;
1594	ts->idle_calls = idle_calls;
1595	ts->idle_sleeps = idle_sleeps;
1596}
1597#endif
1598
1599/*
1600 * Async notification about clocksource changes
1601 */
1602void tick_clock_notify(void)
1603{
1604	int cpu;
1605
1606	for_each_possible_cpu(cpu)
1607		set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1608}
1609
1610/*
1611 * Async notification about clock event changes
1612 */
1613void tick_oneshot_notify(void)
1614{
1615	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1616
1617	set_bit(0, &ts->check_clocks);
1618}
1619
1620/*
1621 * Check if a change happened, which makes oneshot possible.
1622 *
1623 * Called cyclically from the hrtimer softirq (driven by the timer
1624 * softirq). 'allow_nohz' signals that we can switch into low-res NOHZ
1625 * mode, because high resolution timers are disabled (either compile
1626 * or runtime). Called with interrupts disabled.
1627 */
1628int tick_check_oneshot_change(int allow_nohz)
1629{
1630	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1631
1632	if (!test_and_clear_bit(0, &ts->check_clocks))
1633		return 0;
1634
1635	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1636		return 0;
1637
1638	if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1639		return 0;
1640
1641	if (!allow_nohz)
1642		return 1;
1643
1644	tick_nohz_switch_to_nohz();
1645	return 0;
1646}