Linux Audio

Check our new training course

Loading...
v4.10.11
 
   1/*
   2 *  linux/kernel/time/tick-sched.c
   3 *
   4 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   5 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   6 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
   7 *
   8 *  No idle tick implementation for low and high resolution timers
   9 *
  10 *  Started by: Thomas Gleixner and Ingo Molnar
  11 *
  12 *  Distribute under GPLv2.
  13 */
  14#include <linux/cpu.h>
  15#include <linux/err.h>
  16#include <linux/hrtimer.h>
  17#include <linux/interrupt.h>
  18#include <linux/kernel_stat.h>
  19#include <linux/percpu.h>
 
  20#include <linux/profile.h>
  21#include <linux/sched.h>
 
 
 
  22#include <linux/module.h>
  23#include <linux/irq_work.h>
  24#include <linux/posix-timers.h>
  25#include <linux/context_tracking.h>
 
  26
  27#include <asm/irq_regs.h>
  28
  29#include "tick-internal.h"
  30
  31#include <trace/events/timer.h>
  32
  33/*
  34 * Per-CPU nohz control structure
  35 */
  36static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  37
  38struct tick_sched *tick_get_tick_sched(int cpu)
  39{
  40	return &per_cpu(tick_cpu_sched, cpu);
  41}
  42
  43#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
  44/*
  45 * The time, when the last jiffy update happened. Protected by jiffies_lock.
  46 */
  47static ktime_t last_jiffies_update;
  48
  49/*
  50 * Must be called with interrupts disabled !
  51 */
  52static void tick_do_update_jiffies64(ktime_t now)
  53{
  54	unsigned long ticks = 0;
  55	ktime_t delta;
  56
  57	/*
  58	 * Do a quick check without holding jiffies_lock:
 
  59	 */
  60	delta = ktime_sub(now, last_jiffies_update);
  61	if (delta < tick_period)
  62		return;
  63
  64	/* Reevaluate with jiffies_lock held */
  65	write_seqlock(&jiffies_lock);
 
  66
  67	delta = ktime_sub(now, last_jiffies_update);
  68	if (delta >= tick_period) {
  69
  70		delta = ktime_sub(delta, tick_period);
  71		last_jiffies_update = ktime_add(last_jiffies_update,
  72						tick_period);
 
  73
  74		/* Slow path for long timeouts */
  75		if (unlikely(delta >= tick_period)) {
  76			s64 incr = ktime_to_ns(tick_period);
  77
  78			ticks = ktime_divns(delta, incr);
  79
  80			last_jiffies_update = ktime_add_ns(last_jiffies_update,
  81							   incr * ticks);
 
 
  82		}
  83		do_timer(++ticks);
  84
  85		/* Keep the tick_next_period variable up to date */
  86		tick_next_period = ktime_add(last_jiffies_update, tick_period);
  87	} else {
  88		write_sequnlock(&jiffies_lock);
 
  89		return;
  90	}
  91	write_sequnlock(&jiffies_lock);
 
  92	update_wall_time();
  93}
  94
  95/*
  96 * Initialize and return retrieve the jiffies update.
  97 */
  98static ktime_t tick_init_jiffy_update(void)
  99{
 100	ktime_t period;
 101
 102	write_seqlock(&jiffies_lock);
 
 103	/* Did we start the jiffies update yet ? */
 104	if (last_jiffies_update == 0)
 105		last_jiffies_update = tick_next_period;
 106	period = last_jiffies_update;
 107	write_sequnlock(&jiffies_lock);
 
 108	return period;
 109}
 110
 111
 112static void tick_sched_do_timer(ktime_t now)
 113{
 114	int cpu = smp_processor_id();
 115
 116#ifdef CONFIG_NO_HZ_COMMON
 117	/*
 118	 * Check if the do_timer duty was dropped. We don't care about
 119	 * concurrency: This happens only when the CPU in charge went
 120	 * into a long sleep. If two CPUs happen to assign themselves to
 121	 * this duty, then the jiffies update is still serialized by
 122	 * jiffies_lock.
 
 
 
 123	 */
 124	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)
 125	    && !tick_nohz_full_cpu(cpu))
 
 
 126		tick_do_timer_cpu = cpu;
 
 127#endif
 128
 129	/* Check, if the jiffies need an update */
 130	if (tick_do_timer_cpu == cpu)
 131		tick_do_update_jiffies64(now);
 
 
 
 132}
 133
 134static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 135{
 136#ifdef CONFIG_NO_HZ_COMMON
 137	/*
 138	 * When we are idle and the tick is stopped, we have to touch
 139	 * the watchdog as we might not schedule for a really long
 140	 * time. This happens on complete idle SMP systems while
 141	 * waiting on the login prompt. We also increment the "start of
 142	 * idle" jiffy stamp so the idle accounting adjustment we do
 143	 * when we go busy again does not account too much ticks.
 144	 */
 145	if (ts->tick_stopped) {
 146		touch_softlockup_watchdog_sched();
 147		if (is_idle_task(current))
 148			ts->idle_jiffies++;
 
 
 
 
 
 
 149	}
 150#endif
 151	update_process_times(user_mode(regs));
 152	profile_tick(CPU_PROFILING);
 153}
 154#endif
 155
 156#ifdef CONFIG_NO_HZ_FULL
 157cpumask_var_t tick_nohz_full_mask;
 158cpumask_var_t housekeeping_mask;
 159bool tick_nohz_full_running;
 
 160static atomic_t tick_dep_mask;
 161
 162static bool check_tick_dependency(atomic_t *dep)
 163{
 164	int val = atomic_read(dep);
 165
 166	if (val & TICK_DEP_MASK_POSIX_TIMER) {
 167		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
 168		return true;
 169	}
 170
 171	if (val & TICK_DEP_MASK_PERF_EVENTS) {
 172		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
 173		return true;
 174	}
 175
 176	if (val & TICK_DEP_MASK_SCHED) {
 177		trace_tick_stop(0, TICK_DEP_MASK_SCHED);
 178		return true;
 179	}
 180
 181	if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
 182		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
 183		return true;
 184	}
 185
 
 
 
 
 
 186	return false;
 187}
 188
 189static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
 190{
 191	WARN_ON_ONCE(!irqs_disabled());
 192
 193	if (unlikely(!cpu_online(cpu)))
 194		return false;
 195
 196	if (check_tick_dependency(&tick_dep_mask))
 197		return false;
 198
 199	if (check_tick_dependency(&ts->tick_dep_mask))
 200		return false;
 201
 202	if (check_tick_dependency(&current->tick_dep_mask))
 203		return false;
 204
 205	if (check_tick_dependency(&current->signal->tick_dep_mask))
 206		return false;
 207
 208	return true;
 209}
 210
 211static void nohz_full_kick_func(struct irq_work *work)
 212{
 213	/* Empty, the tick restart happens on tick_nohz_irq_exit() */
 214}
 215
 216static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
 217	.func = nohz_full_kick_func,
 
 218};
 219
 220/*
 221 * Kick this CPU if it's full dynticks in order to force it to
 222 * re-evaluate its dependency on the tick and restart it if necessary.
 223 * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
 224 * is NMI safe.
 225 */
 226static void tick_nohz_full_kick(void)
 227{
 228	if (!tick_nohz_full_cpu(smp_processor_id()))
 229		return;
 230
 231	irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
 232}
 233
 234/*
 235 * Kick the CPU if it's full dynticks in order to force it to
 236 * re-evaluate its dependency on the tick and restart it if necessary.
 237 */
 238void tick_nohz_full_kick_cpu(int cpu)
 239{
 240	if (!tick_nohz_full_cpu(cpu))
 241		return;
 242
 243	irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
 244}
 245
 246/*
 247 * Kick all full dynticks CPUs in order to force these to re-evaluate
 248 * their dependency on the tick and restart it if necessary.
 249 */
 250static void tick_nohz_full_kick_all(void)
 251{
 252	int cpu;
 253
 254	if (!tick_nohz_full_running)
 255		return;
 256
 257	preempt_disable();
 258	for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
 259		tick_nohz_full_kick_cpu(cpu);
 260	preempt_enable();
 261}
 262
 263static void tick_nohz_dep_set_all(atomic_t *dep,
 264				  enum tick_dep_bits bit)
 265{
 266	int prev;
 267
 268	prev = atomic_fetch_or(BIT(bit), dep);
 269	if (!prev)
 270		tick_nohz_full_kick_all();
 271}
 272
 273/*
 274 * Set a global tick dependency. Used by perf events that rely on freq and
 275 * by unstable clock.
 276 */
 277void tick_nohz_dep_set(enum tick_dep_bits bit)
 278{
 279	tick_nohz_dep_set_all(&tick_dep_mask, bit);
 280}
 281
 282void tick_nohz_dep_clear(enum tick_dep_bits bit)
 283{
 284	atomic_andnot(BIT(bit), &tick_dep_mask);
 285}
 286
 287/*
 288 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
 289 * manage events throttling.
 290 */
 291void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
 292{
 293	int prev;
 294	struct tick_sched *ts;
 295
 296	ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 297
 298	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
 299	if (!prev) {
 300		preempt_disable();
 301		/* Perf needs local kick that is NMI safe */
 302		if (cpu == smp_processor_id()) {
 303			tick_nohz_full_kick();
 304		} else {
 305			/* Remote irq work not NMI-safe */
 306			if (!WARN_ON_ONCE(in_nmi()))
 307				tick_nohz_full_kick_cpu(cpu);
 308		}
 309		preempt_enable();
 310	}
 311}
 
 312
 313void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
 314{
 315	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 316
 317	atomic_andnot(BIT(bit), &ts->tick_dep_mask);
 318}
 
 319
 320/*
 321 * Set a per-task tick dependency. Posix CPU timers need this in order to elapse
 322 * per task timers.
 323 */
 324void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
 325{
 326	/*
 327	 * We could optimize this with just kicking the target running the task
 328	 * if that noise matters for nohz full users.
 329	 */
 330	tick_nohz_dep_set_all(&tsk->tick_dep_mask, bit);
 
 
 
 
 
 
 
 
 331}
 
 332
 333void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
 334{
 335	atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
 336}
 
 337
 338/*
 339 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
 340 * per process timers.
 341 */
 342void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
 343{
 344	tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
 345}
 346
 347void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
 348{
 349	atomic_andnot(BIT(bit), &sig->tick_dep_mask);
 350}
 351
 352/*
 353 * Re-evaluate the need for the tick as we switch the current task.
 354 * It might need the tick due to per task/process properties:
 355 * perf events, posix CPU timers, ...
 356 */
 357void __tick_nohz_task_switch(void)
 358{
 359	unsigned long flags;
 360	struct tick_sched *ts;
 361
 362	local_irq_save(flags);
 363
 364	if (!tick_nohz_full_cpu(smp_processor_id()))
 365		goto out;
 366
 367	ts = this_cpu_ptr(&tick_cpu_sched);
 368
 369	if (ts->tick_stopped) {
 370		if (atomic_read(&current->tick_dep_mask) ||
 371		    atomic_read(&current->signal->tick_dep_mask))
 372			tick_nohz_full_kick();
 373	}
 374out:
 375	local_irq_restore(flags);
 376}
 377
 378/* Parse the boot-time nohz CPU list from the kernel parameters. */
 379static int __init tick_nohz_full_setup(char *str)
 380{
 381	alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
 382	if (cpulist_parse(str, tick_nohz_full_mask) < 0) {
 383		pr_warn("NO_HZ: Incorrect nohz_full cpumask\n");
 384		free_bootmem_cpumask_var(tick_nohz_full_mask);
 385		return 1;
 386	}
 387	tick_nohz_full_running = true;
 388
 389	return 1;
 390}
 391__setup("nohz_full=", tick_nohz_full_setup);
 392
 393static int tick_nohz_cpu_down(unsigned int cpu)
 394{
 395	/*
 396	 * The boot CPU handles housekeeping duty (unbound timers,
 397	 * workqueues, timekeeping, ...) on behalf of full dynticks
 398	 * CPUs. It must remain online when nohz full is enabled.
 399	 */
 400	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
 401		return -EBUSY;
 402	return 0;
 403}
 404
 405static int tick_nohz_init_all(void)
 406{
 407	int err = -1;
 408
 409#ifdef CONFIG_NO_HZ_FULL_ALL
 410	if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
 411		WARN(1, "NO_HZ: Can't allocate full dynticks cpumask\n");
 412		return err;
 413	}
 414	err = 0;
 415	cpumask_setall(tick_nohz_full_mask);
 416	tick_nohz_full_running = true;
 417#endif
 418	return err;
 419}
 420
 421void __init tick_nohz_init(void)
 422{
 423	int cpu, ret;
 424
 425	if (!tick_nohz_full_running) {
 426		if (tick_nohz_init_all() < 0)
 427			return;
 428	}
 429
 430	if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
 431		WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n");
 432		cpumask_clear(tick_nohz_full_mask);
 433		tick_nohz_full_running = false;
 434		return;
 435	}
 436
 437	/*
 438	 * Full dynticks uses irq work to drive the tick rescheduling on safe
 439	 * locking contexts. But then we need irq work to raise its own
 440	 * interrupts to avoid circular dependency on the tick
 441	 */
 442	if (!arch_irq_work_has_interrupt()) {
 443		pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
 444		cpumask_clear(tick_nohz_full_mask);
 445		cpumask_copy(housekeeping_mask, cpu_possible_mask);
 446		tick_nohz_full_running = false;
 447		return;
 448	}
 449
 450	cpu = smp_processor_id();
 451
 452	if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
 453		pr_warn("NO_HZ: Clearing %d from nohz_full range for timekeeping\n",
 454			cpu);
 455		cpumask_clear_cpu(cpu, tick_nohz_full_mask);
 
 
 
 456	}
 457
 458	cpumask_andnot(housekeeping_mask,
 459		       cpu_possible_mask, tick_nohz_full_mask);
 460
 461	for_each_cpu(cpu, tick_nohz_full_mask)
 462		context_tracking_cpu_set(cpu);
 463
 464	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 465					"kernel/nohz:predown", NULL,
 466					tick_nohz_cpu_down);
 467	WARN_ON(ret < 0);
 468	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 469		cpumask_pr_args(tick_nohz_full_mask));
 470
 471	/*
 472	 * We need at least one CPU to handle housekeeping work such
 473	 * as timekeeping, unbound timers, workqueues, ...
 474	 */
 475	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
 476}
 477#endif
 478
 479/*
 480 * NOHZ - aka dynamic tick functionality
 481 */
 482#ifdef CONFIG_NO_HZ_COMMON
 483/*
 484 * NO HZ enabled ?
 485 */
 486bool tick_nohz_enabled __read_mostly  = true;
 487unsigned long tick_nohz_active  __read_mostly;
 488/*
 489 * Enable / Disable tickless mode
 490 */
 491static int __init setup_tick_nohz(char *str)
 492{
 493	return (kstrtobool(str, &tick_nohz_enabled) == 0);
 494}
 495
 496__setup("nohz=", setup_tick_nohz);
 497
 498int tick_nohz_tick_stopped(void)
 499{
 500	return __this_cpu_read(tick_cpu_sched.tick_stopped);
 
 
 
 
 
 
 
 
 
 501}
 502
 503/**
 504 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 505 *
 506 * Called from interrupt entry when the CPU was idle
 507 *
 508 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 509 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 510 * value. We do this unconditionally on any CPU, as we don't know whether the
 511 * CPU, which has the update task assigned is in a long sleep.
 512 */
 513static void tick_nohz_update_jiffies(ktime_t now)
 514{
 515	unsigned long flags;
 516
 517	__this_cpu_write(tick_cpu_sched.idle_waketime, now);
 518
 519	local_irq_save(flags);
 520	tick_do_update_jiffies64(now);
 521	local_irq_restore(flags);
 522
 523	touch_softlockup_watchdog_sched();
 524}
 525
 526/*
 527 * Updates the per-CPU time idle statistics counters
 528 */
 529static void
 530update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
 531{
 532	ktime_t delta;
 533
 534	if (ts->idle_active) {
 535		delta = ktime_sub(now, ts->idle_entrytime);
 536		if (nr_iowait_cpu(cpu) > 0)
 537			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
 538		else
 539			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 540		ts->idle_entrytime = now;
 541	}
 542
 543	if (last_update_time)
 544		*last_update_time = ktime_to_us(now);
 545
 546}
 547
 548static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
 549{
 550	update_ts_time_stats(smp_processor_id(), ts, now, NULL);
 551	ts->idle_active = 0;
 552
 553	sched_clock_idle_wakeup_event(0);
 554}
 555
 556static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 557{
 558	ktime_t now = ktime_get();
 559
 560	ts->idle_entrytime = now;
 561	ts->idle_active = 1;
 562	sched_clock_idle_sleep_event();
 563	return now;
 564}
 565
 566/**
 567 * get_cpu_idle_time_us - get the total idle time of a CPU
 568 * @cpu: CPU number to query
 569 * @last_update_time: variable to store update time in. Do not update
 570 * counters if NULL.
 571 *
 572 * Return the cumulative idle time (since boot) for a given
 573 * CPU, in microseconds.
 574 *
 575 * This time is measured via accounting rather than sampling,
 576 * and is as accurate as ktime_get() is.
 577 *
 578 * This function returns -1 if NOHZ is not enabled.
 579 */
 580u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 581{
 582	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 583	ktime_t now, idle;
 584
 585	if (!tick_nohz_active)
 586		return -1;
 587
 588	now = ktime_get();
 589	if (last_update_time) {
 590		update_ts_time_stats(cpu, ts, now, last_update_time);
 591		idle = ts->idle_sleeptime;
 592	} else {
 593		if (ts->idle_active && !nr_iowait_cpu(cpu)) {
 594			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 595
 596			idle = ktime_add(ts->idle_sleeptime, delta);
 597		} else {
 598			idle = ts->idle_sleeptime;
 599		}
 600	}
 601
 602	return ktime_to_us(idle);
 603
 604}
 605EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 606
 607/**
 608 * get_cpu_iowait_time_us - get the total iowait time of a CPU
 609 * @cpu: CPU number to query
 610 * @last_update_time: variable to store update time in. Do not update
 611 * counters if NULL.
 612 *
 613 * Return the cumulative iowait time (since boot) for a given
 614 * CPU, in microseconds.
 615 *
 616 * This time is measured via accounting rather than sampling,
 617 * and is as accurate as ktime_get() is.
 618 *
 619 * This function returns -1 if NOHZ is not enabled.
 620 */
 621u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 622{
 623	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 624	ktime_t now, iowait;
 625
 626	if (!tick_nohz_active)
 627		return -1;
 628
 629	now = ktime_get();
 630	if (last_update_time) {
 631		update_ts_time_stats(cpu, ts, now, last_update_time);
 632		iowait = ts->iowait_sleeptime;
 633	} else {
 634		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
 635			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 636
 637			iowait = ktime_add(ts->iowait_sleeptime, delta);
 638		} else {
 639			iowait = ts->iowait_sleeptime;
 640		}
 641	}
 642
 643	return ktime_to_us(iowait);
 644}
 645EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 646
 647static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 648{
 649	hrtimer_cancel(&ts->sched_timer);
 650	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 651
 652	/* Forward the time to expire in the future */
 653	hrtimer_forward(&ts->sched_timer, now, tick_period);
 654
 655	if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 656		hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
 657	else
 
 658		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
 
 
 
 
 
 
 
 659}
 660
 661static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 662					 ktime_t now, int cpu)
 
 
 
 
 663{
 664	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 665	u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
 666	unsigned long seq, basejiff;
 667	ktime_t	tick;
 668
 669	/* Read jiffies and the time when jiffies were updated last */
 670	do {
 671		seq = read_seqbegin(&jiffies_lock);
 672		basemono = last_jiffies_update;
 673		basejiff = jiffies;
 674	} while (read_seqretry(&jiffies_lock, seq));
 675	ts->last_jiffies = basejiff;
 
 676
 677	if (rcu_needs_cpu(basemono, &next_rcu) ||
 678	    arch_needs_cpu() || irq_work_needs_cpu()) {
 
 
 
 
 
 
 
 
 
 
 679		next_tick = basemono + TICK_NSEC;
 680	} else {
 681		/*
 682		 * Get the next pending timer. If high resolution
 683		 * timers are enabled this only takes the timer wheel
 684		 * timers into account. If high resolution timers are
 685		 * disabled this also looks at the next expiring
 686		 * hrtimer.
 687		 */
 688		next_tmr = get_next_timer_interrupt(basejiff, basemono);
 689		ts->next_timer = next_tmr;
 690		/* Take the next rcu event into account */
 691		next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
 692	}
 693
 694	/*
 695	 * If the tick is due in the next period, keep it ticking or
 696	 * force prod the timer.
 697	 */
 698	delta = next_tick - basemono;
 699	if (delta <= (u64)TICK_NSEC) {
 700		tick = 0;
 701
 702		/*
 703		 * Tell the timer code that the base is not idle, i.e. undo
 704		 * the effect of get_next_timer_interrupt():
 705		 */
 706		timer_clear_idle();
 707		/*
 708		 * We've not stopped the tick yet, and there's a timer in the
 709		 * next period, so no point in stopping it either, bail.
 710		 */
 711		if (!ts->tick_stopped)
 712			goto out;
 713
 714		/*
 715		 * If, OTOH, we did stop it, but there's a pending (expired)
 716		 * timer reprogram the timer hardware to fire now.
 717		 *
 718		 * We will not restart the tick proper, just prod the timer
 719		 * hardware into firing an interrupt to process the pending
 720		 * timers. Just like tick_irq_exit() will not restart the tick
 721		 * for 'normal' interrupts.
 722		 *
 723		 * Only once we exit the idle loop will we re-enable the tick,
 724		 * see tick_nohz_idle_exit().
 725		 */
 726		if (delta == 0) {
 727			tick_nohz_restart(ts, now);
 728			goto out;
 729		}
 730	}
 731
 732	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 733	 * If this CPU is the one which updates jiffies, then give up
 734	 * the assignment and let it be taken by the CPU which runs
 735	 * the tick timer next, which might be this CPU as well. If we
 736	 * don't drop this here the jiffies might be stale and
 737	 * do_timer() never invoked. Keep track of the fact that it
 738	 * was the one which had the do_timer() duty last. If this CPU
 739	 * is the one which had the do_timer() duty last, we limit the
 740	 * sleep time to the timekeeping max_deferment value.
 741	 * Otherwise we can sleep as long as we want.
 742	 */
 743	delta = timekeeping_max_deferment();
 744	if (cpu == tick_do_timer_cpu) {
 745		tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 746		ts->do_timer_last = 1;
 747	} else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
 748		delta = KTIME_MAX;
 749		ts->do_timer_last = 0;
 750	} else if (!ts->do_timer_last) {
 751		delta = KTIME_MAX;
 752	}
 753
 754#ifdef CONFIG_NO_HZ_FULL
 755	/* Limit the tick delta to the maximum scheduler deferment */
 756	if (!ts->inidle)
 757		delta = min(delta, scheduler_tick_max_deferment());
 758#endif
 759
 760	/* Calculate the next expiry time */
 761	if (delta < (KTIME_MAX - basemono))
 762		expires = basemono + delta;
 763	else
 764		expires = KTIME_MAX;
 765
 766	expires = min_t(u64, expires, next_tick);
 767	tick = expires;
 768
 769	/* Skip reprogram of event if its not changed */
 770	if (ts->tick_stopped && (expires == dev->next_event))
 771		goto out;
 
 
 
 
 
 
 
 
 772
 773	/*
 774	 * nohz_stop_sched_tick can be called several times before
 775	 * the nohz_restart_sched_tick is called. This happens when
 776	 * interrupts arrive which do not cause a reschedule. In the
 777	 * first call we save the current tick time, so we can restart
 778	 * the scheduler tick in nohz_restart_sched_tick.
 779	 */
 780	if (!ts->tick_stopped) {
 781		nohz_balance_enter_idle(cpu);
 782		calc_load_enter_idle();
 783		cpu_load_update_nohz_start();
 784
 785		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
 786		ts->tick_stopped = 1;
 787		trace_tick_stop(1, TICK_DEP_MASK_NONE);
 788	}
 789
 
 
 790	/*
 791	 * If the expiration time == KTIME_MAX, then we simply stop
 792	 * the tick timer.
 793	 */
 794	if (unlikely(expires == KTIME_MAX)) {
 795		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 796			hrtimer_cancel(&ts->sched_timer);
 797		goto out;
 798	}
 799
 800	if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 801		hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
 802	else
 
 
 803		tick_program_event(tick, 1);
 804out:
 805	/* Update the estimated sleep length */
 806	ts->sleep_length = ktime_sub(dev->next_event, now);
 807	return tick;
 
 
 808}
 809
 
 
 
 
 
 
 
 
 
 
 810static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 811{
 812	/* Update jiffies first */
 813	tick_do_update_jiffies64(now);
 814	cpu_load_update_nohz_stop();
 815
 816	/*
 817	 * Clear the timer idle flag, so we avoid IPIs on remote queueing and
 818	 * the clock forward checks in the enqueue path:
 819	 */
 820	timer_clear_idle();
 821
 822	calc_load_exit_idle();
 823	touch_softlockup_watchdog_sched();
 824	/*
 825	 * Cancel the scheduled timer and restore the tick
 826	 */
 827	ts->tick_stopped  = 0;
 828	ts->idle_exittime = now;
 829
 830	tick_nohz_restart(ts, now);
 831}
 832
 833static void tick_nohz_full_update_tick(struct tick_sched *ts)
 834{
 835#ifdef CONFIG_NO_HZ_FULL
 836	int cpu = smp_processor_id();
 837
 838	if (!tick_nohz_full_cpu(cpu))
 839		return;
 840
 841	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
 842		return;
 843
 844	if (can_stop_full_tick(cpu, ts))
 845		tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
 846	else if (ts->tick_stopped)
 847		tick_nohz_restart_sched_tick(ts, ktime_get());
 848#endif
 849}
 850
 851static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 852{
 853	/*
 854	 * If this CPU is offline and it is the one which updates
 855	 * jiffies, then give up the assignment and let it be taken by
 856	 * the CPU which runs the tick timer next. If we don't drop
 857	 * this here the jiffies might be stale and do_timer() never
 858	 * invoked.
 859	 */
 860	if (unlikely(!cpu_online(cpu))) {
 861		if (cpu == tick_do_timer_cpu)
 862			tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 
 
 
 
 
 863		return false;
 864	}
 865
 866	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
 867		ts->sleep_length = NSEC_PER_SEC / HZ;
 868		return false;
 869	}
 870
 871	if (need_resched())
 872		return false;
 873
 874	if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
 875		static int ratelimit;
 876
 877		if (ratelimit < 10 &&
 878		    (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
 879			pr_warn("NOHZ: local_softirq_pending %02x\n",
 880				(unsigned int) local_softirq_pending());
 881			ratelimit++;
 882		}
 883		return false;
 884	}
 885
 886	if (tick_nohz_full_enabled()) {
 887		/*
 888		 * Keep the tick alive to guarantee timekeeping progression
 889		 * if there are full dynticks CPUs around
 890		 */
 891		if (tick_do_timer_cpu == cpu)
 892			return false;
 893		/*
 894		 * Boot safety: make sure the timekeeping duty has been
 895		 * assigned before entering dyntick-idle mode,
 
 896		 */
 897		if (tick_do_timer_cpu == TICK_DO_TIMER_NONE)
 
 
 
 
 898			return false;
 899	}
 900
 901	return true;
 902}
 903
 904static void __tick_nohz_idle_enter(struct tick_sched *ts)
 905{
 906	ktime_t now, expires;
 907	int cpu = smp_processor_id();
 908
 909	now = tick_nohz_start_idle(ts);
 
 
 
 
 
 
 
 
 
 
 
 910
 911	if (can_stop_idle_tick(cpu, ts)) {
 912		int was_stopped = ts->tick_stopped;
 913
 914		ts->idle_calls++;
 915
 916		expires = tick_nohz_stop_sched_tick(ts, now, cpu);
 917		if (expires > 0LL) {
 918			ts->idle_sleeps++;
 919			ts->idle_expires = expires;
 920		}
 921
 922		if (!was_stopped && ts->tick_stopped)
 923			ts->idle_jiffies = ts->last_jiffies;
 
 
 
 
 924	}
 925}
 926
 927/**
 928 * tick_nohz_idle_enter - stop the idle tick from the idle task
 929 *
 930 * When the next event is more than a tick into the future, stop the idle tick
 931 * Called when we start the idle loop.
 932 *
 933 * The arch is responsible of calling:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 934 *
 935 * - rcu_idle_enter() after its last use of RCU before the CPU is put
 936 *  to sleep.
 937 * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
 938 */
 939void tick_nohz_idle_enter(void)
 940{
 941	struct tick_sched *ts;
 942
 943	WARN_ON_ONCE(irqs_disabled());
 944
 945	/*
 946	 * Update the idle state in the scheduler domain hierarchy
 947	 * when tick_nohz_stop_sched_tick() is called from the idle loop.
 948	 * State will be updated to busy during the first busy tick after
 949	 * exiting idle.
 950	 */
 951	set_cpu_sd_state_idle();
 952
 953	local_irq_disable();
 954
 955	ts = this_cpu_ptr(&tick_cpu_sched);
 
 
 
 956	ts->inidle = 1;
 957	__tick_nohz_idle_enter(ts);
 958
 959	local_irq_enable();
 960}
 961
 962/**
 963 * tick_nohz_irq_exit - update next tick event from interrupt exit
 964 *
 965 * When an interrupt fires while we are idle and it doesn't cause
 966 * a reschedule, it may still add, modify or delete a timer, enqueue
 967 * an RCU callback, etc...
 968 * So we need to re-calculate and reprogram the next tick event.
 969 */
 970void tick_nohz_irq_exit(void)
 971{
 972	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 973
 974	if (ts->inidle)
 975		__tick_nohz_idle_enter(ts);
 976	else
 977		tick_nohz_full_update_tick(ts);
 978}
 979
 980/**
 981 * tick_nohz_get_sleep_length - return the length of the current sleep
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 982 *
 983 * Called from power state control code with interrupts disabled
 984 */
 985ktime_t tick_nohz_get_sleep_length(void)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 986{
 987	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 988
 989	return ts->sleep_length;
 990}
 991
 992static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
 993{
 994#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 995	unsigned long ticks;
 996
 997	if (vtime_accounting_cpu_enabled())
 998		return;
 999	/*
1000	 * We stopped the tick in idle. Update process times would miss the
1001	 * time we slept as update_process_times does only a 1 tick
1002	 * accounting. Enforce that this is accounted to idle !
1003	 */
1004	ticks = jiffies - ts->idle_jiffies;
1005	/*
1006	 * We might be one off. Do not randomly account a huge number of ticks!
1007	 */
1008	if (ticks && ticks < LONG_MAX)
1009		account_idle_ticks(ticks);
1010#endif
1011}
1012
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1013/**
1014 * tick_nohz_idle_exit - restart the idle tick from the idle task
1015 *
1016 * Restart the idle tick when the CPU is woken up from idle
1017 * This also exit the RCU extended quiescent state. The CPU
1018 * can use RCU again after this function is called.
1019 */
1020void tick_nohz_idle_exit(void)
1021{
1022	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
1023	ktime_t now;
1024
1025	local_irq_disable();
1026
1027	WARN_ON_ONCE(!ts->inidle);
 
1028
1029	ts->inidle = 0;
 
 
1030
1031	if (ts->idle_active || ts->tick_stopped)
1032		now = ktime_get();
1033
1034	if (ts->idle_active)
1035		tick_nohz_stop_idle(ts, now);
1036
1037	if (ts->tick_stopped) {
1038		tick_nohz_restart_sched_tick(ts, now);
1039		tick_nohz_account_idle_ticks(ts);
1040	}
1041
1042	local_irq_enable();
1043}
1044
1045/*
1046 * The nohz low res interrupt handler
1047 */
1048static void tick_nohz_handler(struct clock_event_device *dev)
1049{
1050	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1051	struct pt_regs *regs = get_irq_regs();
1052	ktime_t now = ktime_get();
1053
1054	dev->next_event = KTIME_MAX;
1055
1056	tick_sched_do_timer(now);
1057	tick_sched_handle(ts, regs);
1058
1059	/* No need to reprogram if we are running tickless  */
1060	if (unlikely(ts->tick_stopped))
1061		return;
1062
1063	hrtimer_forward(&ts->sched_timer, now, tick_period);
1064	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1065}
1066
1067static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1068{
1069	if (!tick_nohz_enabled)
1070		return;
1071	ts->nohz_mode = mode;
1072	/* One update is enough */
1073	if (!test_and_set_bit(0, &tick_nohz_active))
1074		timers_update_migration(true);
1075}
1076
1077/**
1078 * tick_nohz_switch_to_nohz - switch to nohz mode
1079 */
1080static void tick_nohz_switch_to_nohz(void)
1081{
1082	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1083	ktime_t next;
1084
1085	if (!tick_nohz_enabled)
1086		return;
1087
1088	if (tick_switch_to_oneshot(tick_nohz_handler))
1089		return;
1090
1091	/*
1092	 * Recycle the hrtimer in ts, so we can share the
1093	 * hrtimer_forward with the highres code.
1094	 */
1095	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1096	/* Get the next period */
1097	next = tick_init_jiffy_update();
1098
1099	hrtimer_set_expires(&ts->sched_timer, next);
1100	hrtimer_forward_now(&ts->sched_timer, tick_period);
1101	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1102	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1103}
1104
1105static inline void tick_nohz_irq_enter(void)
1106{
1107	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1108	ktime_t now;
1109
1110	if (!ts->idle_active && !ts->tick_stopped)
1111		return;
1112	now = ktime_get();
1113	if (ts->idle_active)
1114		tick_nohz_stop_idle(ts, now);
1115	if (ts->tick_stopped)
1116		tick_nohz_update_jiffies(now);
1117}
1118
1119#else
1120
1121static inline void tick_nohz_switch_to_nohz(void) { }
1122static inline void tick_nohz_irq_enter(void) { }
1123static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1124
1125#endif /* CONFIG_NO_HZ_COMMON */
1126
1127/*
1128 * Called from irq_enter to notify about the possible interruption of idle()
1129 */
1130void tick_irq_enter(void)
1131{
1132	tick_check_oneshot_broadcast_this_cpu();
1133	tick_nohz_irq_enter();
1134}
1135
1136/*
1137 * High resolution timer specific code
1138 */
1139#ifdef CONFIG_HIGH_RES_TIMERS
1140/*
1141 * We rearm the timer until we get disabled by the idle code.
1142 * Called with interrupts disabled.
1143 */
1144static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1145{
1146	struct tick_sched *ts =
1147		container_of(timer, struct tick_sched, sched_timer);
1148	struct pt_regs *regs = get_irq_regs();
1149	ktime_t now = ktime_get();
1150
1151	tick_sched_do_timer(now);
1152
1153	/*
1154	 * Do not call, when we are not in irq context and have
1155	 * no valid regs pointer
1156	 */
1157	if (regs)
1158		tick_sched_handle(ts, regs);
 
 
1159
1160	/* No need to reprogram if we are in idle or full dynticks mode */
1161	if (unlikely(ts->tick_stopped))
1162		return HRTIMER_NORESTART;
1163
1164	hrtimer_forward(timer, now, tick_period);
1165
1166	return HRTIMER_RESTART;
1167}
1168
1169static int sched_skew_tick;
1170
1171static int __init skew_tick(char *str)
1172{
1173	get_option(&str, &sched_skew_tick);
1174
1175	return 0;
1176}
1177early_param("skew_tick", skew_tick);
1178
1179/**
1180 * tick_setup_sched_timer - setup the tick emulation timer
1181 */
1182void tick_setup_sched_timer(void)
1183{
1184	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1185	ktime_t now = ktime_get();
1186
1187	/*
1188	 * Emulate tick processing via per-CPU hrtimers:
1189	 */
1190	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1191	ts->sched_timer.function = tick_sched_timer;
1192
1193	/* Get the next period (per-CPU) */
1194	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1195
1196	/* Offset the tick to avert jiffies_lock contention. */
1197	if (sched_skew_tick) {
1198		u64 offset = ktime_to_ns(tick_period) >> 1;
1199		do_div(offset, num_possible_cpus());
1200		offset *= smp_processor_id();
1201		hrtimer_add_expires_ns(&ts->sched_timer, offset);
1202	}
1203
1204	hrtimer_forward(&ts->sched_timer, now, tick_period);
1205	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
1206	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1207}
1208#endif /* HIGH_RES_TIMERS */
1209
1210#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1211void tick_cancel_sched_timer(int cpu)
1212{
1213	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1214
1215# ifdef CONFIG_HIGH_RES_TIMERS
1216	if (ts->sched_timer.base)
1217		hrtimer_cancel(&ts->sched_timer);
1218# endif
1219
1220	memset(ts, 0, sizeof(*ts));
1221}
1222#endif
1223
1224/**
1225 * Async notification about clocksource changes
1226 */
1227void tick_clock_notify(void)
1228{
1229	int cpu;
1230
1231	for_each_possible_cpu(cpu)
1232		set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1233}
1234
1235/*
1236 * Async notification about clock event changes
1237 */
1238void tick_oneshot_notify(void)
1239{
1240	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1241
1242	set_bit(0, &ts->check_clocks);
1243}
1244
1245/**
1246 * Check, if a change happened, which makes oneshot possible.
1247 *
1248 * Called cyclic from the hrtimer softirq (driven by the timer
1249 * softirq) allow_nohz signals, that we can switch into low-res nohz
1250 * mode, because high resolution timers are disabled (either compile
1251 * or runtime). Called with interrupts disabled.
1252 */
1253int tick_check_oneshot_change(int allow_nohz)
1254{
1255	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1256
1257	if (!test_and_clear_bit(0, &ts->check_clocks))
1258		return 0;
1259
1260	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1261		return 0;
1262
1263	if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1264		return 0;
1265
1266	if (!allow_nohz)
1267		return 1;
1268
1269	tick_nohz_switch_to_nohz();
1270	return 0;
1271}
v5.9
   1// SPDX-License-Identifier: GPL-2.0
   2/*
 
 
   3 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   4 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   5 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
   6 *
   7 *  No idle tick implementation for low and high resolution timers
   8 *
   9 *  Started by: Thomas Gleixner and Ingo Molnar
 
 
  10 */
  11#include <linux/cpu.h>
  12#include <linux/err.h>
  13#include <linux/hrtimer.h>
  14#include <linux/interrupt.h>
  15#include <linux/kernel_stat.h>
  16#include <linux/percpu.h>
  17#include <linux/nmi.h>
  18#include <linux/profile.h>
  19#include <linux/sched/signal.h>
  20#include <linux/sched/clock.h>
  21#include <linux/sched/stat.h>
  22#include <linux/sched/nohz.h>
  23#include <linux/module.h>
  24#include <linux/irq_work.h>
  25#include <linux/posix-timers.h>
  26#include <linux/context_tracking.h>
  27#include <linux/mm.h>
  28
  29#include <asm/irq_regs.h>
  30
  31#include "tick-internal.h"
  32
  33#include <trace/events/timer.h>
  34
  35/*
  36 * Per-CPU nohz control structure
  37 */
  38static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  39
  40struct tick_sched *tick_get_tick_sched(int cpu)
  41{
  42	return &per_cpu(tick_cpu_sched, cpu);
  43}
  44
  45#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
  46/*
  47 * The time, when the last jiffy update happened. Protected by jiffies_lock.
  48 */
  49static ktime_t last_jiffies_update;
  50
  51/*
  52 * Must be called with interrupts disabled !
  53 */
  54static void tick_do_update_jiffies64(ktime_t now)
  55{
  56	unsigned long ticks = 0;
  57	ktime_t delta;
  58
  59	/*
  60	 * Do a quick check without holding jiffies_lock:
  61	 * The READ_ONCE() pairs with two updates done later in this function.
  62	 */
  63	delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
  64	if (delta < tick_period)
  65		return;
  66
  67	/* Reevaluate with jiffies_lock held */
  68	raw_spin_lock(&jiffies_lock);
  69	write_seqcount_begin(&jiffies_seq);
  70
  71	delta = ktime_sub(now, last_jiffies_update);
  72	if (delta >= tick_period) {
  73
  74		delta = ktime_sub(delta, tick_period);
  75		/* Pairs with the lockless read in this function. */
  76		WRITE_ONCE(last_jiffies_update,
  77			   ktime_add(last_jiffies_update, tick_period));
  78
  79		/* Slow path for long timeouts */
  80		if (unlikely(delta >= tick_period)) {
  81			s64 incr = ktime_to_ns(tick_period);
  82
  83			ticks = ktime_divns(delta, incr);
  84
  85			/* Pairs with the lockless read in this function. */
  86			WRITE_ONCE(last_jiffies_update,
  87				   ktime_add_ns(last_jiffies_update,
  88						incr * ticks));
  89		}
  90		do_timer(++ticks);
  91
  92		/* Keep the tick_next_period variable up to date */
  93		tick_next_period = ktime_add(last_jiffies_update, tick_period);
  94	} else {
  95		write_seqcount_end(&jiffies_seq);
  96		raw_spin_unlock(&jiffies_lock);
  97		return;
  98	}
  99	write_seqcount_end(&jiffies_seq);
 100	raw_spin_unlock(&jiffies_lock);
 101	update_wall_time();
 102}
 103
 104/*
 105 * Initialize and return retrieve the jiffies update.
 106 */
 107static ktime_t tick_init_jiffy_update(void)
 108{
 109	ktime_t period;
 110
 111	raw_spin_lock(&jiffies_lock);
 112	write_seqcount_begin(&jiffies_seq);
 113	/* Did we start the jiffies update yet ? */
 114	if (last_jiffies_update == 0)
 115		last_jiffies_update = tick_next_period;
 116	period = last_jiffies_update;
 117	write_seqcount_end(&jiffies_seq);
 118	raw_spin_unlock(&jiffies_lock);
 119	return period;
 120}
 121
 122static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
 
 123{
 124	int cpu = smp_processor_id();
 125
 126#ifdef CONFIG_NO_HZ_COMMON
 127	/*
 128	 * Check if the do_timer duty was dropped. We don't care about
 129	 * concurrency: This happens only when the CPU in charge went
 130	 * into a long sleep. If two CPUs happen to assign themselves to
 131	 * this duty, then the jiffies update is still serialized by
 132	 * jiffies_lock.
 133	 *
 134	 * If nohz_full is enabled, this should not happen because the
 135	 * tick_do_timer_cpu never relinquishes.
 136	 */
 137	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
 138#ifdef CONFIG_NO_HZ_FULL
 139		WARN_ON(tick_nohz_full_running);
 140#endif
 141		tick_do_timer_cpu = cpu;
 142	}
 143#endif
 144
 145	/* Check, if the jiffies need an update */
 146	if (tick_do_timer_cpu == cpu)
 147		tick_do_update_jiffies64(now);
 148
 149	if (ts->inidle)
 150		ts->got_idle_tick = 1;
 151}
 152
 153static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 154{
 155#ifdef CONFIG_NO_HZ_COMMON
 156	/*
 157	 * When we are idle and the tick is stopped, we have to touch
 158	 * the watchdog as we might not schedule for a really long
 159	 * time. This happens on complete idle SMP systems while
 160	 * waiting on the login prompt. We also increment the "start of
 161	 * idle" jiffy stamp so the idle accounting adjustment we do
 162	 * when we go busy again does not account too much ticks.
 163	 */
 164	if (ts->tick_stopped) {
 165		touch_softlockup_watchdog_sched();
 166		if (is_idle_task(current))
 167			ts->idle_jiffies++;
 168		/*
 169		 * In case the current tick fired too early past its expected
 170		 * expiration, make sure we don't bypass the next clock reprogramming
 171		 * to the same deadline.
 172		 */
 173		ts->next_tick = 0;
 174	}
 175#endif
 176	update_process_times(user_mode(regs));
 177	profile_tick(CPU_PROFILING);
 178}
 179#endif
 180
 181#ifdef CONFIG_NO_HZ_FULL
 182cpumask_var_t tick_nohz_full_mask;
 
 183bool tick_nohz_full_running;
 184EXPORT_SYMBOL_GPL(tick_nohz_full_running);
 185static atomic_t tick_dep_mask;
 186
 187static bool check_tick_dependency(atomic_t *dep)
 188{
 189	int val = atomic_read(dep);
 190
 191	if (val & TICK_DEP_MASK_POSIX_TIMER) {
 192		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
 193		return true;
 194	}
 195
 196	if (val & TICK_DEP_MASK_PERF_EVENTS) {
 197		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
 198		return true;
 199	}
 200
 201	if (val & TICK_DEP_MASK_SCHED) {
 202		trace_tick_stop(0, TICK_DEP_MASK_SCHED);
 203		return true;
 204	}
 205
 206	if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
 207		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
 208		return true;
 209	}
 210
 211	if (val & TICK_DEP_MASK_RCU) {
 212		trace_tick_stop(0, TICK_DEP_MASK_RCU);
 213		return true;
 214	}
 215
 216	return false;
 217}
 218
 219static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
 220{
 221	lockdep_assert_irqs_disabled();
 222
 223	if (unlikely(!cpu_online(cpu)))
 224		return false;
 225
 226	if (check_tick_dependency(&tick_dep_mask))
 227		return false;
 228
 229	if (check_tick_dependency(&ts->tick_dep_mask))
 230		return false;
 231
 232	if (check_tick_dependency(&current->tick_dep_mask))
 233		return false;
 234
 235	if (check_tick_dependency(&current->signal->tick_dep_mask))
 236		return false;
 237
 238	return true;
 239}
 240
 241static void nohz_full_kick_func(struct irq_work *work)
 242{
 243	/* Empty, the tick restart happens on tick_nohz_irq_exit() */
 244}
 245
 246static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
 247	.func = nohz_full_kick_func,
 248	.flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
 249};
 250
 251/*
 252 * Kick this CPU if it's full dynticks in order to force it to
 253 * re-evaluate its dependency on the tick and restart it if necessary.
 254 * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
 255 * is NMI safe.
 256 */
 257static void tick_nohz_full_kick(void)
 258{
 259	if (!tick_nohz_full_cpu(smp_processor_id()))
 260		return;
 261
 262	irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
 263}
 264
 265/*
 266 * Kick the CPU if it's full dynticks in order to force it to
 267 * re-evaluate its dependency on the tick and restart it if necessary.
 268 */
 269void tick_nohz_full_kick_cpu(int cpu)
 270{
 271	if (!tick_nohz_full_cpu(cpu))
 272		return;
 273
 274	irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
 275}
 276
 277/*
 278 * Kick all full dynticks CPUs in order to force these to re-evaluate
 279 * their dependency on the tick and restart it if necessary.
 280 */
 281static void tick_nohz_full_kick_all(void)
 282{
 283	int cpu;
 284
 285	if (!tick_nohz_full_running)
 286		return;
 287
 288	preempt_disable();
 289	for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
 290		tick_nohz_full_kick_cpu(cpu);
 291	preempt_enable();
 292}
 293
 294static void tick_nohz_dep_set_all(atomic_t *dep,
 295				  enum tick_dep_bits bit)
 296{
 297	int prev;
 298
 299	prev = atomic_fetch_or(BIT(bit), dep);
 300	if (!prev)
 301		tick_nohz_full_kick_all();
 302}
 303
 304/*
 305 * Set a global tick dependency. Used by perf events that rely on freq and
 306 * by unstable clock.
 307 */
 308void tick_nohz_dep_set(enum tick_dep_bits bit)
 309{
 310	tick_nohz_dep_set_all(&tick_dep_mask, bit);
 311}
 312
 313void tick_nohz_dep_clear(enum tick_dep_bits bit)
 314{
 315	atomic_andnot(BIT(bit), &tick_dep_mask);
 316}
 317
 318/*
 319 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
 320 * manage events throttling.
 321 */
 322void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
 323{
 324	int prev;
 325	struct tick_sched *ts;
 326
 327	ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 328
 329	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
 330	if (!prev) {
 331		preempt_disable();
 332		/* Perf needs local kick that is NMI safe */
 333		if (cpu == smp_processor_id()) {
 334			tick_nohz_full_kick();
 335		} else {
 336			/* Remote irq work not NMI-safe */
 337			if (!WARN_ON_ONCE(in_nmi()))
 338				tick_nohz_full_kick_cpu(cpu);
 339		}
 340		preempt_enable();
 341	}
 342}
 343EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
 344
 345void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
 346{
 347	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 348
 349	atomic_andnot(BIT(bit), &ts->tick_dep_mask);
 350}
 351EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
 352
 353/*
 354 * Set a per-task tick dependency. RCU need this. Also posix CPU timers
 355 * in order to elapse per task timers.
 356 */
 357void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
 358{
 359	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
 360		if (tsk == current) {
 361			preempt_disable();
 362			tick_nohz_full_kick();
 363			preempt_enable();
 364		} else {
 365			/*
 366			 * Some future tick_nohz_full_kick_task()
 367			 * should optimize this.
 368			 */
 369			tick_nohz_full_kick_all();
 370		}
 371	}
 372}
 373EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
 374
 375void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
 376{
 377	atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
 378}
 379EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
 380
 381/*
 382 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
 383 * per process timers.
 384 */
 385void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
 386{
 387	tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
 388}
 389
 390void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
 391{
 392	atomic_andnot(BIT(bit), &sig->tick_dep_mask);
 393}
 394
 395/*
 396 * Re-evaluate the need for the tick as we switch the current task.
 397 * It might need the tick due to per task/process properties:
 398 * perf events, posix CPU timers, ...
 399 */
 400void __tick_nohz_task_switch(void)
 401{
 402	unsigned long flags;
 403	struct tick_sched *ts;
 404
 405	local_irq_save(flags);
 406
 407	if (!tick_nohz_full_cpu(smp_processor_id()))
 408		goto out;
 409
 410	ts = this_cpu_ptr(&tick_cpu_sched);
 411
 412	if (ts->tick_stopped) {
 413		if (atomic_read(&current->tick_dep_mask) ||
 414		    atomic_read(&current->signal->tick_dep_mask))
 415			tick_nohz_full_kick();
 416	}
 417out:
 418	local_irq_restore(flags);
 419}
 420
 421/* Get the boot-time nohz CPU list from the kernel parameters. */
 422void __init tick_nohz_full_setup(cpumask_var_t cpumask)
 423{
 424	alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
 425	cpumask_copy(tick_nohz_full_mask, cpumask);
 
 
 
 
 426	tick_nohz_full_running = true;
 
 
 427}
 428EXPORT_SYMBOL_GPL(tick_nohz_full_setup);
 429
 430static int tick_nohz_cpu_down(unsigned int cpu)
 431{
 432	/*
 433	 * The tick_do_timer_cpu CPU handles housekeeping duty (unbound
 434	 * timers, workqueues, timekeeping, ...) on behalf of full dynticks
 435	 * CPUs. It must remain online when nohz full is enabled.
 436	 */
 437	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
 438		return -EBUSY;
 439	return 0;
 440}
 441
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 442void __init tick_nohz_init(void)
 443{
 444	int cpu, ret;
 445
 446	if (!tick_nohz_full_running)
 
 
 
 
 
 
 
 
 447		return;
 
 448
 449	/*
 450	 * Full dynticks uses irq work to drive the tick rescheduling on safe
 451	 * locking contexts. But then we need irq work to raise its own
 452	 * interrupts to avoid circular dependency on the tick
 453	 */
 454	if (!arch_irq_work_has_interrupt()) {
 455		pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
 456		cpumask_clear(tick_nohz_full_mask);
 
 457		tick_nohz_full_running = false;
 458		return;
 459	}
 460
 461	if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
 462			!IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
 463		cpu = smp_processor_id();
 464
 465		if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
 466			pr_warn("NO_HZ: Clearing %d from nohz_full range "
 467				"for timekeeping\n", cpu);
 468			cpumask_clear_cpu(cpu, tick_nohz_full_mask);
 469		}
 470	}
 471
 
 
 
 472	for_each_cpu(cpu, tick_nohz_full_mask)
 473		context_tracking_cpu_set(cpu);
 474
 475	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 476					"kernel/nohz:predown", NULL,
 477					tick_nohz_cpu_down);
 478	WARN_ON(ret < 0);
 479	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 480		cpumask_pr_args(tick_nohz_full_mask));
 
 
 
 
 
 
 481}
 482#endif
 483
 484/*
 485 * NOHZ - aka dynamic tick functionality
 486 */
 487#ifdef CONFIG_NO_HZ_COMMON
 488/*
 489 * NO HZ enabled ?
 490 */
 491bool tick_nohz_enabled __read_mostly  = true;
 492unsigned long tick_nohz_active  __read_mostly;
 493/*
 494 * Enable / Disable tickless mode
 495 */
 496static int __init setup_tick_nohz(char *str)
 497{
 498	return (kstrtobool(str, &tick_nohz_enabled) == 0);
 499}
 500
 501__setup("nohz=", setup_tick_nohz);
 502
 503bool tick_nohz_tick_stopped(void)
 504{
 505	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 506
 507	return ts->tick_stopped;
 508}
 509
 510bool tick_nohz_tick_stopped_cpu(int cpu)
 511{
 512	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
 513
 514	return ts->tick_stopped;
 515}
 516
 517/**
 518 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 519 *
 520 * Called from interrupt entry when the CPU was idle
 521 *
 522 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 523 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 524 * value. We do this unconditionally on any CPU, as we don't know whether the
 525 * CPU, which has the update task assigned is in a long sleep.
 526 */
 527static void tick_nohz_update_jiffies(ktime_t now)
 528{
 529	unsigned long flags;
 530
 531	__this_cpu_write(tick_cpu_sched.idle_waketime, now);
 532
 533	local_irq_save(flags);
 534	tick_do_update_jiffies64(now);
 535	local_irq_restore(flags);
 536
 537	touch_softlockup_watchdog_sched();
 538}
 539
 540/*
 541 * Updates the per-CPU time idle statistics counters
 542 */
 543static void
 544update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
 545{
 546	ktime_t delta;
 547
 548	if (ts->idle_active) {
 549		delta = ktime_sub(now, ts->idle_entrytime);
 550		if (nr_iowait_cpu(cpu) > 0)
 551			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
 552		else
 553			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 554		ts->idle_entrytime = now;
 555	}
 556
 557	if (last_update_time)
 558		*last_update_time = ktime_to_us(now);
 559
 560}
 561
 562static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
 563{
 564	update_ts_time_stats(smp_processor_id(), ts, now, NULL);
 565	ts->idle_active = 0;
 566
 567	sched_clock_idle_wakeup_event();
 568}
 569
 570static void tick_nohz_start_idle(struct tick_sched *ts)
 571{
 572	ts->idle_entrytime = ktime_get();
 
 
 573	ts->idle_active = 1;
 574	sched_clock_idle_sleep_event();
 
 575}
 576
 577/**
 578 * get_cpu_idle_time_us - get the total idle time of a CPU
 579 * @cpu: CPU number to query
 580 * @last_update_time: variable to store update time in. Do not update
 581 * counters if NULL.
 582 *
 583 * Return the cumulative idle time (since boot) for a given
 584 * CPU, in microseconds.
 585 *
 586 * This time is measured via accounting rather than sampling,
 587 * and is as accurate as ktime_get() is.
 588 *
 589 * This function returns -1 if NOHZ is not enabled.
 590 */
 591u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 592{
 593	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 594	ktime_t now, idle;
 595
 596	if (!tick_nohz_active)
 597		return -1;
 598
 599	now = ktime_get();
 600	if (last_update_time) {
 601		update_ts_time_stats(cpu, ts, now, last_update_time);
 602		idle = ts->idle_sleeptime;
 603	} else {
 604		if (ts->idle_active && !nr_iowait_cpu(cpu)) {
 605			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 606
 607			idle = ktime_add(ts->idle_sleeptime, delta);
 608		} else {
 609			idle = ts->idle_sleeptime;
 610		}
 611	}
 612
 613	return ktime_to_us(idle);
 614
 615}
 616EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 617
 618/**
 619 * get_cpu_iowait_time_us - get the total iowait time of a CPU
 620 * @cpu: CPU number to query
 621 * @last_update_time: variable to store update time in. Do not update
 622 * counters if NULL.
 623 *
 624 * Return the cumulative iowait time (since boot) for a given
 625 * CPU, in microseconds.
 626 *
 627 * This time is measured via accounting rather than sampling,
 628 * and is as accurate as ktime_get() is.
 629 *
 630 * This function returns -1 if NOHZ is not enabled.
 631 */
 632u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 633{
 634	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 635	ktime_t now, iowait;
 636
 637	if (!tick_nohz_active)
 638		return -1;
 639
 640	now = ktime_get();
 641	if (last_update_time) {
 642		update_ts_time_stats(cpu, ts, now, last_update_time);
 643		iowait = ts->iowait_sleeptime;
 644	} else {
 645		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
 646			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 647
 648			iowait = ktime_add(ts->iowait_sleeptime, delta);
 649		} else {
 650			iowait = ts->iowait_sleeptime;
 651		}
 652	}
 653
 654	return ktime_to_us(iowait);
 655}
 656EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 657
 658static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 659{
 660	hrtimer_cancel(&ts->sched_timer);
 661	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 662
 663	/* Forward the time to expire in the future */
 664	hrtimer_forward(&ts->sched_timer, now, tick_period);
 665
 666	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 667		hrtimer_start_expires(&ts->sched_timer,
 668				      HRTIMER_MODE_ABS_PINNED_HARD);
 669	} else {
 670		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
 671	}
 672
 673	/*
 674	 * Reset to make sure next tick stop doesn't get fooled by past
 675	 * cached clock deadline.
 676	 */
 677	ts->next_tick = 0;
 678}
 679
 680static inline bool local_timer_softirq_pending(void)
 681{
 682	return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
 683}
 684
 685static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 686{
 
 687	u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
 688	unsigned long basejiff;
 689	unsigned int seq;
 690
 691	/* Read jiffies and the time when jiffies were updated last */
 692	do {
 693		seq = read_seqcount_begin(&jiffies_seq);
 694		basemono = last_jiffies_update;
 695		basejiff = jiffies;
 696	} while (read_seqcount_retry(&jiffies_seq, seq));
 697	ts->last_jiffies = basejiff;
 698	ts->timer_expires_base = basemono;
 699
 700	/*
 701	 * Keep the periodic tick, when RCU, architecture or irq_work
 702	 * requests it.
 703	 * Aside of that check whether the local timer softirq is
 704	 * pending. If so its a bad idea to call get_next_timer_interrupt()
 705	 * because there is an already expired timer, so it will request
 706	 * immeditate expiry, which rearms the hardware timer with a
 707	 * minimal delta which brings us back to this place
 708	 * immediately. Lather, rinse and repeat...
 709	 */
 710	if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
 711	    irq_work_needs_cpu() || local_timer_softirq_pending()) {
 712		next_tick = basemono + TICK_NSEC;
 713	} else {
 714		/*
 715		 * Get the next pending timer. If high resolution
 716		 * timers are enabled this only takes the timer wheel
 717		 * timers into account. If high resolution timers are
 718		 * disabled this also looks at the next expiring
 719		 * hrtimer.
 720		 */
 721		next_tmr = get_next_timer_interrupt(basejiff, basemono);
 722		ts->next_timer = next_tmr;
 723		/* Take the next rcu event into account */
 724		next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
 725	}
 726
 727	/*
 728	 * If the tick is due in the next period, keep it ticking or
 729	 * force prod the timer.
 730	 */
 731	delta = next_tick - basemono;
 732	if (delta <= (u64)TICK_NSEC) {
 
 
 733		/*
 734		 * Tell the timer code that the base is not idle, i.e. undo
 735		 * the effect of get_next_timer_interrupt():
 736		 */
 737		timer_clear_idle();
 738		/*
 739		 * We've not stopped the tick yet, and there's a timer in the
 740		 * next period, so no point in stopping it either, bail.
 741		 */
 742		if (!ts->tick_stopped) {
 743			ts->timer_expires = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 744			goto out;
 745		}
 746	}
 747
 748	/*
 749	 * If this CPU is the one which had the do_timer() duty last, we limit
 750	 * the sleep time to the timekeeping max_deferment value.
 751	 * Otherwise we can sleep as long as we want.
 752	 */
 753	delta = timekeeping_max_deferment();
 754	if (cpu != tick_do_timer_cpu &&
 755	    (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
 756		delta = KTIME_MAX;
 757
 758	/* Calculate the next expiry time */
 759	if (delta < (KTIME_MAX - basemono))
 760		expires = basemono + delta;
 761	else
 762		expires = KTIME_MAX;
 763
 764	ts->timer_expires = min_t(u64, expires, next_tick);
 765
 766out:
 767	return ts->timer_expires;
 768}
 769
 770static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
 771{
 772	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 773	u64 basemono = ts->timer_expires_base;
 774	u64 expires = ts->timer_expires;
 775	ktime_t tick = expires;
 776
 777	/* Make sure we won't be trying to stop it twice in a row. */
 778	ts->timer_expires_base = 0;
 779
 780	/*
 781	 * If this CPU is the one which updates jiffies, then give up
 782	 * the assignment and let it be taken by the CPU which runs
 783	 * the tick timer next, which might be this CPU as well. If we
 784	 * don't drop this here the jiffies might be stale and
 785	 * do_timer() never invoked. Keep track of the fact that it
 786	 * was the one which had the do_timer() duty last.
 
 
 
 787	 */
 
 788	if (cpu == tick_do_timer_cpu) {
 789		tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 790		ts->do_timer_last = 1;
 791	} else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
 
 792		ts->do_timer_last = 0;
 
 
 793	}
 794
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 795	/* Skip reprogram of event if its not changed */
 796	if (ts->tick_stopped && (expires == ts->next_tick)) {
 797		/* Sanity check: make sure clockevent is actually programmed */
 798		if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
 799			return;
 800
 801		WARN_ON_ONCE(1);
 802		printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
 803			    basemono, ts->next_tick, dev->next_event,
 804			    hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
 805	}
 806
 807	/*
 808	 * nohz_stop_sched_tick can be called several times before
 809	 * the nohz_restart_sched_tick is called. This happens when
 810	 * interrupts arrive which do not cause a reschedule. In the
 811	 * first call we save the current tick time, so we can restart
 812	 * the scheduler tick in nohz_restart_sched_tick.
 813	 */
 814	if (!ts->tick_stopped) {
 815		calc_load_nohz_start();
 816		quiet_vmstat();
 
 817
 818		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
 819		ts->tick_stopped = 1;
 820		trace_tick_stop(1, TICK_DEP_MASK_NONE);
 821	}
 822
 823	ts->next_tick = tick;
 824
 825	/*
 826	 * If the expiration time == KTIME_MAX, then we simply stop
 827	 * the tick timer.
 828	 */
 829	if (unlikely(expires == KTIME_MAX)) {
 830		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 831			hrtimer_cancel(&ts->sched_timer);
 832		return;
 833	}
 834
 835	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 836		hrtimer_start(&ts->sched_timer, tick,
 837			      HRTIMER_MODE_ABS_PINNED_HARD);
 838	} else {
 839		hrtimer_set_expires(&ts->sched_timer, tick);
 840		tick_program_event(tick, 1);
 841	}
 842}
 843
 844static void tick_nohz_retain_tick(struct tick_sched *ts)
 845{
 846	ts->timer_expires_base = 0;
 847}
 848
 849#ifdef CONFIG_NO_HZ_FULL
 850static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
 851{
 852	if (tick_nohz_next_event(ts, cpu))
 853		tick_nohz_stop_tick(ts, cpu);
 854	else
 855		tick_nohz_retain_tick(ts);
 856}
 857#endif /* CONFIG_NO_HZ_FULL */
 858
 859static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 860{
 861	/* Update jiffies first */
 862	tick_do_update_jiffies64(now);
 
 863
 864	/*
 865	 * Clear the timer idle flag, so we avoid IPIs on remote queueing and
 866	 * the clock forward checks in the enqueue path:
 867	 */
 868	timer_clear_idle();
 869
 870	calc_load_nohz_stop();
 871	touch_softlockup_watchdog_sched();
 872	/*
 873	 * Cancel the scheduled timer and restore the tick
 874	 */
 875	ts->tick_stopped  = 0;
 876	ts->idle_exittime = now;
 877
 878	tick_nohz_restart(ts, now);
 879}
 880
 881static void tick_nohz_full_update_tick(struct tick_sched *ts)
 882{
 883#ifdef CONFIG_NO_HZ_FULL
 884	int cpu = smp_processor_id();
 885
 886	if (!tick_nohz_full_cpu(cpu))
 887		return;
 888
 889	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
 890		return;
 891
 892	if (can_stop_full_tick(cpu, ts))
 893		tick_nohz_stop_sched_tick(ts, cpu);
 894	else if (ts->tick_stopped)
 895		tick_nohz_restart_sched_tick(ts, ktime_get());
 896#endif
 897}
 898
 899static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 900{
 901	/*
 902	 * If this CPU is offline and it is the one which updates
 903	 * jiffies, then give up the assignment and let it be taken by
 904	 * the CPU which runs the tick timer next. If we don't drop
 905	 * this here the jiffies might be stale and do_timer() never
 906	 * invoked.
 907	 */
 908	if (unlikely(!cpu_online(cpu))) {
 909		if (cpu == tick_do_timer_cpu)
 910			tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 911		/*
 912		 * Make sure the CPU doesn't get fooled by obsolete tick
 913		 * deadline if it comes back online later.
 914		 */
 915		ts->next_tick = 0;
 916		return false;
 917	}
 918
 919	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
 
 920		return false;
 
 921
 922	if (need_resched())
 923		return false;
 924
 925	if (unlikely(local_softirq_pending())) {
 926		static int ratelimit;
 927
 928		if (ratelimit < 10 &&
 929		    (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
 930			pr_warn("NOHZ: local_softirq_pending %02x\n",
 931				(unsigned int) local_softirq_pending());
 932			ratelimit++;
 933		}
 934		return false;
 935	}
 936
 937	if (tick_nohz_full_enabled()) {
 938		/*
 939		 * Keep the tick alive to guarantee timekeeping progression
 940		 * if there are full dynticks CPUs around
 941		 */
 942		if (tick_do_timer_cpu == cpu)
 943			return false;
 944		/*
 945		 * Boot safety: make sure the timekeeping duty has been
 946		 * assigned before entering dyntick-idle mode,
 947		 * tick_do_timer_cpu is TICK_DO_TIMER_BOOT
 948		 */
 949		if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_BOOT))
 950			return false;
 951
 952		/* Should not happen for nohz-full */
 953		if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
 954			return false;
 955	}
 956
 957	return true;
 958}
 959
 960static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 961{
 962	ktime_t expires;
 963	int cpu = smp_processor_id();
 964
 965	/*
 966	 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
 967	 * tick timer expiration time is known already.
 968	 */
 969	if (ts->timer_expires_base)
 970		expires = ts->timer_expires;
 971	else if (can_stop_idle_tick(cpu, ts))
 972		expires = tick_nohz_next_event(ts, cpu);
 973	else
 974		return;
 975
 976	ts->idle_calls++;
 977
 978	if (expires > 0LL) {
 979		int was_stopped = ts->tick_stopped;
 980
 981		tick_nohz_stop_tick(ts, cpu);
 982
 983		ts->idle_sleeps++;
 984		ts->idle_expires = expires;
 
 
 
 985
 986		if (!was_stopped && ts->tick_stopped) {
 987			ts->idle_jiffies = ts->last_jiffies;
 988			nohz_balance_enter_idle(cpu);
 989		}
 990	} else {
 991		tick_nohz_retain_tick(ts);
 992	}
 993}
 994
 995/**
 996 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
 997 *
 998 * When the next event is more than a tick into the future, stop the idle tick
 999 */
1000void tick_nohz_idle_stop_tick(void)
1001{
1002	__tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
1003}
1004
1005void tick_nohz_idle_retain_tick(void)
1006{
1007	tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
1008	/*
1009	 * Undo the effect of get_next_timer_interrupt() called from
1010	 * tick_nohz_next_event().
1011	 */
1012	timer_clear_idle();
1013}
1014
1015/**
1016 * tick_nohz_idle_enter - prepare for entering idle on the current CPU
1017 *
1018 * Called when we start the idle loop.
 
 
1019 */
1020void tick_nohz_idle_enter(void)
1021{
1022	struct tick_sched *ts;
1023
1024	lockdep_assert_irqs_enabled();
 
 
 
 
 
 
 
 
1025
1026	local_irq_disable();
1027
1028	ts = this_cpu_ptr(&tick_cpu_sched);
1029
1030	WARN_ON_ONCE(ts->timer_expires_base);
1031
1032	ts->inidle = 1;
1033	tick_nohz_start_idle(ts);
1034
1035	local_irq_enable();
1036}
1037
1038/**
1039 * tick_nohz_irq_exit - update next tick event from interrupt exit
1040 *
1041 * When an interrupt fires while we are idle and it doesn't cause
1042 * a reschedule, it may still add, modify or delete a timer, enqueue
1043 * an RCU callback, etc...
1044 * So we need to re-calculate and reprogram the next tick event.
1045 */
1046void tick_nohz_irq_exit(void)
1047{
1048	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1049
1050	if (ts->inidle)
1051		tick_nohz_start_idle(ts);
1052	else
1053		tick_nohz_full_update_tick(ts);
1054}
1055
1056/**
1057 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
1058 */
1059bool tick_nohz_idle_got_tick(void)
1060{
1061	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1062
1063	if (ts->got_idle_tick) {
1064		ts->got_idle_tick = 0;
1065		return true;
1066	}
1067	return false;
1068}
1069
1070/**
1071 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
1072 * or the tick, whatever that expires first. Note that, if the tick has been
1073 * stopped, it returns the next hrtimer.
1074 *
1075 * Called from power state control code with interrupts disabled
1076 */
1077ktime_t tick_nohz_get_next_hrtimer(void)
1078{
1079	return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
1080}
1081
1082/**
1083 * tick_nohz_get_sleep_length - return the expected length of the current sleep
1084 * @delta_next: duration until the next event if the tick cannot be stopped
1085 *
1086 * Called from power state control code with interrupts disabled
1087 */
1088ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
1089{
1090	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
1091	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1092	int cpu = smp_processor_id();
1093	/*
1094	 * The idle entry time is expected to be a sufficient approximation of
1095	 * the current time at this point.
1096	 */
1097	ktime_t now = ts->idle_entrytime;
1098	ktime_t next_event;
1099
1100	WARN_ON_ONCE(!ts->inidle);
1101
1102	*delta_next = ktime_sub(dev->next_event, now);
1103
1104	if (!can_stop_idle_tick(cpu, ts))
1105		return *delta_next;
1106
1107	next_event = tick_nohz_next_event(ts, cpu);
1108	if (!next_event)
1109		return *delta_next;
1110
1111	/*
1112	 * If the next highres timer to expire is earlier than next_event, the
1113	 * idle governor needs to know that.
1114	 */
1115	next_event = min_t(u64, next_event,
1116			   hrtimer_next_event_without(&ts->sched_timer));
1117
1118	return ktime_sub(next_event, now);
1119}
1120
1121/**
1122 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1123 * for a particular CPU.
1124 *
1125 * Called from the schedutil frequency scaling governor in scheduler context.
1126 */
1127unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
1128{
1129	struct tick_sched *ts = tick_get_tick_sched(cpu);
1130
1131	return ts->idle_calls;
1132}
1133
1134/**
1135 * tick_nohz_get_idle_calls - return the current idle calls counter value
1136 *
1137 * Called from the schedutil frequency scaling governor in scheduler context.
1138 */
1139unsigned long tick_nohz_get_idle_calls(void)
1140{
1141	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1142
1143	return ts->idle_calls;
1144}
1145
1146static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
1147{
1148#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1149	unsigned long ticks;
1150
1151	if (vtime_accounting_enabled_this_cpu())
1152		return;
1153	/*
1154	 * We stopped the tick in idle. Update process times would miss the
1155	 * time we slept as update_process_times does only a 1 tick
1156	 * accounting. Enforce that this is accounted to idle !
1157	 */
1158	ticks = jiffies - ts->idle_jiffies;
1159	/*
1160	 * We might be one off. Do not randomly account a huge number of ticks!
1161	 */
1162	if (ticks && ticks < LONG_MAX)
1163		account_idle_ticks(ticks);
1164#endif
1165}
1166
1167static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)
1168{
1169	tick_nohz_restart_sched_tick(ts, now);
1170	tick_nohz_account_idle_ticks(ts);
1171}
1172
1173void tick_nohz_idle_restart_tick(void)
1174{
1175	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1176
1177	if (ts->tick_stopped)
1178		__tick_nohz_idle_restart_tick(ts, ktime_get());
1179}
1180
1181/**
1182 * tick_nohz_idle_exit - restart the idle tick from the idle task
1183 *
1184 * Restart the idle tick when the CPU is woken up from idle
1185 * This also exit the RCU extended quiescent state. The CPU
1186 * can use RCU again after this function is called.
1187 */
1188void tick_nohz_idle_exit(void)
1189{
1190	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1191	bool idle_active, tick_stopped;
1192	ktime_t now;
1193
1194	local_irq_disable();
1195
1196	WARN_ON_ONCE(!ts->inidle);
1197	WARN_ON_ONCE(ts->timer_expires_base);
1198
1199	ts->inidle = 0;
1200	idle_active = ts->idle_active;
1201	tick_stopped = ts->tick_stopped;
1202
1203	if (idle_active || tick_stopped)
1204		now = ktime_get();
1205
1206	if (idle_active)
1207		tick_nohz_stop_idle(ts, now);
1208
1209	if (tick_stopped)
1210		__tick_nohz_idle_restart_tick(ts, now);
 
 
1211
1212	local_irq_enable();
1213}
1214
1215/*
1216 * The nohz low res interrupt handler
1217 */
1218static void tick_nohz_handler(struct clock_event_device *dev)
1219{
1220	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1221	struct pt_regs *regs = get_irq_regs();
1222	ktime_t now = ktime_get();
1223
1224	dev->next_event = KTIME_MAX;
1225
1226	tick_sched_do_timer(ts, now);
1227	tick_sched_handle(ts, regs);
1228
1229	/* No need to reprogram if we are running tickless  */
1230	if (unlikely(ts->tick_stopped))
1231		return;
1232
1233	hrtimer_forward(&ts->sched_timer, now, tick_period);
1234	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1235}
1236
1237static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1238{
1239	if (!tick_nohz_enabled)
1240		return;
1241	ts->nohz_mode = mode;
1242	/* One update is enough */
1243	if (!test_and_set_bit(0, &tick_nohz_active))
1244		timers_update_nohz();
1245}
1246
1247/**
1248 * tick_nohz_switch_to_nohz - switch to nohz mode
1249 */
1250static void tick_nohz_switch_to_nohz(void)
1251{
1252	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1253	ktime_t next;
1254
1255	if (!tick_nohz_enabled)
1256		return;
1257
1258	if (tick_switch_to_oneshot(tick_nohz_handler))
1259		return;
1260
1261	/*
1262	 * Recycle the hrtimer in ts, so we can share the
1263	 * hrtimer_forward with the highres code.
1264	 */
1265	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1266	/* Get the next period */
1267	next = tick_init_jiffy_update();
1268
1269	hrtimer_set_expires(&ts->sched_timer, next);
1270	hrtimer_forward_now(&ts->sched_timer, tick_period);
1271	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1272	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1273}
1274
1275static inline void tick_nohz_irq_enter(void)
1276{
1277	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1278	ktime_t now;
1279
1280	if (!ts->idle_active && !ts->tick_stopped)
1281		return;
1282	now = ktime_get();
1283	if (ts->idle_active)
1284		tick_nohz_stop_idle(ts, now);
1285	if (ts->tick_stopped)
1286		tick_nohz_update_jiffies(now);
1287}
1288
1289#else
1290
1291static inline void tick_nohz_switch_to_nohz(void) { }
1292static inline void tick_nohz_irq_enter(void) { }
1293static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1294
1295#endif /* CONFIG_NO_HZ_COMMON */
1296
1297/*
1298 * Called from irq_enter to notify about the possible interruption of idle()
1299 */
1300void tick_irq_enter(void)
1301{
1302	tick_check_oneshot_broadcast_this_cpu();
1303	tick_nohz_irq_enter();
1304}
1305
1306/*
1307 * High resolution timer specific code
1308 */
1309#ifdef CONFIG_HIGH_RES_TIMERS
1310/*
1311 * We rearm the timer until we get disabled by the idle code.
1312 * Called with interrupts disabled.
1313 */
1314static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1315{
1316	struct tick_sched *ts =
1317		container_of(timer, struct tick_sched, sched_timer);
1318	struct pt_regs *regs = get_irq_regs();
1319	ktime_t now = ktime_get();
1320
1321	tick_sched_do_timer(ts, now);
1322
1323	/*
1324	 * Do not call, when we are not in irq context and have
1325	 * no valid regs pointer
1326	 */
1327	if (regs)
1328		tick_sched_handle(ts, regs);
1329	else
1330		ts->next_tick = 0;
1331
1332	/* No need to reprogram if we are in idle or full dynticks mode */
1333	if (unlikely(ts->tick_stopped))
1334		return HRTIMER_NORESTART;
1335
1336	hrtimer_forward(timer, now, tick_period);
1337
1338	return HRTIMER_RESTART;
1339}
1340
1341static int sched_skew_tick;
1342
1343static int __init skew_tick(char *str)
1344{
1345	get_option(&str, &sched_skew_tick);
1346
1347	return 0;
1348}
1349early_param("skew_tick", skew_tick);
1350
1351/**
1352 * tick_setup_sched_timer - setup the tick emulation timer
1353 */
1354void tick_setup_sched_timer(void)
1355{
1356	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1357	ktime_t now = ktime_get();
1358
1359	/*
1360	 * Emulate tick processing via per-CPU hrtimers:
1361	 */
1362	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1363	ts->sched_timer.function = tick_sched_timer;
1364
1365	/* Get the next period (per-CPU) */
1366	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1367
1368	/* Offset the tick to avert jiffies_lock contention. */
1369	if (sched_skew_tick) {
1370		u64 offset = ktime_to_ns(tick_period) >> 1;
1371		do_div(offset, num_possible_cpus());
1372		offset *= smp_processor_id();
1373		hrtimer_add_expires_ns(&ts->sched_timer, offset);
1374	}
1375
1376	hrtimer_forward(&ts->sched_timer, now, tick_period);
1377	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
1378	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1379}
1380#endif /* HIGH_RES_TIMERS */
1381
1382#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1383void tick_cancel_sched_timer(int cpu)
1384{
1385	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1386
1387# ifdef CONFIG_HIGH_RES_TIMERS
1388	if (ts->sched_timer.base)
1389		hrtimer_cancel(&ts->sched_timer);
1390# endif
1391
1392	memset(ts, 0, sizeof(*ts));
1393}
1394#endif
1395
1396/**
1397 * Async notification about clocksource changes
1398 */
1399void tick_clock_notify(void)
1400{
1401	int cpu;
1402
1403	for_each_possible_cpu(cpu)
1404		set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1405}
1406
1407/*
1408 * Async notification about clock event changes
1409 */
1410void tick_oneshot_notify(void)
1411{
1412	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1413
1414	set_bit(0, &ts->check_clocks);
1415}
1416
1417/**
1418 * Check, if a change happened, which makes oneshot possible.
1419 *
1420 * Called cyclic from the hrtimer softirq (driven by the timer
1421 * softirq) allow_nohz signals, that we can switch into low-res nohz
1422 * mode, because high resolution timers are disabled (either compile
1423 * or runtime). Called with interrupts disabled.
1424 */
1425int tick_check_oneshot_change(int allow_nohz)
1426{
1427	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1428
1429	if (!test_and_clear_bit(0, &ts->check_clocks))
1430		return 0;
1431
1432	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1433		return 0;
1434
1435	if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1436		return 0;
1437
1438	if (!allow_nohz)
1439		return 1;
1440
1441	tick_nohz_switch_to_nohz();
1442	return 0;
1443}