Loading...
1/*
2 * linux/kernel/time/tick-sched.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * No idle tick implementation for low and high resolution timers
9 *
10 * Started by: Thomas Gleixner and Ingo Molnar
11 *
12 * Distribute under GPLv2.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/kernel_stat.h>
19#include <linux/percpu.h>
20#include <linux/profile.h>
21#include <linux/sched.h>
22#include <linux/module.h>
23
24#include <asm/irq_regs.h>
25
26#include "tick-internal.h"
27
28/*
29 * Per cpu nohz control structure
30 */
31static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
32
33/*
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
35 */
36static ktime_t last_jiffies_update;
37
38struct tick_sched *tick_get_tick_sched(int cpu)
39{
40 return &per_cpu(tick_cpu_sched, cpu);
41}
42
43/*
44 * Must be called with interrupts disabled !
45 */
46static void tick_do_update_jiffies64(ktime_t now)
47{
48 unsigned long ticks = 0;
49 ktime_t delta;
50
51 /*
52 * Do a quick check without holding xtime_lock:
53 */
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 < tick_period.tv64)
56 return;
57
58 /* Reevalute with xtime_lock held */
59 write_seqlock(&xtime_lock);
60
61 delta = ktime_sub(now, last_jiffies_update);
62 if (delta.tv64 >= tick_period.tv64) {
63
64 delta = ktime_sub(delta, tick_period);
65 last_jiffies_update = ktime_add(last_jiffies_update,
66 tick_period);
67
68 /* Slow path for long timeouts */
69 if (unlikely(delta.tv64 >= tick_period.tv64)) {
70 s64 incr = ktime_to_ns(tick_period);
71
72 ticks = ktime_divns(delta, incr);
73
74 last_jiffies_update = ktime_add_ns(last_jiffies_update,
75 incr * ticks);
76 }
77 do_timer(++ticks);
78
79 /* Keep the tick_next_period variable up to date */
80 tick_next_period = ktime_add(last_jiffies_update, tick_period);
81 }
82 write_sequnlock(&xtime_lock);
83}
84
85/*
86 * Initialize and return retrieve the jiffies update.
87 */
88static ktime_t tick_init_jiffy_update(void)
89{
90 ktime_t period;
91
92 write_seqlock(&xtime_lock);
93 /* Did we start the jiffies update yet ? */
94 if (last_jiffies_update.tv64 == 0)
95 last_jiffies_update = tick_next_period;
96 period = last_jiffies_update;
97 write_sequnlock(&xtime_lock);
98 return period;
99}
100
101/*
102 * NOHZ - aka dynamic tick functionality
103 */
104#ifdef CONFIG_NO_HZ
105/*
106 * NO HZ enabled ?
107 */
108static int tick_nohz_enabled __read_mostly = 1;
109
110/*
111 * Enable / Disable tickless mode
112 */
113static int __init setup_tick_nohz(char *str)
114{
115 if (!strcmp(str, "off"))
116 tick_nohz_enabled = 0;
117 else if (!strcmp(str, "on"))
118 tick_nohz_enabled = 1;
119 else
120 return 0;
121 return 1;
122}
123
124__setup("nohz=", setup_tick_nohz);
125
126/**
127 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
128 *
129 * Called from interrupt entry when the CPU was idle
130 *
131 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
132 * must be updated. Otherwise an interrupt handler could use a stale jiffy
133 * value. We do this unconditionally on any cpu, as we don't know whether the
134 * cpu, which has the update task assigned is in a long sleep.
135 */
136static void tick_nohz_update_jiffies(ktime_t now)
137{
138 int cpu = smp_processor_id();
139 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
140 unsigned long flags;
141
142 cpumask_clear_cpu(cpu, nohz_cpu_mask);
143 ts->idle_waketime = now;
144
145 local_irq_save(flags);
146 tick_do_update_jiffies64(now);
147 local_irq_restore(flags);
148
149 touch_softlockup_watchdog();
150}
151
152/*
153 * Updates the per cpu time idle statistics counters
154 */
155static void
156update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
157{
158 ktime_t delta;
159
160 if (ts->idle_active) {
161 delta = ktime_sub(now, ts->idle_entrytime);
162 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
163 if (nr_iowait_cpu(cpu) > 0)
164 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
165 ts->idle_entrytime = now;
166 }
167
168 if (last_update_time)
169 *last_update_time = ktime_to_us(now);
170
171}
172
173static void tick_nohz_stop_idle(int cpu, ktime_t now)
174{
175 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
176
177 update_ts_time_stats(cpu, ts, now, NULL);
178 ts->idle_active = 0;
179
180 sched_clock_idle_wakeup_event(0);
181}
182
183static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
184{
185 ktime_t now;
186
187 now = ktime_get();
188
189 update_ts_time_stats(cpu, ts, now, NULL);
190
191 ts->idle_entrytime = now;
192 ts->idle_active = 1;
193 sched_clock_idle_sleep_event();
194 return now;
195}
196
197/**
198 * get_cpu_idle_time_us - get the total idle time of a cpu
199 * @cpu: CPU number to query
200 * @last_update_time: variable to store update time in
201 *
202 * Return the cummulative idle time (since boot) for a given
203 * CPU, in microseconds. The idle time returned includes
204 * the iowait time (unlike what "top" and co report).
205 *
206 * This time is measured via accounting rather than sampling,
207 * and is as accurate as ktime_get() is.
208 *
209 * This function returns -1 if NOHZ is not enabled.
210 */
211u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
212{
213 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
214
215 if (!tick_nohz_enabled)
216 return -1;
217
218 update_ts_time_stats(cpu, ts, ktime_get(), last_update_time);
219
220 return ktime_to_us(ts->idle_sleeptime);
221}
222EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
223
224/*
225 * get_cpu_iowait_time_us - get the total iowait time of a cpu
226 * @cpu: CPU number to query
227 * @last_update_time: variable to store update time in
228 *
229 * Return the cummulative iowait time (since boot) for a given
230 * CPU, in microseconds.
231 *
232 * This time is measured via accounting rather than sampling,
233 * and is as accurate as ktime_get() is.
234 *
235 * This function returns -1 if NOHZ is not enabled.
236 */
237u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
238{
239 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
240
241 if (!tick_nohz_enabled)
242 return -1;
243
244 update_ts_time_stats(cpu, ts, ktime_get(), last_update_time);
245
246 return ktime_to_us(ts->iowait_sleeptime);
247}
248EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
249
250/**
251 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
252 *
253 * When the next event is more than a tick into the future, stop the idle tick
254 * Called either from the idle loop or from irq_exit() when an idle period was
255 * just interrupted by an interrupt which did not cause a reschedule.
256 */
257void tick_nohz_stop_sched_tick(int inidle)
258{
259 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
260 struct tick_sched *ts;
261 ktime_t last_update, expires, now;
262 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
263 u64 time_delta;
264 int cpu;
265
266 local_irq_save(flags);
267
268 cpu = smp_processor_id();
269 ts = &per_cpu(tick_cpu_sched, cpu);
270
271 /*
272 * Call to tick_nohz_start_idle stops the last_update_time from being
273 * updated. Thus, it must not be called in the event we are called from
274 * irq_exit() with the prior state different than idle.
275 */
276 if (!inidle && !ts->inidle)
277 goto end;
278
279 /*
280 * Set ts->inidle unconditionally. Even if the system did not
281 * switch to NOHZ mode the cpu frequency governers rely on the
282 * update of the idle time accounting in tick_nohz_start_idle().
283 */
284 ts->inidle = 1;
285
286 now = tick_nohz_start_idle(cpu, ts);
287
288 /*
289 * If this cpu is offline and it is the one which updates
290 * jiffies, then give up the assignment and let it be taken by
291 * the cpu which runs the tick timer next. If we don't drop
292 * this here the jiffies might be stale and do_timer() never
293 * invoked.
294 */
295 if (unlikely(!cpu_online(cpu))) {
296 if (cpu == tick_do_timer_cpu)
297 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
298 }
299
300 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
301 goto end;
302
303 if (need_resched())
304 goto end;
305
306 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
307 static int ratelimit;
308
309 if (ratelimit < 10) {
310 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
311 (unsigned int) local_softirq_pending());
312 ratelimit++;
313 }
314 goto end;
315 }
316
317 ts->idle_calls++;
318 /* Read jiffies and the time when jiffies were updated last */
319 do {
320 seq = read_seqbegin(&xtime_lock);
321 last_update = last_jiffies_update;
322 last_jiffies = jiffies;
323 time_delta = timekeeping_max_deferment();
324 } while (read_seqretry(&xtime_lock, seq));
325
326 if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) ||
327 arch_needs_cpu(cpu)) {
328 next_jiffies = last_jiffies + 1;
329 delta_jiffies = 1;
330 } else {
331 /* Get the next timer wheel timer */
332 next_jiffies = get_next_timer_interrupt(last_jiffies);
333 delta_jiffies = next_jiffies - last_jiffies;
334 }
335 /*
336 * Do not stop the tick, if we are only one off
337 * or if the cpu is required for rcu
338 */
339 if (!ts->tick_stopped && delta_jiffies == 1)
340 goto out;
341
342 /* Schedule the tick, if we are at least one jiffie off */
343 if ((long)delta_jiffies >= 1) {
344
345 /*
346 * If this cpu is the one which updates jiffies, then
347 * give up the assignment and let it be taken by the
348 * cpu which runs the tick timer next, which might be
349 * this cpu as well. If we don't drop this here the
350 * jiffies might be stale and do_timer() never
351 * invoked. Keep track of the fact that it was the one
352 * which had the do_timer() duty last. If this cpu is
353 * the one which had the do_timer() duty last, we
354 * limit the sleep time to the timekeeping
355 * max_deferement value which we retrieved
356 * above. Otherwise we can sleep as long as we want.
357 */
358 if (cpu == tick_do_timer_cpu) {
359 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
360 ts->do_timer_last = 1;
361 } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
362 time_delta = KTIME_MAX;
363 ts->do_timer_last = 0;
364 } else if (!ts->do_timer_last) {
365 time_delta = KTIME_MAX;
366 }
367
368 /*
369 * calculate the expiry time for the next timer wheel
370 * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals
371 * that there is no timer pending or at least extremely
372 * far into the future (12 days for HZ=1000). In this
373 * case we set the expiry to the end of time.
374 */
375 if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
376 /*
377 * Calculate the time delta for the next timer event.
378 * If the time delta exceeds the maximum time delta
379 * permitted by the current clocksource then adjust
380 * the time delta accordingly to ensure the
381 * clocksource does not wrap.
382 */
383 time_delta = min_t(u64, time_delta,
384 tick_period.tv64 * delta_jiffies);
385 }
386
387 if (time_delta < KTIME_MAX)
388 expires = ktime_add_ns(last_update, time_delta);
389 else
390 expires.tv64 = KTIME_MAX;
391
392 if (delta_jiffies > 1)
393 cpumask_set_cpu(cpu, nohz_cpu_mask);
394
395 /* Skip reprogram of event if its not changed */
396 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
397 goto out;
398
399 /*
400 * nohz_stop_sched_tick can be called several times before
401 * the nohz_restart_sched_tick is called. This happens when
402 * interrupts arrive which do not cause a reschedule. In the
403 * first call we save the current tick time, so we can restart
404 * the scheduler tick in nohz_restart_sched_tick.
405 */
406 if (!ts->tick_stopped) {
407 select_nohz_load_balancer(1);
408
409 ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
410 ts->tick_stopped = 1;
411 ts->idle_jiffies = last_jiffies;
412 rcu_enter_nohz();
413 }
414
415 ts->idle_sleeps++;
416
417 /* Mark expires */
418 ts->idle_expires = expires;
419
420 /*
421 * If the expiration time == KTIME_MAX, then
422 * in this case we simply stop the tick timer.
423 */
424 if (unlikely(expires.tv64 == KTIME_MAX)) {
425 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
426 hrtimer_cancel(&ts->sched_timer);
427 goto out;
428 }
429
430 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
431 hrtimer_start(&ts->sched_timer, expires,
432 HRTIMER_MODE_ABS_PINNED);
433 /* Check, if the timer was already in the past */
434 if (hrtimer_active(&ts->sched_timer))
435 goto out;
436 } else if (!tick_program_event(expires, 0))
437 goto out;
438 /*
439 * We are past the event already. So we crossed a
440 * jiffie boundary. Update jiffies and raise the
441 * softirq.
442 */
443 tick_do_update_jiffies64(ktime_get());
444 cpumask_clear_cpu(cpu, nohz_cpu_mask);
445 }
446 raise_softirq_irqoff(TIMER_SOFTIRQ);
447out:
448 ts->next_jiffies = next_jiffies;
449 ts->last_jiffies = last_jiffies;
450 ts->sleep_length = ktime_sub(dev->next_event, now);
451end:
452 local_irq_restore(flags);
453}
454
455/**
456 * tick_nohz_get_sleep_length - return the length of the current sleep
457 *
458 * Called from power state control code with interrupts disabled
459 */
460ktime_t tick_nohz_get_sleep_length(void)
461{
462 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
463
464 return ts->sleep_length;
465}
466
467static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
468{
469 hrtimer_cancel(&ts->sched_timer);
470 hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
471
472 while (1) {
473 /* Forward the time to expire in the future */
474 hrtimer_forward(&ts->sched_timer, now, tick_period);
475
476 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
477 hrtimer_start_expires(&ts->sched_timer,
478 HRTIMER_MODE_ABS_PINNED);
479 /* Check, if the timer was already in the past */
480 if (hrtimer_active(&ts->sched_timer))
481 break;
482 } else {
483 if (!tick_program_event(
484 hrtimer_get_expires(&ts->sched_timer), 0))
485 break;
486 }
487 /* Update jiffies and reread time */
488 tick_do_update_jiffies64(now);
489 now = ktime_get();
490 }
491}
492
493/**
494 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
495 *
496 * Restart the idle tick when the CPU is woken up from idle
497 */
498void tick_nohz_restart_sched_tick(void)
499{
500 int cpu = smp_processor_id();
501 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
502#ifndef CONFIG_VIRT_CPU_ACCOUNTING
503 unsigned long ticks;
504#endif
505 ktime_t now;
506
507 local_irq_disable();
508 if (ts->idle_active || (ts->inidle && ts->tick_stopped))
509 now = ktime_get();
510
511 if (ts->idle_active)
512 tick_nohz_stop_idle(cpu, now);
513
514 if (!ts->inidle || !ts->tick_stopped) {
515 ts->inidle = 0;
516 local_irq_enable();
517 return;
518 }
519
520 ts->inidle = 0;
521
522 rcu_exit_nohz();
523
524 /* Update jiffies first */
525 select_nohz_load_balancer(0);
526 tick_do_update_jiffies64(now);
527 cpumask_clear_cpu(cpu, nohz_cpu_mask);
528
529#ifndef CONFIG_VIRT_CPU_ACCOUNTING
530 /*
531 * We stopped the tick in idle. Update process times would miss the
532 * time we slept as update_process_times does only a 1 tick
533 * accounting. Enforce that this is accounted to idle !
534 */
535 ticks = jiffies - ts->idle_jiffies;
536 /*
537 * We might be one off. Do not randomly account a huge number of ticks!
538 */
539 if (ticks && ticks < LONG_MAX)
540 account_idle_ticks(ticks);
541#endif
542
543 touch_softlockup_watchdog();
544 /*
545 * Cancel the scheduled timer and restore the tick
546 */
547 ts->tick_stopped = 0;
548 ts->idle_exittime = now;
549
550 tick_nohz_restart(ts, now);
551
552 local_irq_enable();
553}
554
555static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
556{
557 hrtimer_forward(&ts->sched_timer, now, tick_period);
558 return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
559}
560
561/*
562 * The nohz low res interrupt handler
563 */
564static void tick_nohz_handler(struct clock_event_device *dev)
565{
566 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
567 struct pt_regs *regs = get_irq_regs();
568 int cpu = smp_processor_id();
569 ktime_t now = ktime_get();
570
571 dev->next_event.tv64 = KTIME_MAX;
572
573 /*
574 * Check if the do_timer duty was dropped. We don't care about
575 * concurrency: This happens only when the cpu in charge went
576 * into a long sleep. If two cpus happen to assign themself to
577 * this duty, then the jiffies update is still serialized by
578 * xtime_lock.
579 */
580 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
581 tick_do_timer_cpu = cpu;
582
583 /* Check, if the jiffies need an update */
584 if (tick_do_timer_cpu == cpu)
585 tick_do_update_jiffies64(now);
586
587 /*
588 * When we are idle and the tick is stopped, we have to touch
589 * the watchdog as we might not schedule for a really long
590 * time. This happens on complete idle SMP systems while
591 * waiting on the login prompt. We also increment the "start
592 * of idle" jiffy stamp so the idle accounting adjustment we
593 * do when we go busy again does not account too much ticks.
594 */
595 if (ts->tick_stopped) {
596 touch_softlockup_watchdog();
597 ts->idle_jiffies++;
598 }
599
600 update_process_times(user_mode(regs));
601 profile_tick(CPU_PROFILING);
602
603 while (tick_nohz_reprogram(ts, now)) {
604 now = ktime_get();
605 tick_do_update_jiffies64(now);
606 }
607}
608
609/**
610 * tick_nohz_switch_to_nohz - switch to nohz mode
611 */
612static void tick_nohz_switch_to_nohz(void)
613{
614 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
615 ktime_t next;
616
617 if (!tick_nohz_enabled)
618 return;
619
620 local_irq_disable();
621 if (tick_switch_to_oneshot(tick_nohz_handler)) {
622 local_irq_enable();
623 return;
624 }
625
626 ts->nohz_mode = NOHZ_MODE_LOWRES;
627
628 /*
629 * Recycle the hrtimer in ts, so we can share the
630 * hrtimer_forward with the highres code.
631 */
632 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
633 /* Get the next period */
634 next = tick_init_jiffy_update();
635
636 for (;;) {
637 hrtimer_set_expires(&ts->sched_timer, next);
638 if (!tick_program_event(next, 0))
639 break;
640 next = ktime_add(next, tick_period);
641 }
642 local_irq_enable();
643
644 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n", smp_processor_id());
645}
646
647/*
648 * When NOHZ is enabled and the tick is stopped, we need to kick the
649 * tick timer from irq_enter() so that the jiffies update is kept
650 * alive during long running softirqs. That's ugly as hell, but
651 * correctness is key even if we need to fix the offending softirq in
652 * the first place.
653 *
654 * Note, this is different to tick_nohz_restart. We just kick the
655 * timer and do not touch the other magic bits which need to be done
656 * when idle is left.
657 */
658static void tick_nohz_kick_tick(int cpu, ktime_t now)
659{
660#if 0
661 /* Switch back to 2.6.27 behaviour */
662
663 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
664 ktime_t delta;
665
666 /*
667 * Do not touch the tick device, when the next expiry is either
668 * already reached or less/equal than the tick period.
669 */
670 delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
671 if (delta.tv64 <= tick_period.tv64)
672 return;
673
674 tick_nohz_restart(ts, now);
675#endif
676}
677
678static inline void tick_check_nohz(int cpu)
679{
680 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
681 ktime_t now;
682
683 if (!ts->idle_active && !ts->tick_stopped)
684 return;
685 now = ktime_get();
686 if (ts->idle_active)
687 tick_nohz_stop_idle(cpu, now);
688 if (ts->tick_stopped) {
689 tick_nohz_update_jiffies(now);
690 tick_nohz_kick_tick(cpu, now);
691 }
692}
693
694#else
695
696static inline void tick_nohz_switch_to_nohz(void) { }
697static inline void tick_check_nohz(int cpu) { }
698
699#endif /* NO_HZ */
700
701/*
702 * Called from irq_enter to notify about the possible interruption of idle()
703 */
704void tick_check_idle(int cpu)
705{
706 tick_check_oneshot_broadcast(cpu);
707 tick_check_nohz(cpu);
708}
709
710/*
711 * High resolution timer specific code
712 */
713#ifdef CONFIG_HIGH_RES_TIMERS
714/*
715 * We rearm the timer until we get disabled by the idle code.
716 * Called with interrupts disabled and timer->base->cpu_base->lock held.
717 */
718static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
719{
720 struct tick_sched *ts =
721 container_of(timer, struct tick_sched, sched_timer);
722 struct pt_regs *regs = get_irq_regs();
723 ktime_t now = ktime_get();
724 int cpu = smp_processor_id();
725
726#ifdef CONFIG_NO_HZ
727 /*
728 * Check if the do_timer duty was dropped. We don't care about
729 * concurrency: This happens only when the cpu in charge went
730 * into a long sleep. If two cpus happen to assign themself to
731 * this duty, then the jiffies update is still serialized by
732 * xtime_lock.
733 */
734 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
735 tick_do_timer_cpu = cpu;
736#endif
737
738 /* Check, if the jiffies need an update */
739 if (tick_do_timer_cpu == cpu)
740 tick_do_update_jiffies64(now);
741
742 /*
743 * Do not call, when we are not in irq context and have
744 * no valid regs pointer
745 */
746 if (regs) {
747 /*
748 * When we are idle and the tick is stopped, we have to touch
749 * the watchdog as we might not schedule for a really long
750 * time. This happens on complete idle SMP systems while
751 * waiting on the login prompt. We also increment the "start of
752 * idle" jiffy stamp so the idle accounting adjustment we do
753 * when we go busy again does not account too much ticks.
754 */
755 if (ts->tick_stopped) {
756 touch_softlockup_watchdog();
757 ts->idle_jiffies++;
758 }
759 update_process_times(user_mode(regs));
760 profile_tick(CPU_PROFILING);
761 }
762
763 hrtimer_forward(timer, now, tick_period);
764
765 return HRTIMER_RESTART;
766}
767
768/**
769 * tick_setup_sched_timer - setup the tick emulation timer
770 */
771void tick_setup_sched_timer(void)
772{
773 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
774 ktime_t now = ktime_get();
775
776 /*
777 * Emulate tick processing via per-CPU hrtimers:
778 */
779 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
780 ts->sched_timer.function = tick_sched_timer;
781
782 /* Get the next period (per cpu) */
783 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
784
785 for (;;) {
786 hrtimer_forward(&ts->sched_timer, now, tick_period);
787 hrtimer_start_expires(&ts->sched_timer,
788 HRTIMER_MODE_ABS_PINNED);
789 /* Check, if the timer was already in the past */
790 if (hrtimer_active(&ts->sched_timer))
791 break;
792 now = ktime_get();
793 }
794
795#ifdef CONFIG_NO_HZ
796 if (tick_nohz_enabled) {
797 ts->nohz_mode = NOHZ_MODE_HIGHRES;
798 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n", smp_processor_id());
799 }
800#endif
801}
802#endif /* HIGH_RES_TIMERS */
803
804#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
805void tick_cancel_sched_timer(int cpu)
806{
807 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
808
809# ifdef CONFIG_HIGH_RES_TIMERS
810 if (ts->sched_timer.base)
811 hrtimer_cancel(&ts->sched_timer);
812# endif
813
814 ts->nohz_mode = NOHZ_MODE_INACTIVE;
815}
816#endif
817
818/**
819 * Async notification about clocksource changes
820 */
821void tick_clock_notify(void)
822{
823 int cpu;
824
825 for_each_possible_cpu(cpu)
826 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
827}
828
829/*
830 * Async notification about clock event changes
831 */
832void tick_oneshot_notify(void)
833{
834 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
835
836 set_bit(0, &ts->check_clocks);
837}
838
839/**
840 * Check, if a change happened, which makes oneshot possible.
841 *
842 * Called cyclic from the hrtimer softirq (driven by the timer
843 * softirq) allow_nohz signals, that we can switch into low-res nohz
844 * mode, because high resolution timers are disabled (either compile
845 * or runtime).
846 */
847int tick_check_oneshot_change(int allow_nohz)
848{
849 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
850
851 if (!test_and_clear_bit(0, &ts->check_clocks))
852 return 0;
853
854 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
855 return 0;
856
857 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
858 return 0;
859
860 if (!allow_nohz)
861 return 1;
862
863 tick_nohz_switch_to_nohz();
864 return 0;
865}
1/*
2 * linux/kernel/time/tick-sched.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * No idle tick implementation for low and high resolution timers
9 *
10 * Started by: Thomas Gleixner and Ingo Molnar
11 *
12 * Distribute under GPLv2.
13 */
14#include <linux/cpu.h>
15#include <linux/err.h>
16#include <linux/hrtimer.h>
17#include <linux/interrupt.h>
18#include <linux/kernel_stat.h>
19#include <linux/percpu.h>
20#include <linux/profile.h>
21#include <linux/sched.h>
22#include <linux/module.h>
23
24#include <asm/irq_regs.h>
25
26#include "tick-internal.h"
27
28/*
29 * Per cpu nohz control structure
30 */
31static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
32
33/*
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
35 */
36static ktime_t last_jiffies_update;
37
38struct tick_sched *tick_get_tick_sched(int cpu)
39{
40 return &per_cpu(tick_cpu_sched, cpu);
41}
42
43/*
44 * Must be called with interrupts disabled !
45 */
46static void tick_do_update_jiffies64(ktime_t now)
47{
48 unsigned long ticks = 0;
49 ktime_t delta;
50
51 /*
52 * Do a quick check without holding xtime_lock:
53 */
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 < tick_period.tv64)
56 return;
57
58 /* Reevalute with xtime_lock held */
59 write_seqlock(&xtime_lock);
60
61 delta = ktime_sub(now, last_jiffies_update);
62 if (delta.tv64 >= tick_period.tv64) {
63
64 delta = ktime_sub(delta, tick_period);
65 last_jiffies_update = ktime_add(last_jiffies_update,
66 tick_period);
67
68 /* Slow path for long timeouts */
69 if (unlikely(delta.tv64 >= tick_period.tv64)) {
70 s64 incr = ktime_to_ns(tick_period);
71
72 ticks = ktime_divns(delta, incr);
73
74 last_jiffies_update = ktime_add_ns(last_jiffies_update,
75 incr * ticks);
76 }
77 do_timer(++ticks);
78
79 /* Keep the tick_next_period variable up to date */
80 tick_next_period = ktime_add(last_jiffies_update, tick_period);
81 }
82 write_sequnlock(&xtime_lock);
83}
84
85/*
86 * Initialize and return retrieve the jiffies update.
87 */
88static ktime_t tick_init_jiffy_update(void)
89{
90 ktime_t period;
91
92 write_seqlock(&xtime_lock);
93 /* Did we start the jiffies update yet ? */
94 if (last_jiffies_update.tv64 == 0)
95 last_jiffies_update = tick_next_period;
96 period = last_jiffies_update;
97 write_sequnlock(&xtime_lock);
98 return period;
99}
100
101/*
102 * NOHZ - aka dynamic tick functionality
103 */
104#ifdef CONFIG_NO_HZ
105/*
106 * NO HZ enabled ?
107 */
108static int tick_nohz_enabled __read_mostly = 1;
109
110/*
111 * Enable / Disable tickless mode
112 */
113static int __init setup_tick_nohz(char *str)
114{
115 if (!strcmp(str, "off"))
116 tick_nohz_enabled = 0;
117 else if (!strcmp(str, "on"))
118 tick_nohz_enabled = 1;
119 else
120 return 0;
121 return 1;
122}
123
124__setup("nohz=", setup_tick_nohz);
125
126/**
127 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
128 *
129 * Called from interrupt entry when the CPU was idle
130 *
131 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
132 * must be updated. Otherwise an interrupt handler could use a stale jiffy
133 * value. We do this unconditionally on any cpu, as we don't know whether the
134 * cpu, which has the update task assigned is in a long sleep.
135 */
136static void tick_nohz_update_jiffies(ktime_t now)
137{
138 int cpu = smp_processor_id();
139 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
140 unsigned long flags;
141
142 ts->idle_waketime = now;
143
144 local_irq_save(flags);
145 tick_do_update_jiffies64(now);
146 local_irq_restore(flags);
147
148 calc_load_exit_idle();
149 touch_softlockup_watchdog();
150}
151
152/*
153 * Updates the per cpu time idle statistics counters
154 */
155static void
156update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
157{
158 ktime_t delta;
159
160 if (ts->idle_active) {
161 delta = ktime_sub(now, ts->idle_entrytime);
162 if (nr_iowait_cpu(cpu) > 0)
163 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
164 else
165 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
166 ts->idle_entrytime = now;
167 }
168
169 if (last_update_time)
170 *last_update_time = ktime_to_us(now);
171
172}
173
174static void tick_nohz_stop_idle(int cpu, ktime_t now)
175{
176 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
177
178 update_ts_time_stats(cpu, ts, now, NULL);
179 ts->idle_active = 0;
180
181 sched_clock_idle_wakeup_event(0);
182}
183
184static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
185{
186 ktime_t now = ktime_get();
187
188 ts->idle_entrytime = now;
189 ts->idle_active = 1;
190 sched_clock_idle_sleep_event();
191 return now;
192}
193
194/**
195 * get_cpu_idle_time_us - get the total idle time of a cpu
196 * @cpu: CPU number to query
197 * @last_update_time: variable to store update time in. Do not update
198 * counters if NULL.
199 *
200 * Return the cummulative idle time (since boot) for a given
201 * CPU, in microseconds.
202 *
203 * This time is measured via accounting rather than sampling,
204 * and is as accurate as ktime_get() is.
205 *
206 * This function returns -1 if NOHZ is not enabled.
207 */
208u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
209{
210 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
211 ktime_t now, idle;
212
213 if (!tick_nohz_enabled)
214 return -1;
215
216 now = ktime_get();
217 if (last_update_time) {
218 update_ts_time_stats(cpu, ts, now, last_update_time);
219 idle = ts->idle_sleeptime;
220 } else {
221 if (ts->idle_active && !nr_iowait_cpu(cpu)) {
222 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
223
224 idle = ktime_add(ts->idle_sleeptime, delta);
225 } else {
226 idle = ts->idle_sleeptime;
227 }
228 }
229
230 return ktime_to_us(idle);
231
232}
233EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
234
235/**
236 * get_cpu_iowait_time_us - get the total iowait time of a cpu
237 * @cpu: CPU number to query
238 * @last_update_time: variable to store update time in. Do not update
239 * counters if NULL.
240 *
241 * Return the cummulative iowait time (since boot) for a given
242 * CPU, in microseconds.
243 *
244 * This time is measured via accounting rather than sampling,
245 * and is as accurate as ktime_get() is.
246 *
247 * This function returns -1 if NOHZ is not enabled.
248 */
249u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
250{
251 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
252 ktime_t now, iowait;
253
254 if (!tick_nohz_enabled)
255 return -1;
256
257 now = ktime_get();
258 if (last_update_time) {
259 update_ts_time_stats(cpu, ts, now, last_update_time);
260 iowait = ts->iowait_sleeptime;
261 } else {
262 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
263 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
264
265 iowait = ktime_add(ts->iowait_sleeptime, delta);
266 } else {
267 iowait = ts->iowait_sleeptime;
268 }
269 }
270
271 return ktime_to_us(iowait);
272}
273EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
274
275static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
276{
277 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
278 unsigned long rcu_delta_jiffies;
279 ktime_t last_update, expires, now;
280 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
281 u64 time_delta;
282 int cpu;
283
284 cpu = smp_processor_id();
285 ts = &per_cpu(tick_cpu_sched, cpu);
286
287 now = tick_nohz_start_idle(cpu, ts);
288
289 /*
290 * If this cpu is offline and it is the one which updates
291 * jiffies, then give up the assignment and let it be taken by
292 * the cpu which runs the tick timer next. If we don't drop
293 * this here the jiffies might be stale and do_timer() never
294 * invoked.
295 */
296 if (unlikely(!cpu_online(cpu))) {
297 if (cpu == tick_do_timer_cpu)
298 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
299 }
300
301 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
302 return;
303
304 if (need_resched())
305 return;
306
307 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
308 static int ratelimit;
309
310 if (ratelimit < 10) {
311 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
312 (unsigned int) local_softirq_pending());
313 ratelimit++;
314 }
315 return;
316 }
317
318 ts->idle_calls++;
319 /* Read jiffies and the time when jiffies were updated last */
320 do {
321 seq = read_seqbegin(&xtime_lock);
322 last_update = last_jiffies_update;
323 last_jiffies = jiffies;
324 time_delta = timekeeping_max_deferment();
325 } while (read_seqretry(&xtime_lock, seq));
326
327 if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) ||
328 arch_needs_cpu(cpu)) {
329 next_jiffies = last_jiffies + 1;
330 delta_jiffies = 1;
331 } else {
332 /* Get the next timer wheel timer */
333 next_jiffies = get_next_timer_interrupt(last_jiffies);
334 delta_jiffies = next_jiffies - last_jiffies;
335 if (rcu_delta_jiffies < delta_jiffies) {
336 next_jiffies = last_jiffies + rcu_delta_jiffies;
337 delta_jiffies = rcu_delta_jiffies;
338 }
339 }
340 /*
341 * Do not stop the tick, if we are only one off
342 * or if the cpu is required for rcu
343 */
344 if (!ts->tick_stopped && delta_jiffies == 1)
345 goto out;
346
347 /* Schedule the tick, if we are at least one jiffie off */
348 if ((long)delta_jiffies >= 1) {
349
350 /*
351 * If this cpu is the one which updates jiffies, then
352 * give up the assignment and let it be taken by the
353 * cpu which runs the tick timer next, which might be
354 * this cpu as well. If we don't drop this here the
355 * jiffies might be stale and do_timer() never
356 * invoked. Keep track of the fact that it was the one
357 * which had the do_timer() duty last. If this cpu is
358 * the one which had the do_timer() duty last, we
359 * limit the sleep time to the timekeeping
360 * max_deferement value which we retrieved
361 * above. Otherwise we can sleep as long as we want.
362 */
363 if (cpu == tick_do_timer_cpu) {
364 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
365 ts->do_timer_last = 1;
366 } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
367 time_delta = KTIME_MAX;
368 ts->do_timer_last = 0;
369 } else if (!ts->do_timer_last) {
370 time_delta = KTIME_MAX;
371 }
372
373 /*
374 * calculate the expiry time for the next timer wheel
375 * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals
376 * that there is no timer pending or at least extremely
377 * far into the future (12 days for HZ=1000). In this
378 * case we set the expiry to the end of time.
379 */
380 if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
381 /*
382 * Calculate the time delta for the next timer event.
383 * If the time delta exceeds the maximum time delta
384 * permitted by the current clocksource then adjust
385 * the time delta accordingly to ensure the
386 * clocksource does not wrap.
387 */
388 time_delta = min_t(u64, time_delta,
389 tick_period.tv64 * delta_jiffies);
390 }
391
392 if (time_delta < KTIME_MAX)
393 expires = ktime_add_ns(last_update, time_delta);
394 else
395 expires.tv64 = KTIME_MAX;
396
397 /* Skip reprogram of event if its not changed */
398 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
399 goto out;
400
401 /*
402 * nohz_stop_sched_tick can be called several times before
403 * the nohz_restart_sched_tick is called. This happens when
404 * interrupts arrive which do not cause a reschedule. In the
405 * first call we save the current tick time, so we can restart
406 * the scheduler tick in nohz_restart_sched_tick.
407 */
408 if (!ts->tick_stopped) {
409 select_nohz_load_balancer(1);
410 calc_load_enter_idle();
411
412 ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
413 ts->tick_stopped = 1;
414 ts->idle_jiffies = last_jiffies;
415 }
416
417 ts->idle_sleeps++;
418
419 /* Mark expires */
420 ts->idle_expires = expires;
421
422 /*
423 * If the expiration time == KTIME_MAX, then
424 * in this case we simply stop the tick timer.
425 */
426 if (unlikely(expires.tv64 == KTIME_MAX)) {
427 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
428 hrtimer_cancel(&ts->sched_timer);
429 goto out;
430 }
431
432 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
433 hrtimer_start(&ts->sched_timer, expires,
434 HRTIMER_MODE_ABS_PINNED);
435 /* Check, if the timer was already in the past */
436 if (hrtimer_active(&ts->sched_timer))
437 goto out;
438 } else if (!tick_program_event(expires, 0))
439 goto out;
440 /*
441 * We are past the event already. So we crossed a
442 * jiffie boundary. Update jiffies and raise the
443 * softirq.
444 */
445 tick_do_update_jiffies64(ktime_get());
446 }
447 raise_softirq_irqoff(TIMER_SOFTIRQ);
448out:
449 ts->next_jiffies = next_jiffies;
450 ts->last_jiffies = last_jiffies;
451 ts->sleep_length = ktime_sub(dev->next_event, now);
452}
453
454/**
455 * tick_nohz_idle_enter - stop the idle tick from the idle task
456 *
457 * When the next event is more than a tick into the future, stop the idle tick
458 * Called when we start the idle loop.
459 *
460 * The arch is responsible of calling:
461 *
462 * - rcu_idle_enter() after its last use of RCU before the CPU is put
463 * to sleep.
464 * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
465 */
466void tick_nohz_idle_enter(void)
467{
468 struct tick_sched *ts;
469
470 WARN_ON_ONCE(irqs_disabled());
471
472 /*
473 * Update the idle state in the scheduler domain hierarchy
474 * when tick_nohz_stop_sched_tick() is called from the idle loop.
475 * State will be updated to busy during the first busy tick after
476 * exiting idle.
477 */
478 set_cpu_sd_state_idle();
479
480 local_irq_disable();
481
482 ts = &__get_cpu_var(tick_cpu_sched);
483 /*
484 * set ts->inidle unconditionally. even if the system did not
485 * switch to nohz mode the cpu frequency governers rely on the
486 * update of the idle time accounting in tick_nohz_start_idle().
487 */
488 ts->inidle = 1;
489 tick_nohz_stop_sched_tick(ts);
490
491 local_irq_enable();
492}
493
494/**
495 * tick_nohz_irq_exit - update next tick event from interrupt exit
496 *
497 * When an interrupt fires while we are idle and it doesn't cause
498 * a reschedule, it may still add, modify or delete a timer, enqueue
499 * an RCU callback, etc...
500 * So we need to re-calculate and reprogram the next tick event.
501 */
502void tick_nohz_irq_exit(void)
503{
504 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
505
506 if (!ts->inidle)
507 return;
508
509 tick_nohz_stop_sched_tick(ts);
510}
511
512/**
513 * tick_nohz_get_sleep_length - return the length of the current sleep
514 *
515 * Called from power state control code with interrupts disabled
516 */
517ktime_t tick_nohz_get_sleep_length(void)
518{
519 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
520
521 return ts->sleep_length;
522}
523
524static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
525{
526 hrtimer_cancel(&ts->sched_timer);
527 hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
528
529 while (1) {
530 /* Forward the time to expire in the future */
531 hrtimer_forward(&ts->sched_timer, now, tick_period);
532
533 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
534 hrtimer_start_expires(&ts->sched_timer,
535 HRTIMER_MODE_ABS_PINNED);
536 /* Check, if the timer was already in the past */
537 if (hrtimer_active(&ts->sched_timer))
538 break;
539 } else {
540 if (!tick_program_event(
541 hrtimer_get_expires(&ts->sched_timer), 0))
542 break;
543 }
544 /* Reread time and update jiffies */
545 now = ktime_get();
546 tick_do_update_jiffies64(now);
547 }
548}
549
550/**
551 * tick_nohz_idle_exit - restart the idle tick from the idle task
552 *
553 * Restart the idle tick when the CPU is woken up from idle
554 * This also exit the RCU extended quiescent state. The CPU
555 * can use RCU again after this function is called.
556 */
557void tick_nohz_idle_exit(void)
558{
559 int cpu = smp_processor_id();
560 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
561#ifndef CONFIG_VIRT_CPU_ACCOUNTING
562 unsigned long ticks;
563#endif
564 ktime_t now;
565
566 local_irq_disable();
567
568 WARN_ON_ONCE(!ts->inidle);
569
570 ts->inidle = 0;
571
572 if (ts->idle_active || ts->tick_stopped)
573 now = ktime_get();
574
575 if (ts->idle_active)
576 tick_nohz_stop_idle(cpu, now);
577
578 if (!ts->tick_stopped) {
579 local_irq_enable();
580 return;
581 }
582
583 /* Update jiffies first */
584 select_nohz_load_balancer(0);
585 tick_do_update_jiffies64(now);
586 update_cpu_load_nohz();
587
588#ifndef CONFIG_VIRT_CPU_ACCOUNTING
589 /*
590 * We stopped the tick in idle. Update process times would miss the
591 * time we slept as update_process_times does only a 1 tick
592 * accounting. Enforce that this is accounted to idle !
593 */
594 ticks = jiffies - ts->idle_jiffies;
595 /*
596 * We might be one off. Do not randomly account a huge number of ticks!
597 */
598 if (ticks && ticks < LONG_MAX)
599 account_idle_ticks(ticks);
600#endif
601
602 calc_load_exit_idle();
603 touch_softlockup_watchdog();
604 /*
605 * Cancel the scheduled timer and restore the tick
606 */
607 ts->tick_stopped = 0;
608 ts->idle_exittime = now;
609
610 tick_nohz_restart(ts, now);
611
612 local_irq_enable();
613}
614
615static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
616{
617 hrtimer_forward(&ts->sched_timer, now, tick_period);
618 return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
619}
620
621/*
622 * The nohz low res interrupt handler
623 */
624static void tick_nohz_handler(struct clock_event_device *dev)
625{
626 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
627 struct pt_regs *regs = get_irq_regs();
628 int cpu = smp_processor_id();
629 ktime_t now = ktime_get();
630
631 dev->next_event.tv64 = KTIME_MAX;
632
633 /*
634 * Check if the do_timer duty was dropped. We don't care about
635 * concurrency: This happens only when the cpu in charge went
636 * into a long sleep. If two cpus happen to assign themself to
637 * this duty, then the jiffies update is still serialized by
638 * xtime_lock.
639 */
640 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
641 tick_do_timer_cpu = cpu;
642
643 /* Check, if the jiffies need an update */
644 if (tick_do_timer_cpu == cpu)
645 tick_do_update_jiffies64(now);
646
647 /*
648 * When we are idle and the tick is stopped, we have to touch
649 * the watchdog as we might not schedule for a really long
650 * time. This happens on complete idle SMP systems while
651 * waiting on the login prompt. We also increment the "start
652 * of idle" jiffy stamp so the idle accounting adjustment we
653 * do when we go busy again does not account too much ticks.
654 */
655 if (ts->tick_stopped) {
656 touch_softlockup_watchdog();
657 ts->idle_jiffies++;
658 }
659
660 update_process_times(user_mode(regs));
661 profile_tick(CPU_PROFILING);
662
663 while (tick_nohz_reprogram(ts, now)) {
664 now = ktime_get();
665 tick_do_update_jiffies64(now);
666 }
667}
668
669/**
670 * tick_nohz_switch_to_nohz - switch to nohz mode
671 */
672static void tick_nohz_switch_to_nohz(void)
673{
674 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
675 ktime_t next;
676
677 if (!tick_nohz_enabled)
678 return;
679
680 local_irq_disable();
681 if (tick_switch_to_oneshot(tick_nohz_handler)) {
682 local_irq_enable();
683 return;
684 }
685
686 ts->nohz_mode = NOHZ_MODE_LOWRES;
687
688 /*
689 * Recycle the hrtimer in ts, so we can share the
690 * hrtimer_forward with the highres code.
691 */
692 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
693 /* Get the next period */
694 next = tick_init_jiffy_update();
695
696 for (;;) {
697 hrtimer_set_expires(&ts->sched_timer, next);
698 if (!tick_program_event(next, 0))
699 break;
700 next = ktime_add(next, tick_period);
701 }
702 local_irq_enable();
703}
704
705/*
706 * When NOHZ is enabled and the tick is stopped, we need to kick the
707 * tick timer from irq_enter() so that the jiffies update is kept
708 * alive during long running softirqs. That's ugly as hell, but
709 * correctness is key even if we need to fix the offending softirq in
710 * the first place.
711 *
712 * Note, this is different to tick_nohz_restart. We just kick the
713 * timer and do not touch the other magic bits which need to be done
714 * when idle is left.
715 */
716static void tick_nohz_kick_tick(int cpu, ktime_t now)
717{
718#if 0
719 /* Switch back to 2.6.27 behaviour */
720
721 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
722 ktime_t delta;
723
724 /*
725 * Do not touch the tick device, when the next expiry is either
726 * already reached or less/equal than the tick period.
727 */
728 delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
729 if (delta.tv64 <= tick_period.tv64)
730 return;
731
732 tick_nohz_restart(ts, now);
733#endif
734}
735
736static inline void tick_check_nohz(int cpu)
737{
738 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
739 ktime_t now;
740
741 if (!ts->idle_active && !ts->tick_stopped)
742 return;
743 now = ktime_get();
744 if (ts->idle_active)
745 tick_nohz_stop_idle(cpu, now);
746 if (ts->tick_stopped) {
747 tick_nohz_update_jiffies(now);
748 tick_nohz_kick_tick(cpu, now);
749 }
750}
751
752#else
753
754static inline void tick_nohz_switch_to_nohz(void) { }
755static inline void tick_check_nohz(int cpu) { }
756
757#endif /* NO_HZ */
758
759/*
760 * Called from irq_enter to notify about the possible interruption of idle()
761 */
762void tick_check_idle(int cpu)
763{
764 tick_check_oneshot_broadcast(cpu);
765 tick_check_nohz(cpu);
766}
767
768/*
769 * High resolution timer specific code
770 */
771#ifdef CONFIG_HIGH_RES_TIMERS
772/*
773 * We rearm the timer until we get disabled by the idle code.
774 * Called with interrupts disabled and timer->base->cpu_base->lock held.
775 */
776static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
777{
778 struct tick_sched *ts =
779 container_of(timer, struct tick_sched, sched_timer);
780 struct pt_regs *regs = get_irq_regs();
781 ktime_t now = ktime_get();
782 int cpu = smp_processor_id();
783
784#ifdef CONFIG_NO_HZ
785 /*
786 * Check if the do_timer duty was dropped. We don't care about
787 * concurrency: This happens only when the cpu in charge went
788 * into a long sleep. If two cpus happen to assign themself to
789 * this duty, then the jiffies update is still serialized by
790 * xtime_lock.
791 */
792 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
793 tick_do_timer_cpu = cpu;
794#endif
795
796 /* Check, if the jiffies need an update */
797 if (tick_do_timer_cpu == cpu)
798 tick_do_update_jiffies64(now);
799
800 /*
801 * Do not call, when we are not in irq context and have
802 * no valid regs pointer
803 */
804 if (regs) {
805 /*
806 * When we are idle and the tick is stopped, we have to touch
807 * the watchdog as we might not schedule for a really long
808 * time. This happens on complete idle SMP systems while
809 * waiting on the login prompt. We also increment the "start of
810 * idle" jiffy stamp so the idle accounting adjustment we do
811 * when we go busy again does not account too much ticks.
812 */
813 if (ts->tick_stopped) {
814 touch_softlockup_watchdog();
815 ts->idle_jiffies++;
816 }
817 update_process_times(user_mode(regs));
818 profile_tick(CPU_PROFILING);
819 }
820
821 hrtimer_forward(timer, now, tick_period);
822
823 return HRTIMER_RESTART;
824}
825
826static int sched_skew_tick;
827
828static int __init skew_tick(char *str)
829{
830 get_option(&str, &sched_skew_tick);
831
832 return 0;
833}
834early_param("skew_tick", skew_tick);
835
836/**
837 * tick_setup_sched_timer - setup the tick emulation timer
838 */
839void tick_setup_sched_timer(void)
840{
841 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
842 ktime_t now = ktime_get();
843
844 /*
845 * Emulate tick processing via per-CPU hrtimers:
846 */
847 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
848 ts->sched_timer.function = tick_sched_timer;
849
850 /* Get the next period (per cpu) */
851 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
852
853 /* Offset the tick to avert xtime_lock contention. */
854 if (sched_skew_tick) {
855 u64 offset = ktime_to_ns(tick_period) >> 1;
856 do_div(offset, num_possible_cpus());
857 offset *= smp_processor_id();
858 hrtimer_add_expires_ns(&ts->sched_timer, offset);
859 }
860
861 for (;;) {
862 hrtimer_forward(&ts->sched_timer, now, tick_period);
863 hrtimer_start_expires(&ts->sched_timer,
864 HRTIMER_MODE_ABS_PINNED);
865 /* Check, if the timer was already in the past */
866 if (hrtimer_active(&ts->sched_timer))
867 break;
868 now = ktime_get();
869 }
870
871#ifdef CONFIG_NO_HZ
872 if (tick_nohz_enabled)
873 ts->nohz_mode = NOHZ_MODE_HIGHRES;
874#endif
875}
876#endif /* HIGH_RES_TIMERS */
877
878#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
879void tick_cancel_sched_timer(int cpu)
880{
881 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
882
883# ifdef CONFIG_HIGH_RES_TIMERS
884 if (ts->sched_timer.base)
885 hrtimer_cancel(&ts->sched_timer);
886# endif
887
888 ts->nohz_mode = NOHZ_MODE_INACTIVE;
889}
890#endif
891
892/**
893 * Async notification about clocksource changes
894 */
895void tick_clock_notify(void)
896{
897 int cpu;
898
899 for_each_possible_cpu(cpu)
900 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
901}
902
903/*
904 * Async notification about clock event changes
905 */
906void tick_oneshot_notify(void)
907{
908 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
909
910 set_bit(0, &ts->check_clocks);
911}
912
913/**
914 * Check, if a change happened, which makes oneshot possible.
915 *
916 * Called cyclic from the hrtimer softirq (driven by the timer
917 * softirq) allow_nohz signals, that we can switch into low-res nohz
918 * mode, because high resolution timers are disabled (either compile
919 * or runtime).
920 */
921int tick_check_oneshot_change(int allow_nohz)
922{
923 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
924
925 if (!test_and_clear_bit(0, &ts->check_clocks))
926 return 0;
927
928 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
929 return 0;
930
931 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
932 return 0;
933
934 if (!allow_nohz)
935 return 1;
936
937 tick_nohz_switch_to_nohz();
938 return 0;
939}