2 * linux/kernel/time/tick-sched.c
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
8 * No idle tick implementation for low and high resolution timers
10 * Started by: Thomas Gleixner and Ingo Molnar
12 * Distribute under GPLv2.
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/percpu.h>
20 #include <linux/profile.h>
21 #include <linux/sched.h>
22 #include <linux/tick.h>
24 #include <asm/irq_regs.h>
26 #include "tick-internal.h"
29 * Per cpu nohz control structure
31 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
36 static ktime_t last_jiffies_update;
38 struct tick_sched *tick_get_tick_sched(int cpu)
40 return &per_cpu(tick_cpu_sched, cpu);
44 * Must be called with interrupts disabled !
46 static void tick_do_update_jiffies64(ktime_t now)
48 unsigned long ticks = 0;
51 /* Reevalute with xtime_lock held */
52 write_seqlock(&xtime_lock);
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 >= tick_period.tv64) {
57 delta = ktime_sub(delta, tick_period);
58 last_jiffies_update = ktime_add(last_jiffies_update,
61 /* Slow path for long timeouts */
62 if (unlikely(delta.tv64 >= tick_period.tv64)) {
63 s64 incr = ktime_to_ns(tick_period);
65 ticks = ktime_divns(delta, incr);
67 last_jiffies_update = ktime_add_ns(last_jiffies_update,
72 write_sequnlock(&xtime_lock);
76 * Initialize and return retrieve the jiffies update.
78 static ktime_t tick_init_jiffy_update(void)
82 write_seqlock(&xtime_lock);
83 /* Did we start the jiffies update yet ? */
84 if (last_jiffies_update.tv64 == 0)
85 last_jiffies_update = tick_next_period;
86 period = last_jiffies_update;
87 write_sequnlock(&xtime_lock);
92 * NOHZ - aka dynamic tick functionality
98 static int tick_nohz_enabled __read_mostly = 1;
101 * Enable / Disable tickless mode
103 static int __init setup_tick_nohz(char *str)
105 if (!strcmp(str, "off"))
106 tick_nohz_enabled = 0;
107 else if (!strcmp(str, "on"))
108 tick_nohz_enabled = 1;
114 __setup("nohz=", setup_tick_nohz);
117 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
119 * Called from interrupt entry when the CPU was idle
121 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
122 * must be updated. Otherwise an interrupt handler could use a stale jiffy
123 * value. We do this unconditionally on any cpu, as we don't know whether the
124 * cpu, which has the update task assigned is in a long sleep.
126 void tick_nohz_update_jiffies(void)
128 int cpu = smp_processor_id();
129 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
133 if (!ts->tick_stopped)
136 touch_softlockup_watchdog();
138 cpu_clear(cpu, nohz_cpu_mask);
140 ts->idle_waketime = now;
142 local_irq_save(flags);
143 tick_do_update_jiffies64(now);
144 local_irq_restore(flags);
147 void tick_nohz_stop_idle(int cpu)
149 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
151 if (ts->idle_active) {
154 delta = ktime_sub(now, ts->idle_entrytime);
155 ts->idle_lastupdate = now;
156 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
161 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
166 if (ts->idle_active) {
167 delta = ktime_sub(now, ts->idle_entrytime);
168 ts->idle_lastupdate = now;
169 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
171 ts->idle_entrytime = now;
176 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
178 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
180 *last_update_time = ktime_to_us(ts->idle_lastupdate);
181 return ktime_to_us(ts->idle_sleeptime);
185 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
187 * When the next event is more than a tick into the future, stop the idle tick
188 * Called either from the idle loop or from irq_exit() when an idle period was
189 * just interrupted by an interrupt which did not cause a reschedule.
191 void tick_nohz_stop_sched_tick(void)
193 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
194 struct tick_sched *ts;
195 ktime_t last_update, expires, now;
196 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
199 local_irq_save(flags);
201 cpu = smp_processor_id();
202 ts = &per_cpu(tick_cpu_sched, cpu);
203 now = tick_nohz_start_idle(ts);
206 * If this cpu is offline and it is the one which updates
207 * jiffies, then give up the assignment and let it be taken by
208 * the cpu which runs the tick timer next. If we don't drop
209 * this here the jiffies might be stale and do_timer() never
212 if (unlikely(!cpu_online(cpu))) {
213 if (cpu == tick_do_timer_cpu)
214 tick_do_timer_cpu = -1;
217 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
223 if (unlikely(local_softirq_pending())) {
224 static int ratelimit;
226 if (ratelimit < 10) {
227 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
228 local_softirq_pending());
234 /* Read jiffies and the time when jiffies were updated last */
236 seq = read_seqbegin(&xtime_lock);
237 last_update = last_jiffies_update;
238 last_jiffies = jiffies;
239 } while (read_seqretry(&xtime_lock, seq));
241 /* Get the next timer wheel timer */
242 next_jiffies = get_next_timer_interrupt(last_jiffies);
243 delta_jiffies = next_jiffies - last_jiffies;
245 if (rcu_needs_cpu(cpu))
248 * Do not stop the tick, if we are only one off
249 * or if the cpu is required for rcu
251 if (!ts->tick_stopped && delta_jiffies == 1)
254 /* Schedule the tick, if we are at least one jiffie off */
255 if ((long)delta_jiffies >= 1) {
257 if (delta_jiffies > 1)
258 cpu_set(cpu, nohz_cpu_mask);
260 * nohz_stop_sched_tick can be called several times before
261 * the nohz_restart_sched_tick is called. This happens when
262 * interrupts arrive which do not cause a reschedule. In the
263 * first call we save the current tick time, so we can restart
264 * the scheduler tick in nohz_restart_sched_tick.
266 if (!ts->tick_stopped) {
267 if (select_nohz_load_balancer(1)) {
269 * sched tick not stopped!
271 cpu_clear(cpu, nohz_cpu_mask);
275 ts->idle_tick = ts->sched_timer.expires;
276 ts->tick_stopped = 1;
277 ts->idle_jiffies = last_jiffies;
282 * If this cpu is the one which updates jiffies, then
283 * give up the assignment and let it be taken by the
284 * cpu which runs the tick timer next, which might be
285 * this cpu as well. If we don't drop this here the
286 * jiffies might be stale and do_timer() never
289 if (cpu == tick_do_timer_cpu)
290 tick_do_timer_cpu = -1;
295 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
296 * there is no timer pending or at least extremly far
297 * into the future (12 days for HZ=1000). In this case
298 * we simply stop the tick timer:
300 if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
301 ts->idle_expires.tv64 = KTIME_MAX;
302 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
303 hrtimer_cancel(&ts->sched_timer);
308 * calculate the expiry time for the next timer wheel
311 expires = ktime_add_ns(last_update, tick_period.tv64 *
313 ts->idle_expires = expires;
315 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
316 hrtimer_start(&ts->sched_timer, expires,
318 /* Check, if the timer was already in the past */
319 if (hrtimer_active(&ts->sched_timer))
321 } else if (!tick_program_event(expires, 0))
324 * We are past the event already. So we crossed a
325 * jiffie boundary. Update jiffies and raise the
328 tick_do_update_jiffies64(ktime_get());
329 cpu_clear(cpu, nohz_cpu_mask);
331 raise_softirq_irqoff(TIMER_SOFTIRQ);
333 ts->next_jiffies = next_jiffies;
334 ts->last_jiffies = last_jiffies;
335 ts->sleep_length = ktime_sub(dev->next_event, now);
337 local_irq_restore(flags);
341 * tick_nohz_get_sleep_length - return the length of the current sleep
343 * Called from power state control code with interrupts disabled
345 ktime_t tick_nohz_get_sleep_length(void)
347 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
349 return ts->sleep_length;
353 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
355 * Restart the idle tick when the CPU is woken up from idle
357 void tick_nohz_restart_sched_tick(void)
359 int cpu = smp_processor_id();
360 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
365 tick_nohz_stop_idle(cpu);
367 if (!ts->tick_stopped) {
374 /* Update jiffies first */
375 select_nohz_load_balancer(0);
377 tick_do_update_jiffies64(now);
378 cpu_clear(cpu, nohz_cpu_mask);
381 * We stopped the tick in idle. Update process times would miss the
382 * time we slept as update_process_times does only a 1 tick
383 * accounting. Enforce that this is accounted to idle !
385 ticks = jiffies - ts->idle_jiffies;
387 * We might be one off. Do not randomly account a huge number of ticks!
389 if (ticks && ticks < LONG_MAX) {
390 add_preempt_count(HARDIRQ_OFFSET);
391 account_system_time(current, HARDIRQ_OFFSET,
392 jiffies_to_cputime(ticks));
393 sub_preempt_count(HARDIRQ_OFFSET);
396 touch_softlockup_watchdog();
398 * Cancel the scheduled timer and restore the tick
400 ts->tick_stopped = 0;
401 ts->idle_exittime = now;
402 hrtimer_cancel(&ts->sched_timer);
403 ts->sched_timer.expires = ts->idle_tick;
406 /* Forward the time to expire in the future */
407 hrtimer_forward(&ts->sched_timer, now, tick_period);
409 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
410 hrtimer_start(&ts->sched_timer,
411 ts->sched_timer.expires,
413 /* Check, if the timer was already in the past */
414 if (hrtimer_active(&ts->sched_timer))
417 if (!tick_program_event(ts->sched_timer.expires, 0))
420 /* Update jiffies and reread time */
421 tick_do_update_jiffies64(now);
427 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
429 hrtimer_forward(&ts->sched_timer, now, tick_period);
430 return tick_program_event(ts->sched_timer.expires, 0);
434 * The nohz low res interrupt handler
436 static void tick_nohz_handler(struct clock_event_device *dev)
438 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
439 struct pt_regs *regs = get_irq_regs();
440 int cpu = smp_processor_id();
441 ktime_t now = ktime_get();
443 dev->next_event.tv64 = KTIME_MAX;
446 * Check if the do_timer duty was dropped. We don't care about
447 * concurrency: This happens only when the cpu in charge went
448 * into a long sleep. If two cpus happen to assign themself to
449 * this duty, then the jiffies update is still serialized by
452 if (unlikely(tick_do_timer_cpu == -1))
453 tick_do_timer_cpu = cpu;
455 /* Check, if the jiffies need an update */
456 if (tick_do_timer_cpu == cpu)
457 tick_do_update_jiffies64(now);
460 * When we are idle and the tick is stopped, we have to touch
461 * the watchdog as we might not schedule for a really long
462 * time. This happens on complete idle SMP systems while
463 * waiting on the login prompt. We also increment the "start
464 * of idle" jiffy stamp so the idle accounting adjustment we
465 * do when we go busy again does not account too much ticks.
467 if (ts->tick_stopped) {
468 touch_softlockup_watchdog();
472 update_process_times(user_mode(regs));
473 profile_tick(CPU_PROFILING);
475 /* Do not restart, when we are in the idle loop */
476 if (ts->tick_stopped)
479 while (tick_nohz_reprogram(ts, now)) {
481 tick_do_update_jiffies64(now);
486 * tick_nohz_switch_to_nohz - switch to nohz mode
488 static void tick_nohz_switch_to_nohz(void)
490 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
493 if (!tick_nohz_enabled)
497 if (tick_switch_to_oneshot(tick_nohz_handler)) {
502 ts->nohz_mode = NOHZ_MODE_LOWRES;
505 * Recycle the hrtimer in ts, so we can share the
506 * hrtimer_forward with the highres code.
508 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
509 /* Get the next period */
510 next = tick_init_jiffy_update();
513 ts->sched_timer.expires = next;
514 if (!tick_program_event(next, 0))
516 next = ktime_add(next, tick_period);
520 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
526 static inline void tick_nohz_switch_to_nohz(void) { }
531 * High resolution timer specific code
533 #ifdef CONFIG_HIGH_RES_TIMERS
535 * We rearm the timer until we get disabled by the idle code.
536 * Called with interrupts disabled and timer->base->cpu_base->lock held.
538 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
540 struct tick_sched *ts =
541 container_of(timer, struct tick_sched, sched_timer);
542 struct pt_regs *regs = get_irq_regs();
543 ktime_t now = ktime_get();
544 int cpu = smp_processor_id();
548 * Check if the do_timer duty was dropped. We don't care about
549 * concurrency: This happens only when the cpu in charge went
550 * into a long sleep. If two cpus happen to assign themself to
551 * this duty, then the jiffies update is still serialized by
554 if (unlikely(tick_do_timer_cpu == -1))
555 tick_do_timer_cpu = cpu;
558 /* Check, if the jiffies need an update */
559 if (tick_do_timer_cpu == cpu)
560 tick_do_update_jiffies64(now);
563 * Do not call, when we are not in irq context and have
564 * no valid regs pointer
568 * When we are idle and the tick is stopped, we have to touch
569 * the watchdog as we might not schedule for a really long
570 * time. This happens on complete idle SMP systems while
571 * waiting on the login prompt. We also increment the "start of
572 * idle" jiffy stamp so the idle accounting adjustment we do
573 * when we go busy again does not account too much ticks.
575 if (ts->tick_stopped) {
576 touch_softlockup_watchdog();
579 update_process_times(user_mode(regs));
580 profile_tick(CPU_PROFILING);
583 /* Do not restart, when we are in the idle loop */
584 if (ts->tick_stopped)
585 return HRTIMER_NORESTART;
587 hrtimer_forward(timer, now, tick_period);
589 return HRTIMER_RESTART;
593 * tick_setup_sched_timer - setup the tick emulation timer
595 void tick_setup_sched_timer(void)
597 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
598 ktime_t now = ktime_get();
602 * Emulate tick processing via per-CPU hrtimers:
604 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
605 ts->sched_timer.function = tick_sched_timer;
606 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
608 /* Get the next period (per cpu) */
609 ts->sched_timer.expires = tick_init_jiffy_update();
610 offset = ktime_to_ns(tick_period) >> 1;
611 do_div(offset, num_possible_cpus());
612 offset *= smp_processor_id();
613 ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset);
616 hrtimer_forward(&ts->sched_timer, now, tick_period);
617 hrtimer_start(&ts->sched_timer, ts->sched_timer.expires,
619 /* Check, if the timer was already in the past */
620 if (hrtimer_active(&ts->sched_timer))
626 if (tick_nohz_enabled)
627 ts->nohz_mode = NOHZ_MODE_HIGHRES;
631 void tick_cancel_sched_timer(int cpu)
633 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
635 if (ts->sched_timer.base)
636 hrtimer_cancel(&ts->sched_timer);
638 ts->nohz_mode = NOHZ_MODE_INACTIVE;
640 #endif /* HIGH_RES_TIMERS */
643 * Async notification about clocksource changes
645 void tick_clock_notify(void)
649 for_each_possible_cpu(cpu)
650 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
654 * Async notification about clock event changes
656 void tick_oneshot_notify(void)
658 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
660 set_bit(0, &ts->check_clocks);
664 * Check, if a change happened, which makes oneshot possible.
666 * Called cyclic from the hrtimer softirq (driven by the timer
667 * softirq) allow_nohz signals, that we can switch into low-res nohz
668 * mode, because high resolution timers are disabled (either compile
671 int tick_check_oneshot_change(int allow_nohz)
673 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
675 if (!test_and_clear_bit(0, &ts->check_clocks))
678 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
681 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
687 tick_nohz_switch_to_nohz();