2 * linux/kernel/time/tick-sched.c
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
8 * No idle tick implementation for low and high resolution timers
10 * Started by: Thomas Gleixner and Ingo Molnar
12 * Distribute under GPLv2.
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/percpu.h>
20 #include <linux/profile.h>
21 #include <linux/sched.h>
22 #include <linux/tick.h>
24 #include <asm/irq_regs.h>
26 #include "tick-internal.h"
29 * Per cpu nohz control structure
31 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
36 static ktime_t last_jiffies_update;
38 struct tick_sched *tick_get_tick_sched(int cpu)
40 return &per_cpu(tick_cpu_sched, cpu);
44 * Must be called with interrupts disabled !
46 static void tick_do_update_jiffies64(ktime_t now)
48 unsigned long ticks = 0;
51 /* Reevalute with xtime_lock held */
52 write_seqlock(&xtime_lock);
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 >= tick_period.tv64) {
57 delta = ktime_sub(delta, tick_period);
58 last_jiffies_update = ktime_add(last_jiffies_update,
61 /* Slow path for long timeouts */
62 if (unlikely(delta.tv64 >= tick_period.tv64)) {
63 s64 incr = ktime_to_ns(tick_period);
65 ticks = ktime_divns(delta, incr);
67 last_jiffies_update = ktime_add_ns(last_jiffies_update,
72 write_sequnlock(&xtime_lock);
76 * Initialize and return retrieve the jiffies update.
78 static ktime_t tick_init_jiffy_update(void)
82 write_seqlock(&xtime_lock);
83 /* Did we start the jiffies update yet ? */
84 if (last_jiffies_update.tv64 == 0)
85 last_jiffies_update = tick_next_period;
86 period = last_jiffies_update;
87 write_sequnlock(&xtime_lock);
92 * NOHZ - aka dynamic tick functionality
98 static int tick_nohz_enabled __read_mostly = 1;
101 * Enable / Disable tickless mode
103 static int __init setup_tick_nohz(char *str)
105 if (!strcmp(str, "off"))
106 tick_nohz_enabled = 0;
107 else if (!strcmp(str, "on"))
108 tick_nohz_enabled = 1;
114 __setup("nohz=", setup_tick_nohz);
117 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
119 * Called from interrupt entry when the CPU was idle
121 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
122 * must be updated. Otherwise an interrupt handler could use a stale jiffy
123 * value. We do this unconditionally on any cpu, as we don't know whether the
124 * cpu, which has the update task assigned is in a long sleep.
126 void tick_nohz_update_jiffies(void)
128 int cpu = smp_processor_id();
129 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
133 if (!ts->tick_stopped)
136 touch_softlockup_watchdog();
138 cpu_clear(cpu, nohz_cpu_mask);
140 ts->idle_waketime = now;
142 local_irq_save(flags);
143 tick_do_update_jiffies64(now);
144 local_irq_restore(flags);
147 void tick_nohz_stop_idle(int cpu)
149 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
151 if (ts->idle_active) {
154 delta = ktime_sub(now, ts->idle_entrytime);
155 ts->idle_lastupdate = now;
156 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
161 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
166 if (ts->idle_active) {
167 delta = ktime_sub(now, ts->idle_entrytime);
168 ts->idle_lastupdate = now;
169 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
171 ts->idle_entrytime = now;
176 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
178 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
180 *last_update_time = ktime_to_us(ts->idle_lastupdate);
181 return ktime_to_us(ts->idle_sleeptime);
185 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
187 * When the next event is more than a tick into the future, stop the idle tick
188 * Called either from the idle loop or from irq_exit() when an idle period was
189 * just interrupted by an interrupt which did not cause a reschedule.
191 void tick_nohz_stop_sched_tick(void)
193 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
194 struct tick_sched *ts;
195 ktime_t last_update, expires, now;
196 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
199 local_irq_save(flags);
201 cpu = smp_processor_id();
202 ts = &per_cpu(tick_cpu_sched, cpu);
203 now = tick_nohz_start_idle(ts);
206 * If this cpu is offline and it is the one which updates
207 * jiffies, then give up the assignment and let it be taken by
208 * the cpu which runs the tick timer next. If we don't drop
209 * this here the jiffies might be stale and do_timer() never
212 if (unlikely(!cpu_online(cpu))) {
213 if (cpu == tick_do_timer_cpu)
214 tick_do_timer_cpu = -1;
217 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
223 if (unlikely(local_softirq_pending())) {
224 static int ratelimit;
226 if (ratelimit < 10) {
227 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
228 local_softirq_pending());
234 /* Read jiffies and the time when jiffies were updated last */
236 seq = read_seqbegin(&xtime_lock);
237 last_update = last_jiffies_update;
238 last_jiffies = jiffies;
239 } while (read_seqretry(&xtime_lock, seq));
241 /* Get the next timer wheel timer */
242 next_jiffies = get_next_timer_interrupt(last_jiffies);
243 delta_jiffies = next_jiffies - last_jiffies;
245 if (rcu_needs_cpu(cpu))
248 * Do not stop the tick, if we are only one off
249 * or if the cpu is required for rcu
251 if (!ts->tick_stopped && delta_jiffies == 1)
254 /* Schedule the tick, if we are at least one jiffie off */
255 if ((long)delta_jiffies >= 1) {
257 if (delta_jiffies > 1)
258 cpu_set(cpu, nohz_cpu_mask);
260 * nohz_stop_sched_tick can be called several times before
261 * the nohz_restart_sched_tick is called. This happens when
262 * interrupts arrive which do not cause a reschedule. In the
263 * first call we save the current tick time, so we can restart
264 * the scheduler tick in nohz_restart_sched_tick.
266 if (!ts->tick_stopped) {
267 if (select_nohz_load_balancer(1)) {
269 * sched tick not stopped!
271 cpu_clear(cpu, nohz_cpu_mask);
275 ts->idle_tick = ts->sched_timer.expires;
276 ts->tick_stopped = 1;
277 ts->idle_jiffies = last_jiffies;
282 * If this cpu is the one which updates jiffies, then
283 * give up the assignment and let it be taken by the
284 * cpu which runs the tick timer next, which might be
285 * this cpu as well. If we don't drop this here the
286 * jiffies might be stale and do_timer() never
289 if (cpu == tick_do_timer_cpu)
290 tick_do_timer_cpu = -1;
295 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
296 * there is no timer pending or at least extremly far
297 * into the future (12 days for HZ=1000). In this case
298 * we simply stop the tick timer:
300 if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
301 ts->idle_expires.tv64 = KTIME_MAX;
302 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
303 hrtimer_cancel(&ts->sched_timer);
308 * calculate the expiry time for the next timer wheel
311 expires = ktime_add_ns(last_update, tick_period.tv64 *
313 ts->idle_expires = expires;
315 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
316 hrtimer_start(&ts->sched_timer, expires,
318 /* Check, if the timer was already in the past */
319 if (hrtimer_active(&ts->sched_timer))
321 } else if (!tick_program_event(expires, 0))
324 * We are past the event already. So we crossed a
325 * jiffie boundary. Update jiffies and raise the
328 tick_do_update_jiffies64(ktime_get());
329 cpu_clear(cpu, nohz_cpu_mask);
331 raise_softirq_irqoff(TIMER_SOFTIRQ);
333 ts->next_jiffies = next_jiffies;
334 ts->last_jiffies = last_jiffies;
335 ts->sleep_length = ktime_sub(dev->next_event, now);
337 local_irq_restore(flags);
341 * tick_nohz_get_sleep_length - return the length of the current sleep
343 * Called from power state control code with interrupts disabled
345 ktime_t tick_nohz_get_sleep_length(void)
347 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
349 return ts->sleep_length;
353 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
355 * Restart the idle tick when the CPU is woken up from idle
357 void tick_nohz_restart_sched_tick(void)
359 int cpu = smp_processor_id();
360 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
365 tick_nohz_stop_idle(cpu);
367 if (!ts->tick_stopped) {
374 /* Update jiffies first */
375 select_nohz_load_balancer(0);
377 tick_do_update_jiffies64(now);
378 cpu_clear(cpu, nohz_cpu_mask);
381 * We stopped the tick in idle. Update process times would miss the
382 * time we slept as update_process_times does only a 1 tick
383 * accounting. Enforce that this is accounted to idle !
385 ticks = jiffies - ts->idle_jiffies;
387 * We might be one off. Do not randomly account a huge number of ticks!
389 if (ticks && ticks < LONG_MAX) {
390 add_preempt_count(HARDIRQ_OFFSET);
391 account_system_time(current, HARDIRQ_OFFSET,
392 jiffies_to_cputime(ticks));
393 sub_preempt_count(HARDIRQ_OFFSET);
397 * Cancel the scheduled timer and restore the tick
399 ts->tick_stopped = 0;
400 ts->idle_exittime = now;
401 hrtimer_cancel(&ts->sched_timer);
402 ts->sched_timer.expires = ts->idle_tick;
405 /* Forward the time to expire in the future */
406 hrtimer_forward(&ts->sched_timer, now, tick_period);
408 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
409 hrtimer_start(&ts->sched_timer,
410 ts->sched_timer.expires,
412 /* Check, if the timer was already in the past */
413 if (hrtimer_active(&ts->sched_timer))
416 if (!tick_program_event(ts->sched_timer.expires, 0))
419 /* Update jiffies and reread time */
420 tick_do_update_jiffies64(now);
426 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
428 hrtimer_forward(&ts->sched_timer, now, tick_period);
429 return tick_program_event(ts->sched_timer.expires, 0);
433 * The nohz low res interrupt handler
435 static void tick_nohz_handler(struct clock_event_device *dev)
437 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
438 struct pt_regs *regs = get_irq_regs();
439 int cpu = smp_processor_id();
440 ktime_t now = ktime_get();
442 dev->next_event.tv64 = KTIME_MAX;
445 * Check if the do_timer duty was dropped. We don't care about
446 * concurrency: This happens only when the cpu in charge went
447 * into a long sleep. If two cpus happen to assign themself to
448 * this duty, then the jiffies update is still serialized by
451 if (unlikely(tick_do_timer_cpu == -1))
452 tick_do_timer_cpu = cpu;
454 /* Check, if the jiffies need an update */
455 if (tick_do_timer_cpu == cpu)
456 tick_do_update_jiffies64(now);
459 * When we are idle and the tick is stopped, we have to touch
460 * the watchdog as we might not schedule for a really long
461 * time. This happens on complete idle SMP systems while
462 * waiting on the login prompt. We also increment the "start
463 * of idle" jiffy stamp so the idle accounting adjustment we
464 * do when we go busy again does not account too much ticks.
466 if (ts->tick_stopped) {
467 touch_softlockup_watchdog();
471 update_process_times(user_mode(regs));
472 profile_tick(CPU_PROFILING);
474 /* Do not restart, when we are in the idle loop */
475 if (ts->tick_stopped)
478 while (tick_nohz_reprogram(ts, now)) {
480 tick_do_update_jiffies64(now);
485 * tick_nohz_switch_to_nohz - switch to nohz mode
487 static void tick_nohz_switch_to_nohz(void)
489 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
492 if (!tick_nohz_enabled)
496 if (tick_switch_to_oneshot(tick_nohz_handler)) {
501 ts->nohz_mode = NOHZ_MODE_LOWRES;
504 * Recycle the hrtimer in ts, so we can share the
505 * hrtimer_forward with the highres code.
507 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
508 /* Get the next period */
509 next = tick_init_jiffy_update();
512 ts->sched_timer.expires = next;
513 if (!tick_program_event(next, 0))
515 next = ktime_add(next, tick_period);
519 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
525 static inline void tick_nohz_switch_to_nohz(void) { }
530 * High resolution timer specific code
532 #ifdef CONFIG_HIGH_RES_TIMERS
534 * We rearm the timer until we get disabled by the idle code.
535 * Called with interrupts disabled and timer->base->cpu_base->lock held.
537 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
539 struct tick_sched *ts =
540 container_of(timer, struct tick_sched, sched_timer);
541 struct pt_regs *regs = get_irq_regs();
542 ktime_t now = ktime_get();
543 int cpu = smp_processor_id();
547 * Check if the do_timer duty was dropped. We don't care about
548 * concurrency: This happens only when the cpu in charge went
549 * into a long sleep. If two cpus happen to assign themself to
550 * this duty, then the jiffies update is still serialized by
553 if (unlikely(tick_do_timer_cpu == -1))
554 tick_do_timer_cpu = cpu;
557 /* Check, if the jiffies need an update */
558 if (tick_do_timer_cpu == cpu)
559 tick_do_update_jiffies64(now);
562 * Do not call, when we are not in irq context and have
563 * no valid regs pointer
567 * When we are idle and the tick is stopped, we have to touch
568 * the watchdog as we might not schedule for a really long
569 * time. This happens on complete idle SMP systems while
570 * waiting on the login prompt. We also increment the "start of
571 * idle" jiffy stamp so the idle accounting adjustment we do
572 * when we go busy again does not account too much ticks.
574 if (ts->tick_stopped) {
575 touch_softlockup_watchdog();
578 update_process_times(user_mode(regs));
579 profile_tick(CPU_PROFILING);
582 /* Do not restart, when we are in the idle loop */
583 if (ts->tick_stopped)
584 return HRTIMER_NORESTART;
586 hrtimer_forward(timer, now, tick_period);
588 return HRTIMER_RESTART;
592 * tick_setup_sched_timer - setup the tick emulation timer
594 void tick_setup_sched_timer(void)
596 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
597 ktime_t now = ktime_get();
601 * Emulate tick processing via per-CPU hrtimers:
603 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
604 ts->sched_timer.function = tick_sched_timer;
605 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
607 /* Get the next period (per cpu) */
608 ts->sched_timer.expires = tick_init_jiffy_update();
609 offset = ktime_to_ns(tick_period) >> 1;
610 do_div(offset, num_possible_cpus());
611 offset *= smp_processor_id();
612 ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset);
615 hrtimer_forward(&ts->sched_timer, now, tick_period);
616 hrtimer_start(&ts->sched_timer, ts->sched_timer.expires,
618 /* Check, if the timer was already in the past */
619 if (hrtimer_active(&ts->sched_timer))
625 if (tick_nohz_enabled)
626 ts->nohz_mode = NOHZ_MODE_HIGHRES;
630 void tick_cancel_sched_timer(int cpu)
632 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
634 if (ts->sched_timer.base)
635 hrtimer_cancel(&ts->sched_timer);
637 ts->nohz_mode = NOHZ_MODE_INACTIVE;
639 #endif /* HIGH_RES_TIMERS */
642 * Async notification about clocksource changes
644 void tick_clock_notify(void)
648 for_each_possible_cpu(cpu)
649 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
653 * Async notification about clock event changes
655 void tick_oneshot_notify(void)
657 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
659 set_bit(0, &ts->check_clocks);
663 * Check, if a change happened, which makes oneshot possible.
665 * Called cyclic from the hrtimer softirq (driven by the timer
666 * softirq) allow_nohz signals, that we can switch into low-res nohz
667 * mode, because high resolution timers are disabled (either compile
670 int tick_check_oneshot_change(int allow_nohz)
672 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
674 if (!test_and_clear_bit(0, &ts->check_clocks))
677 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
680 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
686 tick_nohz_switch_to_nohz();