2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
23 #include <asm/s390_ext.h>
24 #include <asm/timer.h>
25 #include <asm/irq_regs.h>
28 static ext_int_info_t ext_int_info_timer;
30 static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
32 DEFINE_PER_CPU(struct s390_idle_data, s390_idle) = {
33 .lock = __SPIN_LOCK_UNLOCKED(s390_idle.lock)
36 static inline __u64 get_vtimer(void)
40 asm volatile("STPT %0" : "=m" (timer));
44 static inline void set_vtimer(__u64 expires)
48 asm volatile (" STPT %0\n" /* Store current cpu timer value */
49 " SPT %1" /* Set new value immediatly afterwards */
50 : "=m" (timer) : "m" (expires) );
51 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
52 S390_lowcore.last_update_timer = expires;
56 * Update process times based on virtual cpu times stored by entry.S
57 * to the lowcore fields user_timer, system_timer & steal_clock.
59 static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
61 struct thread_info *ti = task_thread_info(tsk);
62 __u64 timer, clock, user, system, steal;
64 timer = S390_lowcore.last_update_timer;
65 clock = S390_lowcore.last_update_clock;
66 asm volatile (" STPT %0\n" /* Store current cpu timer value */
67 " STCK %1" /* Store current tod clock value */
68 : "=m" (S390_lowcore.last_update_timer),
69 "=m" (S390_lowcore.last_update_clock) );
70 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
71 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
73 user = S390_lowcore.user_timer - ti->user_timer;
74 S390_lowcore.steal_timer -= user;
75 ti->user_timer = S390_lowcore.user_timer;
76 account_user_time(tsk, user, user);
78 system = S390_lowcore.system_timer - ti->system_timer;
79 S390_lowcore.steal_timer -= system;
80 ti->system_timer = S390_lowcore.system_timer;
81 account_system_time(tsk, hardirq_offset, system, system);
83 steal = S390_lowcore.steal_timer;
84 if ((s64) steal > 0) {
85 S390_lowcore.steal_timer = 0;
86 account_steal_time(steal);
90 void account_vtime(struct task_struct *prev, struct task_struct *next)
92 struct thread_info *ti;
94 do_account_vtime(prev, 0);
95 ti = task_thread_info(prev);
96 ti->user_timer = S390_lowcore.user_timer;
97 ti->system_timer = S390_lowcore.system_timer;
98 ti = task_thread_info(next);
99 S390_lowcore.user_timer = ti->user_timer;
100 S390_lowcore.system_timer = ti->system_timer;
103 void account_process_tick(struct task_struct *tsk, int user_tick)
105 do_account_vtime(tsk, HARDIRQ_OFFSET);
109 * Update process times based on virtual cpu times stored by entry.S
110 * to the lowcore fields user_timer, system_timer & steal_clock.
112 void account_system_vtime(struct task_struct *tsk)
114 struct thread_info *ti = task_thread_info(tsk);
117 timer = S390_lowcore.last_update_timer;
118 S390_lowcore.last_update_timer = get_vtimer();
119 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
121 system = S390_lowcore.system_timer - ti->system_timer;
122 S390_lowcore.steal_timer -= system;
123 ti->system_timer = S390_lowcore.system_timer;
124 account_system_time(tsk, 0, system, system);
126 EXPORT_SYMBOL_GPL(account_system_vtime);
128 void vtime_start_cpu(void)
130 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
131 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
132 __u64 idle_time, expires;
134 /* Account time spent with enabled wait psw loaded as idle time. */
135 idle_time = S390_lowcore.int_clock - idle->idle_enter;
136 account_idle_time(idle_time);
137 S390_lowcore.steal_timer +=
138 idle->idle_enter - S390_lowcore.last_update_clock;
139 S390_lowcore.last_update_clock = S390_lowcore.int_clock;
141 /* Account system time spent going idle. */
142 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
143 S390_lowcore.last_update_timer = S390_lowcore.async_enter_timer;
145 /* Restart vtime CPU timer */
147 /* Program old expire value but first save progress. */
148 expires = vq->idle - S390_lowcore.async_enter_timer;
149 expires += get_vtimer();
152 /* Don't account the CPU timer delta while the cpu was idle. */
153 vq->elapsed -= vq->idle - S390_lowcore.async_enter_timer;
156 spin_lock(&idle->lock);
157 idle->idle_time += idle_time;
158 idle->idle_enter = 0ULL;
160 spin_unlock(&idle->lock);
163 void vtime_stop_cpu(void)
165 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
166 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
169 /* Wait for external, I/O or machine check interrupt. */
170 psw.mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT;
172 /* Check if the CPU timer needs to be reprogrammed. */
174 __u64 vmax = VTIMER_MAX_SLICE;
176 * The inline assembly is equivalent to
177 * vq->idle = get_cpu_timer();
178 * set_cpu_timer(VTIMER_MAX_SLICE);
179 * idle->idle_enter = get_clock();
180 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
181 * PSW_MASK_IO | PSW_MASK_EXT);
182 * The difference is that the inline assembly makes sure that
183 * the last three instruction are stpt, stck and lpsw in that
184 * order. This is done to increase the precision.
191 #else /* CONFIG_64BIT */
194 #endif /* CONFIG_64BIT */
200 #else /* CONFIG_64BIT */
202 #endif /* CONFIG_64BIT */
204 : "=m" (idle->idle_enter), "=m" (vq->idle)
205 : "a" (&psw), "a" (&idle->idle_enter),
206 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
207 : "memory", "cc", "1");
210 * The inline assembly is equivalent to
211 * vq->idle = get_cpu_timer();
212 * idle->idle_enter = get_clock();
213 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
214 * PSW_MASK_IO | PSW_MASK_EXT);
215 * The difference is that the inline assembly makes sure that
216 * the last three instruction are stpt, stck and lpsw in that
217 * order. This is done to increase the precision.
224 #else /* CONFIG_64BIT */
227 #endif /* CONFIG_64BIT */
232 #else /* CONFIG_64BIT */
234 #endif /* CONFIG_64BIT */
236 : "=m" (idle->idle_enter), "=m" (vq->idle)
237 : "a" (&psw), "a" (&idle->idle_enter),
238 "a" (&vq->idle), "m" (psw)
239 : "memory", "cc", "1");
244 * Sorted add to a list. List is linear searched until first bigger
247 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
249 struct vtimer_list *event;
251 list_for_each_entry(event, head, entry) {
252 if (event->expires > timer->expires) {
253 list_add_tail(&timer->entry, &event->entry);
257 list_add_tail(&timer->entry, head);
261 * Do the callback functions of expired vtimer events.
262 * Called from within the interrupt handler.
264 static void do_callbacks(struct list_head *cb_list)
266 struct vtimer_queue *vq;
267 struct vtimer_list *event, *tmp;
269 if (list_empty(cb_list))
272 vq = &__get_cpu_var(virt_cpu_timer);
274 list_for_each_entry_safe(event, tmp, cb_list, entry) {
275 list_del_init(&event->entry);
276 (event->function)(event->data);
277 if (event->interval) {
278 /* Recharge interval timer */
279 event->expires = event->interval + vq->elapsed;
280 spin_lock(&vq->lock);
281 list_add_sorted(event, &vq->list);
282 spin_unlock(&vq->lock);
288 * Handler for the virtual CPU timer.
290 static void do_cpu_timer_interrupt(__u16 error_code)
292 struct vtimer_queue *vq;
293 struct vtimer_list *event, *tmp;
294 struct list_head cb_list; /* the callback queue */
297 INIT_LIST_HEAD(&cb_list);
298 vq = &__get_cpu_var(virt_cpu_timer);
300 /* walk timer list, fire all expired events */
301 spin_lock(&vq->lock);
303 elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer);
304 BUG_ON((s64) elapsed < 0);
306 list_for_each_entry_safe(event, tmp, &vq->list, entry) {
307 if (event->expires < elapsed)
308 /* move expired timer to the callback queue */
309 list_move_tail(&event->entry, &cb_list);
311 event->expires -= elapsed;
313 spin_unlock(&vq->lock);
315 vq->do_spt = list_empty(&cb_list);
316 do_callbacks(&cb_list);
318 /* next event is first in list */
319 next = VTIMER_MAX_SLICE;
320 spin_lock(&vq->lock);
321 if (!list_empty(&vq->list)) {
322 event = list_first_entry(&vq->list, struct vtimer_list, entry);
323 next = event->expires;
326 spin_unlock(&vq->lock);
328 * To improve precision add the time spent by the
329 * interrupt handler to the elapsed time.
330 * Note: CPU timer counts down and we got an interrupt,
331 * the current content is negative
333 elapsed = S390_lowcore.async_enter_timer - get_vtimer();
334 set_vtimer(next - elapsed);
335 vq->timer = next - elapsed;
336 vq->elapsed = elapsed;
339 void init_virt_timer(struct vtimer_list *timer)
341 timer->function = NULL;
342 INIT_LIST_HEAD(&timer->entry);
344 EXPORT_SYMBOL(init_virt_timer);
346 static inline int vtimer_pending(struct vtimer_list *timer)
348 return (!list_empty(&timer->entry));
352 * this function should only run on the specified CPU
354 static void internal_add_vtimer(struct vtimer_list *timer)
356 struct vtimer_queue *vq;
360 vq = &per_cpu(virt_cpu_timer, timer->cpu);
361 spin_lock_irqsave(&vq->lock, flags);
363 BUG_ON(timer->cpu != smp_processor_id());
365 if (list_empty(&vq->list)) {
366 /* First timer on this cpu, just program it. */
367 list_add(&timer->entry, &vq->list);
368 set_vtimer(timer->expires);
369 vq->timer = timer->expires;
372 /* Check progress of old timers. */
373 expires = timer->expires;
375 if (likely((s64) expires < (s64) left)) {
376 /* The new timer expires before the current timer. */
378 vq->elapsed += vq->timer - left;
381 vq->elapsed += vq->timer - left;
384 /* Insert new timer into per cpu list. */
385 timer->expires += vq->elapsed;
386 list_add_sorted(timer, &vq->list);
389 spin_unlock_irqrestore(&vq->lock, flags);
390 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
394 static inline void prepare_vtimer(struct vtimer_list *timer)
396 BUG_ON(!timer->function);
397 BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE);
398 BUG_ON(vtimer_pending(timer));
399 timer->cpu = get_cpu();
403 * add_virt_timer - add an oneshot virtual CPU timer
405 void add_virt_timer(void *new)
407 struct vtimer_list *timer;
409 timer = (struct vtimer_list *)new;
410 prepare_vtimer(timer);
412 internal_add_vtimer(timer);
414 EXPORT_SYMBOL(add_virt_timer);
417 * add_virt_timer_int - add an interval virtual CPU timer
419 void add_virt_timer_periodic(void *new)
421 struct vtimer_list *timer;
423 timer = (struct vtimer_list *)new;
424 prepare_vtimer(timer);
425 timer->interval = timer->expires;
426 internal_add_vtimer(timer);
428 EXPORT_SYMBOL(add_virt_timer_periodic);
430 int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic)
432 struct vtimer_queue *vq;
436 BUG_ON(!timer->function);
437 BUG_ON(!expires || expires > VTIMER_MAX_SLICE);
439 if (timer->expires == expires && vtimer_pending(timer))
443 vq = &per_cpu(virt_cpu_timer, cpu);
445 /* disable interrupts before test if timer is pending */
446 spin_lock_irqsave(&vq->lock, flags);
448 /* if timer isn't pending add it on the current CPU */
449 if (!vtimer_pending(timer)) {
450 spin_unlock_irqrestore(&vq->lock, flags);
453 timer->interval = expires;
456 timer->expires = expires;
458 internal_add_vtimer(timer);
462 /* check if we run on the right CPU */
463 BUG_ON(timer->cpu != cpu);
465 list_del_init(&timer->entry);
466 timer->expires = expires;
468 timer->interval = expires;
470 /* the timer can't expire anymore so we can release the lock */
471 spin_unlock_irqrestore(&vq->lock, flags);
472 internal_add_vtimer(timer);
477 * If we change a pending timer the function must be called on the CPU
478 * where the timer is running on.
480 * returns whether it has modified a pending timer (1) or not (0)
482 int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
484 return __mod_vtimer(timer, expires, 0);
486 EXPORT_SYMBOL(mod_virt_timer);
489 * If we change a pending timer the function must be called on the CPU
490 * where the timer is running on.
492 * returns whether it has modified a pending timer (1) or not (0)
494 int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires)
496 return __mod_vtimer(timer, expires, 1);
498 EXPORT_SYMBOL(mod_virt_timer_periodic);
501 * delete a virtual timer
503 * returns whether the deleted timer was pending (1) or not (0)
505 int del_virt_timer(struct vtimer_list *timer)
508 struct vtimer_queue *vq;
510 /* check if timer is pending */
511 if (!vtimer_pending(timer))
514 vq = &per_cpu(virt_cpu_timer, timer->cpu);
515 spin_lock_irqsave(&vq->lock, flags);
517 /* we don't interrupt a running timer, just let it expire! */
518 list_del_init(&timer->entry);
520 spin_unlock_irqrestore(&vq->lock, flags);
523 EXPORT_SYMBOL(del_virt_timer);
526 * Start the virtual CPU timer on the current CPU.
528 void init_cpu_vtimer(void)
530 struct vtimer_queue *vq;
532 /* initialize per cpu vtimer structure */
533 vq = &__get_cpu_var(virt_cpu_timer);
534 INIT_LIST_HEAD(&vq->list);
535 spin_lock_init(&vq->lock);
537 /* enable cpu timer interrupts */
541 void __init vtime_init(void)
543 /* request the cpu timer external interrupt */
544 if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt,
545 &ext_int_info_timer) != 0)
546 panic("Couldn't request external interrupt 0x1005");
548 /* Enable cpu timer interrupts on the boot cpu. */