Merge master.kernel.org:/pub/scm/linux/kernel/git/davej/x86
[linux-2.6] / arch / i386 / kernel / timers / timer_tsc.c
1 /*
2  * This code largely moved from arch/i386/kernel/time.c.
3  * See comments there for proper credits.
4  *
5  * 2004-06-25    Jesper Juhl
6  *      moved mark_offset_tsc below cpufreq_delayed_get to avoid gcc 3.4
7  *      failing to inline.
8  */
9
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/timex.h>
13 #include <linux/errno.h>
14 #include <linux/cpufreq.h>
15 #include <linux/string.h>
16 #include <linux/jiffies.h>
17
18 #include <asm/timer.h>
19 #include <asm/io.h>
20 /* processor.h for distable_tsc flag */
21 #include <asm/processor.h>
22
23 #include "io_ports.h"
24 #include "mach_timer.h"
25
26 #include <asm/hpet.h>
27 #include <asm/i8253.h>
28
29 #ifdef CONFIG_HPET_TIMER
30 static unsigned long hpet_usec_quotient;
31 static unsigned long hpet_last;
32 static struct timer_opts timer_tsc;
33 #endif
34
35 static inline void cpufreq_delayed_get(void);
36
37 int tsc_disable __devinitdata = 0;
38
39 static int use_tsc;
40 /* Number of usecs that the last interrupt was delayed */
41 static int delay_at_last_interrupt;
42
43 static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
44 static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
45 static unsigned long long monotonic_base;
46 static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
47
48 /* convert from cycles(64bits) => nanoseconds (64bits)
49  *  basic equation:
50  *              ns = cycles / (freq / ns_per_sec)
51  *              ns = cycles * (ns_per_sec / freq)
52  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
53  *              ns = cycles * (10^6 / cpu_khz)
54  *
55  *      Then we use scaling math (suggested by george@mvista.com) to get:
56  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
57  *              ns = cycles * cyc2ns_scale / SC
58  *
59  *      And since SC is a constant power of two, we can convert the div
60  *  into a shift.
61  *
62  *  We can use khz divisor instead of mhz to keep a better percision, since
63  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
64  *  (mathieu.desnoyers@polymtl.ca)
65  *
66  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
67  */
68 static unsigned long cyc2ns_scale; 
69 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
70
71 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
72 {
73         cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
74 }
75
76 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
77 {
78         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
79 }
80
81 static int count2; /* counter for mark_offset_tsc() */
82
83 /* Cached *multiplier* to convert TSC counts to microseconds.
84  * (see the equation below).
85  * Equal to 2^32 * (1 / (clocks per usec) ).
86  * Initialized in time_init.
87  */
88 static unsigned long fast_gettimeoffset_quotient;
89
90 static unsigned long get_offset_tsc(void)
91 {
92         register unsigned long eax, edx;
93
94         /* Read the Time Stamp Counter */
95
96         rdtsc(eax,edx);
97
98         /* .. relative to previous jiffy (32 bits is enough) */
99         eax -= last_tsc_low;    /* tsc_low delta */
100
101         /*
102          * Time offset = (tsc_low delta) * fast_gettimeoffset_quotient
103          *             = (tsc_low delta) * (usecs_per_clock)
104          *             = (tsc_low delta) * (usecs_per_jiffy / clocks_per_jiffy)
105          *
106          * Using a mull instead of a divl saves up to 31 clock cycles
107          * in the critical path.
108          */
109
110         __asm__("mull %2"
111                 :"=a" (eax), "=d" (edx)
112                 :"rm" (fast_gettimeoffset_quotient),
113                  "0" (eax));
114
115         /* our adjusted time offset in microseconds */
116         return delay_at_last_interrupt + edx;
117 }
118
119 static unsigned long long monotonic_clock_tsc(void)
120 {
121         unsigned long long last_offset, this_offset, base;
122         unsigned seq;
123         
124         /* atomically read monotonic base & last_offset */
125         do {
126                 seq = read_seqbegin(&monotonic_lock);
127                 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
128                 base = monotonic_base;
129         } while (read_seqretry(&monotonic_lock, seq));
130
131         /* Read the Time Stamp Counter */
132         rdtscll(this_offset);
133
134         /* return the value in ns */
135         return base + cycles_2_ns(this_offset - last_offset);
136 }
137
138 /*
139  * Scheduler clock - returns current time in nanosec units.
140  */
141 unsigned long long sched_clock(void)
142 {
143         unsigned long long this_offset;
144
145         /*
146          * In the NUMA case we dont use the TSC as they are not
147          * synchronized across all CPUs.
148          */
149 #ifndef CONFIG_NUMA
150         if (!use_tsc)
151 #endif
152                 /* no locking but a rare wrong value is not a big deal */
153                 return jiffies_64 * (1000000000 / HZ);
154
155         /* Read the Time Stamp Counter */
156         rdtscll(this_offset);
157
158         /* return the value in ns */
159         return cycles_2_ns(this_offset);
160 }
161
162 static void delay_tsc(unsigned long loops)
163 {
164         unsigned long bclock, now;
165         
166         rdtscl(bclock);
167         do
168         {
169                 rep_nop();
170                 rdtscl(now);
171         } while ((now-bclock) < loops);
172 }
173
174 #ifdef CONFIG_HPET_TIMER
175 static void mark_offset_tsc_hpet(void)
176 {
177         unsigned long long this_offset, last_offset;
178         unsigned long offset, temp, hpet_current;
179
180         write_seqlock(&monotonic_lock);
181         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
182         /*
183          * It is important that these two operations happen almost at
184          * the same time. We do the RDTSC stuff first, since it's
185          * faster. To avoid any inconsistencies, we need interrupts
186          * disabled locally.
187          */
188         /*
189          * Interrupts are just disabled locally since the timer irq
190          * has the SA_INTERRUPT flag set. -arca
191          */
192         /* read Pentium cycle counter */
193
194         hpet_current = hpet_readl(HPET_COUNTER);
195         rdtsc(last_tsc_low, last_tsc_high);
196
197         /* lost tick compensation */
198         offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
199         if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))) {
200                 int lost_ticks = (offset - hpet_last) / hpet_tick;
201                 jiffies_64 += lost_ticks;
202         }
203         hpet_last = hpet_current;
204
205         /* update the monotonic base value */
206         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
207         monotonic_base += cycles_2_ns(this_offset - last_offset);
208         write_sequnlock(&monotonic_lock);
209
210         /* calculate delay_at_last_interrupt */
211         /*
212          * Time offset = (hpet delta) * ( usecs per HPET clock )
213          *             = (hpet delta) * ( usecs per tick / HPET clocks per tick)
214          *             = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
215          * Where,
216          * hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
217          */
218         delay_at_last_interrupt = hpet_current - offset;
219         ASM_MUL64_REG(temp, delay_at_last_interrupt,
220                         hpet_usec_quotient, delay_at_last_interrupt);
221 }
222 #endif
223
224
225 #ifdef CONFIG_CPU_FREQ
226 #include <linux/workqueue.h>
227
228 static unsigned int cpufreq_delayed_issched = 0;
229 static unsigned int cpufreq_init = 0;
230 static struct work_struct cpufreq_delayed_get_work;
231
232 static void handle_cpufreq_delayed_get(void *v)
233 {
234         unsigned int cpu;
235         for_each_online_cpu(cpu) {
236                 cpufreq_get(cpu);
237         }
238         cpufreq_delayed_issched = 0;
239 }
240
241 /* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
242  * to verify the CPU frequency the timing core thinks the CPU is running
243  * at is still correct.
244  */
245 static inline void cpufreq_delayed_get(void) 
246 {
247         if (cpufreq_init && !cpufreq_delayed_issched) {
248                 cpufreq_delayed_issched = 1;
249                 printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");
250                 schedule_work(&cpufreq_delayed_get_work);
251         }
252 }
253
254 /* If the CPU frequency is scaled, TSC-based delays will need a different
255  * loops_per_jiffy value to function properly.
256  */
257
258 static unsigned int  ref_freq = 0;
259 static unsigned long loops_per_jiffy_ref = 0;
260
261 #ifndef CONFIG_SMP
262 static unsigned long fast_gettimeoffset_ref = 0;
263 static unsigned int cpu_khz_ref = 0;
264 #endif
265
266 static int
267 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
268                        void *data)
269 {
270         struct cpufreq_freqs *freq = data;
271
272         if (val != CPUFREQ_RESUMECHANGE)
273                 write_seqlock_irq(&xtime_lock);
274         if (!ref_freq) {
275                 ref_freq = freq->old;
276                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
277 #ifndef CONFIG_SMP
278                 fast_gettimeoffset_ref = fast_gettimeoffset_quotient;
279                 cpu_khz_ref = cpu_khz;
280 #endif
281         }
282
283         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
284             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
285             (val == CPUFREQ_RESUMECHANGE)) {
286                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
287                         cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
288 #ifndef CONFIG_SMP
289                 if (cpu_khz)
290                         cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
291                 if (use_tsc) {
292                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
293                                 fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);
294                                 set_cyc2ns_scale(cpu_khz);
295                         }
296                 }
297 #endif
298         }
299
300         if (val != CPUFREQ_RESUMECHANGE)
301                 write_sequnlock_irq(&xtime_lock);
302
303         return 0;
304 }
305
306 static struct notifier_block time_cpufreq_notifier_block = {
307         .notifier_call  = time_cpufreq_notifier
308 };
309
310
311 static int __init cpufreq_tsc(void)
312 {
313         int ret;
314         INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
315         ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
316                                         CPUFREQ_TRANSITION_NOTIFIER);
317         if (!ret)
318                 cpufreq_init = 1;
319         return ret;
320 }
321 core_initcall(cpufreq_tsc);
322
323 #else /* CONFIG_CPU_FREQ */
324 static inline void cpufreq_delayed_get(void) { return; }
325 #endif 
326
327 int recalibrate_cpu_khz(void)
328 {
329 #ifndef CONFIG_SMP
330         unsigned int cpu_khz_old = cpu_khz;
331
332         if (cpu_has_tsc) {
333                 local_irq_disable();
334                 init_cpu_khz();
335                 local_irq_enable();
336                 cpu_data[0].loops_per_jiffy =
337                     cpufreq_scale(cpu_data[0].loops_per_jiffy,
338                                   cpu_khz_old,
339                                   cpu_khz);
340                 return 0;
341         } else
342                 return -ENODEV;
343 #else
344         return -ENODEV;
345 #endif
346 }
347 EXPORT_SYMBOL(recalibrate_cpu_khz);
348
349 static void mark_offset_tsc(void)
350 {
351         unsigned long lost,delay;
352         unsigned long delta = last_tsc_low;
353         int count;
354         int countmp;
355         static int count1 = 0;
356         unsigned long long this_offset, last_offset;
357         static int lost_count = 0;
358
359         write_seqlock(&monotonic_lock);
360         last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
361         /*
362          * It is important that these two operations happen almost at
363          * the same time. We do the RDTSC stuff first, since it's
364          * faster. To avoid any inconsistencies, we need interrupts
365          * disabled locally.
366          */
367
368         /*
369          * Interrupts are just disabled locally since the timer irq
370          * has the SA_INTERRUPT flag set. -arca
371          */
372
373         /* read Pentium cycle counter */
374
375         rdtsc(last_tsc_low, last_tsc_high);
376
377         spin_lock(&i8253_lock);
378         outb_p(0x00, PIT_MODE);     /* latch the count ASAP */
379
380         count = inb_p(PIT_CH0);    /* read the latched count */
381         count |= inb(PIT_CH0) << 8;
382
383         /*
384          * VIA686a test code... reset the latch if count > max + 1
385          * from timer_pit.c - cjb
386          */
387         if (count > LATCH) {
388                 outb_p(0x34, PIT_MODE);
389                 outb_p(LATCH & 0xff, PIT_CH0);
390                 outb(LATCH >> 8, PIT_CH0);
391                 count = LATCH - 1;
392         }
393
394         spin_unlock(&i8253_lock);
395
396         if (pit_latch_buggy) {
397                 /* get center value of last 3 time lutch */
398                 if ((count2 >= count && count >= count1)
399                     || (count1 >= count && count >= count2)) {
400                         count2 = count1; count1 = count;
401                 } else if ((count1 >= count2 && count2 >= count)
402                            || (count >= count2 && count2 >= count1)) {
403                         countmp = count;count = count2;
404                         count2 = count1;count1 = countmp;
405                 } else {
406                         count2 = count1; count1 = count; count = count1;
407                 }
408         }
409
410         /* lost tick compensation */
411         delta = last_tsc_low - delta;
412         {
413                 register unsigned long eax, edx;
414                 eax = delta;
415                 __asm__("mull %2"
416                 :"=a" (eax), "=d" (edx)
417                 :"rm" (fast_gettimeoffset_quotient),
418                  "0" (eax));
419                 delta = edx;
420         }
421         delta += delay_at_last_interrupt;
422         lost = delta/(1000000/HZ);
423         delay = delta%(1000000/HZ);
424         if (lost >= 2) {
425                 jiffies_64 += lost-1;
426
427                 /* sanity check to ensure we're not always losing ticks */
428                 if (lost_count++ > 100) {
429                         printk(KERN_WARNING "Losing too many ticks!\n");
430                         printk(KERN_WARNING "TSC cannot be used as a timesource.  \n");
431                         printk(KERN_WARNING "Possible reasons for this are:\n");
432                         printk(KERN_WARNING "  You're running with Speedstep,\n");
433                         printk(KERN_WARNING "  You don't have DMA enabled for your hard disk (see hdparm),\n");
434                         printk(KERN_WARNING "  Incorrect TSC synchronization on an SMP system (see dmesg).\n");
435                         printk(KERN_WARNING "Falling back to a sane timesource now.\n");
436
437                         clock_fallback();
438                 }
439                 /* ... but give the TSC a fair chance */
440                 if (lost_count > 25)
441                         cpufreq_delayed_get();
442         } else
443                 lost_count = 0;
444         /* update the monotonic base value */
445         this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
446         monotonic_base += cycles_2_ns(this_offset - last_offset);
447         write_sequnlock(&monotonic_lock);
448
449         /* calculate delay_at_last_interrupt */
450         count = ((LATCH-1) - count) * TICK_SIZE;
451         delay_at_last_interrupt = (count + LATCH/2) / LATCH;
452
453         /* catch corner case where tick rollover occured
454          * between tsc and pit reads (as noted when
455          * usec delta is > 90% # of usecs/tick)
456          */
457         if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
458                 jiffies_64++;
459 }
460
461 static int __init init_tsc(char* override)
462 {
463
464         /* check clock override */
465         if (override[0] && strncmp(override,"tsc",3)) {
466 #ifdef CONFIG_HPET_TIMER
467                 if (is_hpet_enabled()) {
468                         printk(KERN_ERR "Warning: clock= override failed. Defaulting to tsc\n");
469                 } else
470 #endif
471                 {
472                         return -ENODEV;
473                 }
474         }
475
476         /*
477          * If we have APM enabled or the CPU clock speed is variable
478          * (CPU stops clock on HLT or slows clock to save power)
479          * then the TSC timestamps may diverge by up to 1 jiffy from
480          * 'real time' but nothing will break.
481          * The most frequent case is that the CPU is "woken" from a halt
482          * state by the timer interrupt itself, so we get 0 error. In the
483          * rare cases where a driver would "wake" the CPU and request a
484          * timestamp, the maximum error is < 1 jiffy. But timestamps are
485          * still perfectly ordered.
486          * Note that the TSC counter will be reset if APM suspends
487          * to disk; this won't break the kernel, though, 'cuz we're
488          * smart.  See arch/i386/kernel/apm.c.
489          */
490         /*
491          *      Firstly we have to do a CPU check for chips with
492          *      a potentially buggy TSC. At this point we haven't run
493          *      the ident/bugs checks so we must run this hook as it
494          *      may turn off the TSC flag.
495          *
496          *      NOTE: this doesn't yet handle SMP 486 machines where only
497          *      some CPU's have a TSC. Thats never worked and nobody has
498          *      moaned if you have the only one in the world - you fix it!
499          */
500
501         count2 = LATCH; /* initialize counter for mark_offset_tsc() */
502
503         if (cpu_has_tsc) {
504                 unsigned long tsc_quotient;
505 #ifdef CONFIG_HPET_TIMER
506                 if (is_hpet_enabled() && hpet_use_timer) {
507                         unsigned long result, remain;
508                         printk("Using TSC for gettimeofday\n");
509                         tsc_quotient = calibrate_tsc_hpet(NULL);
510                         timer_tsc.mark_offset = &mark_offset_tsc_hpet;
511                         /*
512                          * Math to calculate hpet to usec multiplier
513                          * Look for the comments at get_offset_tsc_hpet()
514                          */
515                         ASM_DIV64_REG(result, remain, hpet_tick,
516                                         0, KERNEL_TICK_USEC);
517                         if (remain > (hpet_tick >> 1))
518                                 result++; /* rounding the result */
519
520                         hpet_usec_quotient = result;
521                 } else
522 #endif
523                 {
524                         tsc_quotient = calibrate_tsc();
525                 }
526
527                 if (tsc_quotient) {
528                         fast_gettimeoffset_quotient = tsc_quotient;
529                         use_tsc = 1;
530                         /*
531                          *      We could be more selective here I suspect
532                          *      and just enable this for the next intel chips ?
533                          */
534                         /* report CPU clock rate in Hz.
535                          * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
536                          * clock/second. Our precision is about 100 ppm.
537                          */
538                         {       unsigned long eax=0, edx=1000;
539                                 __asm__("divl %2"
540                                 :"=a" (cpu_khz), "=d" (edx)
541                                 :"r" (tsc_quotient),
542                                 "0" (eax), "1" (edx));
543                                 printk("Detected %u.%03u MHz processor.\n",
544                                         cpu_khz / 1000, cpu_khz % 1000);
545                         }
546                         set_cyc2ns_scale(cpu_khz);
547                         return 0;
548                 }
549         }
550         return -ENODEV;
551 }
552
553 static int tsc_resume(void)
554 {
555         write_seqlock(&monotonic_lock);
556         /* Assume this is the last mark offset time */
557         rdtsc(last_tsc_low, last_tsc_high);
558 #ifdef CONFIG_HPET_TIMER
559         if (is_hpet_enabled() && hpet_use_timer)
560                 hpet_last = hpet_readl(HPET_COUNTER);
561 #endif
562         write_sequnlock(&monotonic_lock);
563         return 0;
564 }
565
566 #ifndef CONFIG_X86_TSC
567 /* disable flag for tsc.  Takes effect by clearing the TSC cpu flag
568  * in cpu/common.c */
569 static int __init tsc_setup(char *str)
570 {
571         tsc_disable = 1;
572         return 1;
573 }
574 #else
575 static int __init tsc_setup(char *str)
576 {
577         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
578                                 "cannot disable TSC.\n");
579         return 1;
580 }
581 #endif
582 __setup("notsc", tsc_setup);
583
584
585
586 /************************************************************/
587
588 /* tsc timer_opts struct */
589 static struct timer_opts timer_tsc = {
590         .name = "tsc",
591         .mark_offset = mark_offset_tsc, 
592         .get_offset = get_offset_tsc,
593         .monotonic_clock = monotonic_clock_tsc,
594         .delay = delay_tsc,
595         .read_timer = read_timer_tsc,
596         .resume = tsc_resume,
597 };
598
599 struct init_timer_opts __initdata timer_tsc_init = {
600         .init = init_tsc,
601         .opts = &timer_tsc,
602 };