Pull bugzilla-7897 into release branch
[linux-2.6] / arch / i386 / kernel / tsc.c
1 /*
2  * This code largely moved from arch/i386/kernel/timer/timer_tsc.c
3  * which was originally moved from arch/i386/kernel/time.c.
4  * See comments there for proper credits.
5  */
6
7 #include <linux/clocksource.h>
8 #include <linux/workqueue.h>
9 #include <linux/cpufreq.h>
10 #include <linux/jiffies.h>
11 #include <linux/init.h>
12 #include <linux/dmi.h>
13
14 #include <asm/delay.h>
15 #include <asm/tsc.h>
16 #include <asm/io.h>
17
18 #include "mach_timer.h"
19
20 /*
21  * On some systems the TSC frequency does not
22  * change with the cpu frequency. So we need
23  * an extra value to store the TSC freq
24  */
25 unsigned int tsc_khz;
26 unsigned long long (*custom_sched_clock)(void);
27
28 int tsc_disable;
29
30 #ifdef CONFIG_X86_TSC
31 static int __init tsc_setup(char *str)
32 {
33         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
34                                 "cannot disable TSC.\n");
35         return 1;
36 }
37 #else
38 /*
39  * disable flag for tsc. Takes effect by clearing the TSC cpu flag
40  * in cpu/common.c
41  */
42 static int __init tsc_setup(char *str)
43 {
44         tsc_disable = 1;
45
46         return 1;
47 }
48 #endif
49
50 __setup("notsc", tsc_setup);
51
52 /*
53  * code to mark and check if the TSC is unstable
54  * due to cpufreq or due to unsynced TSCs
55  */
56 static int tsc_unstable;
57
58 static inline int check_tsc_unstable(void)
59 {
60         return tsc_unstable;
61 }
62
63 /* Accellerators for sched_clock()
64  * convert from cycles(64bits) => nanoseconds (64bits)
65  *  basic equation:
66  *              ns = cycles / (freq / ns_per_sec)
67  *              ns = cycles * (ns_per_sec / freq)
68  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
69  *              ns = cycles * (10^6 / cpu_khz)
70  *
71  *      Then we use scaling math (suggested by george@mvista.com) to get:
72  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
73  *              ns = cycles * cyc2ns_scale / SC
74  *
75  *      And since SC is a constant power of two, we can convert the div
76  *  into a shift.
77  *
78  *  We can use khz divisor instead of mhz to keep a better percision, since
79  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
80  *  (mathieu.desnoyers@polymtl.ca)
81  *
82  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
83  */
84 static unsigned long cyc2ns_scale __read_mostly;
85
86 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
87
88 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
89 {
90         cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
91 }
92
93 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
94 {
95         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
96 }
97
98 /*
99  * Scheduler clock - returns current time in nanosec units.
100  */
101 unsigned long long sched_clock(void)
102 {
103         unsigned long long this_offset;
104
105         if (unlikely(custom_sched_clock))
106                 return (*custom_sched_clock)();
107
108         /*
109          * Fall back to jiffies if there's no TSC available:
110          */
111         if (unlikely(tsc_disable))
112                 /* No locking but a rare wrong value is not a big deal: */
113                 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
114
115         /* read the Time Stamp Counter: */
116         rdtscll(this_offset);
117
118         /* return the value in ns */
119         return cycles_2_ns(this_offset);
120 }
121
122 static unsigned long calculate_cpu_khz(void)
123 {
124         unsigned long long start, end;
125         unsigned long count;
126         u64 delta64;
127         int i;
128         unsigned long flags;
129
130         local_irq_save(flags);
131
132         /* run 3 times to ensure the cache is warm */
133         for (i = 0; i < 3; i++) {
134                 mach_prepare_counter();
135                 rdtscll(start);
136                 mach_countup(&count);
137                 rdtscll(end);
138         }
139         /*
140          * Error: ECTCNEVERSET
141          * The CTC wasn't reliable: we got a hit on the very first read,
142          * or the CPU was so fast/slow that the quotient wouldn't fit in
143          * 32 bits..
144          */
145         if (count <= 1)
146                 goto err;
147
148         delta64 = end - start;
149
150         /* cpu freq too fast: */
151         if (delta64 > (1ULL<<32))
152                 goto err;
153
154         /* cpu freq too slow: */
155         if (delta64 <= CALIBRATE_TIME_MSEC)
156                 goto err;
157
158         delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
159         do_div(delta64,CALIBRATE_TIME_MSEC);
160
161         local_irq_restore(flags);
162         return (unsigned long)delta64;
163 err:
164         local_irq_restore(flags);
165         return 0;
166 }
167
168 int recalibrate_cpu_khz(void)
169 {
170 #ifndef CONFIG_SMP
171         unsigned long cpu_khz_old = cpu_khz;
172
173         if (cpu_has_tsc) {
174                 cpu_khz = calculate_cpu_khz();
175                 tsc_khz = cpu_khz;
176                 cpu_data[0].loops_per_jiffy =
177                         cpufreq_scale(cpu_data[0].loops_per_jiffy,
178                                         cpu_khz_old, cpu_khz);
179                 return 0;
180         } else
181                 return -ENODEV;
182 #else
183         return -ENODEV;
184 #endif
185 }
186
187 EXPORT_SYMBOL(recalibrate_cpu_khz);
188
189 void __init tsc_init(void)
190 {
191         if (!cpu_has_tsc || tsc_disable)
192                 goto out_no_tsc;
193
194         cpu_khz = calculate_cpu_khz();
195         tsc_khz = cpu_khz;
196
197         if (!cpu_khz)
198                 goto out_no_tsc;
199
200         printk("Detected %lu.%03lu MHz processor.\n",
201                                 (unsigned long)cpu_khz / 1000,
202                                 (unsigned long)cpu_khz % 1000);
203
204         set_cyc2ns_scale(cpu_khz);
205         use_tsc_delay();
206         return;
207
208 out_no_tsc:
209         /*
210          * Set the tsc_disable flag if there's no TSC support, this
211          * makes it a fast flag for the kernel to see whether it
212          * should be using the TSC.
213          */
214         tsc_disable = 1;
215 }
216
217 #ifdef CONFIG_CPU_FREQ
218
219 /*
220  * if the CPU frequency is scaled, TSC-based delays will need a different
221  * loops_per_jiffy value to function properly.
222  */
223 static unsigned int ref_freq = 0;
224 static unsigned long loops_per_jiffy_ref = 0;
225 static unsigned long cpu_khz_ref = 0;
226
227 static int
228 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
229 {
230         struct cpufreq_freqs *freq = data;
231
232         if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
233                 write_seqlock_irq(&xtime_lock);
234
235         if (!ref_freq) {
236                 if (!freq->old){
237                         ref_freq = freq->new;
238                         goto end;
239                 }
240                 ref_freq = freq->old;
241                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
242                 cpu_khz_ref = cpu_khz;
243         }
244
245         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
246             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
247             (val == CPUFREQ_RESUMECHANGE)) {
248                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
249                         cpu_data[freq->cpu].loops_per_jiffy =
250                                 cpufreq_scale(loops_per_jiffy_ref,
251                                                 ref_freq, freq->new);
252
253                 if (cpu_khz) {
254
255                         if (num_online_cpus() == 1)
256                                 cpu_khz = cpufreq_scale(cpu_khz_ref,
257                                                 ref_freq, freq->new);
258                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
259                                 tsc_khz = cpu_khz;
260                                 set_cyc2ns_scale(cpu_khz);
261                                 /*
262                                  * TSC based sched_clock turns
263                                  * to junk w/ cpufreq
264                                  */
265                                 mark_tsc_unstable();
266                         }
267                 }
268         }
269 end:
270         if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
271                 write_sequnlock_irq(&xtime_lock);
272
273         return 0;
274 }
275
276 static struct notifier_block time_cpufreq_notifier_block = {
277         .notifier_call  = time_cpufreq_notifier
278 };
279
280 static int __init cpufreq_tsc(void)
281 {
282         return cpufreq_register_notifier(&time_cpufreq_notifier_block,
283                                          CPUFREQ_TRANSITION_NOTIFIER);
284 }
285 core_initcall(cpufreq_tsc);
286
287 #endif
288
289 /* clock source code */
290
291 static unsigned long current_tsc_khz = 0;
292
293 static cycle_t read_tsc(void)
294 {
295         cycle_t ret;
296
297         rdtscll(ret);
298
299         return ret;
300 }
301
302 static struct clocksource clocksource_tsc = {
303         .name                   = "tsc",
304         .rating                 = 300,
305         .read                   = read_tsc,
306         .mask                   = CLOCKSOURCE_MASK(64),
307         .mult                   = 0, /* to be set */
308         .shift                  = 22,
309         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
310                                   CLOCK_SOURCE_MUST_VERIFY,
311 };
312
313 void mark_tsc_unstable(void)
314 {
315         if (!tsc_unstable) {
316                 tsc_unstable = 1;
317                 /* Can be called before registration */
318                 if (clocksource_tsc.mult)
319                         clocksource_change_rating(&clocksource_tsc, 0);
320                 else
321                         clocksource_tsc.rating = 0;
322         }
323 }
324 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
325
326 static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d)
327 {
328         printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
329                        d->ident);
330         tsc_unstable = 1;
331         return 0;
332 }
333
334 /* List of systems that have known TSC problems */
335 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
336         {
337          .callback = dmi_mark_tsc_unstable,
338          .ident = "IBM Thinkpad 380XD",
339          .matches = {
340                      DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
341                      DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
342                      },
343          },
344          {}
345 };
346
347 /*
348  * Make an educated guess if the TSC is trustworthy and synchronized
349  * over all CPUs.
350  */
351 __cpuinit int unsynchronized_tsc(void)
352 {
353         if (!cpu_has_tsc || tsc_unstable)
354                 return 1;
355         /*
356          * Intel systems are normally all synchronized.
357          * Exceptions must mark TSC as unstable:
358          */
359         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
360                 /* assume multi socket systems are not synchronized: */
361                 if (num_possible_cpus() > 1)
362                         tsc_unstable = 1;
363         }
364         return tsc_unstable;
365 }
366
367 /*
368  * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
369  */
370 #ifdef CONFIG_MGEODE_LX
371 /* RTSC counts during suspend */
372 #define RTSC_SUSP 0x100
373
374 static void __init check_geode_tsc_reliable(void)
375 {
376         unsigned long val;
377
378         rdmsrl(MSR_GEODE_BUSCONT_CONF0, val);
379         if ((val & RTSC_SUSP))
380                 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
381 }
382 #else
383 static inline void check_geode_tsc_reliable(void) { }
384 #endif
385
386 static int __init init_tsc_clocksource(void)
387 {
388
389         if (cpu_has_tsc && tsc_khz && !tsc_disable) {
390                 /* check blacklist */
391                 dmi_check_system(bad_tsc_dmi_table);
392
393                 unsynchronized_tsc();
394                 check_geode_tsc_reliable();
395                 current_tsc_khz = tsc_khz;
396                 clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
397                                                         clocksource_tsc.shift);
398                 /* lower the rating if we already know its unstable: */
399                 if (check_tsc_unstable()) {
400                         clocksource_tsc.rating = 0;
401                         clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
402                 }
403
404                 return clocksource_register(&clocksource_tsc);
405         }
406
407         return 0;
408 }
409
410 module_init(init_tsc_clocksource);