Merge branch 'merge' of git://git.kernel.org/pub/scm/linux/kernel/git/paulus/powerpc
[linux-2.6] / arch / i386 / kernel / tsc.c
1 /*
2  * This code largely moved from arch/i386/kernel/timer/timer_tsc.c
3  * which was originally moved from arch/i386/kernel/time.c.
4  * See comments there for proper credits.
5  */
6
7 #include <linux/sched.h>
8 #include <linux/clocksource.h>
9 #include <linux/workqueue.h>
10 #include <linux/cpufreq.h>
11 #include <linux/jiffies.h>
12 #include <linux/init.h>
13 #include <linux/dmi.h>
14
15 #include <asm/delay.h>
16 #include <asm/tsc.h>
17 #include <asm/io.h>
18 #include <asm/timer.h>
19
20 #include "mach_timer.h"
21
22 static int tsc_enabled;
23
24 /*
25  * On some systems the TSC frequency does not
26  * change with the cpu frequency. So we need
27  * an extra value to store the TSC freq
28  */
29 unsigned int tsc_khz;
30 EXPORT_SYMBOL_GPL(tsc_khz);
31
32 int tsc_disable;
33
34 #ifdef CONFIG_X86_TSC
35 static int __init tsc_setup(char *str)
36 {
37         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
38                                 "cannot disable TSC.\n");
39         return 1;
40 }
41 #else
42 /*
43  * disable flag for tsc. Takes effect by clearing the TSC cpu flag
44  * in cpu/common.c
45  */
46 static int __init tsc_setup(char *str)
47 {
48         tsc_disable = 1;
49
50         return 1;
51 }
52 #endif
53
54 __setup("notsc", tsc_setup);
55
56 /*
57  * code to mark and check if the TSC is unstable
58  * due to cpufreq or due to unsynced TSCs
59  */
60 static int tsc_unstable;
61
62 int check_tsc_unstable(void)
63 {
64         return tsc_unstable;
65 }
66 EXPORT_SYMBOL_GPL(check_tsc_unstable);
67
68 /* Accellerators for sched_clock()
69  * convert from cycles(64bits) => nanoseconds (64bits)
70  *  basic equation:
71  *              ns = cycles / (freq / ns_per_sec)
72  *              ns = cycles * (ns_per_sec / freq)
73  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
74  *              ns = cycles * (10^6 / cpu_khz)
75  *
76  *      Then we use scaling math (suggested by george@mvista.com) to get:
77  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
78  *              ns = cycles * cyc2ns_scale / SC
79  *
80  *      And since SC is a constant power of two, we can convert the div
81  *  into a shift.
82  *
83  *  We can use khz divisor instead of mhz to keep a better percision, since
84  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
85  *  (mathieu.desnoyers@polymtl.ca)
86  *
87  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
88  */
89 unsigned long cyc2ns_scale __read_mostly;
90
91 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
92
93 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
94 {
95         cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
96 }
97
98 /*
99  * Scheduler clock - returns current time in nanosec units.
100  */
101 unsigned long long native_sched_clock(void)
102 {
103         unsigned long long this_offset;
104
105         /*
106          * Fall back to jiffies if there's no TSC available:
107          * ( But note that we still use it if the TSC is marked
108          *   unstable. We do this because unlike Time Of Day,
109          *   the scheduler clock tolerates small errors and it's
110          *   very important for it to be as fast as the platform
111          *   can achive it. )
112          */
113         if (unlikely(!tsc_enabled && !tsc_unstable))
114                 /* No locking but a rare wrong value is not a big deal: */
115                 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
116
117         /* read the Time Stamp Counter: */
118         rdtscll(this_offset);
119
120         /* return the value in ns */
121         return cycles_2_ns(this_offset);
122 }
123
124 /* We need to define a real function for sched_clock, to override the
125    weak default version */
126 #ifdef CONFIG_PARAVIRT
127 unsigned long long sched_clock(void)
128 {
129         return paravirt_sched_clock();
130 }
131 #else
132 unsigned long long sched_clock(void)
133         __attribute__((alias("native_sched_clock")));
134 #endif
135
136 unsigned long native_calculate_cpu_khz(void)
137 {
138         unsigned long long start, end;
139         unsigned long count;
140         u64 delta64;
141         int i;
142         unsigned long flags;
143
144         local_irq_save(flags);
145
146         /* run 3 times to ensure the cache is warm */
147         for (i = 0; i < 3; i++) {
148                 mach_prepare_counter();
149                 rdtscll(start);
150                 mach_countup(&count);
151                 rdtscll(end);
152         }
153         /*
154          * Error: ECTCNEVERSET
155          * The CTC wasn't reliable: we got a hit on the very first read,
156          * or the CPU was so fast/slow that the quotient wouldn't fit in
157          * 32 bits..
158          */
159         if (count <= 1)
160                 goto err;
161
162         delta64 = end - start;
163
164         /* cpu freq too fast: */
165         if (delta64 > (1ULL<<32))
166                 goto err;
167
168         /* cpu freq too slow: */
169         if (delta64 <= CALIBRATE_TIME_MSEC)
170                 goto err;
171
172         delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
173         do_div(delta64,CALIBRATE_TIME_MSEC);
174
175         local_irq_restore(flags);
176         return (unsigned long)delta64;
177 err:
178         local_irq_restore(flags);
179         return 0;
180 }
181
182 int recalibrate_cpu_khz(void)
183 {
184 #ifndef CONFIG_SMP
185         unsigned long cpu_khz_old = cpu_khz;
186
187         if (cpu_has_tsc) {
188                 cpu_khz = calculate_cpu_khz();
189                 tsc_khz = cpu_khz;
190                 cpu_data[0].loops_per_jiffy =
191                         cpufreq_scale(cpu_data[0].loops_per_jiffy,
192                                         cpu_khz_old, cpu_khz);
193                 return 0;
194         } else
195                 return -ENODEV;
196 #else
197         return -ENODEV;
198 #endif
199 }
200
201 EXPORT_SYMBOL(recalibrate_cpu_khz);
202
203 #ifdef CONFIG_CPU_FREQ
204
205 /*
206  * if the CPU frequency is scaled, TSC-based delays will need a different
207  * loops_per_jiffy value to function properly.
208  */
209 static unsigned int ref_freq = 0;
210 static unsigned long loops_per_jiffy_ref = 0;
211 static unsigned long cpu_khz_ref = 0;
212
213 static int
214 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
215 {
216         struct cpufreq_freqs *freq = data;
217
218         if (!ref_freq) {
219                 if (!freq->old){
220                         ref_freq = freq->new;
221                         return 0;
222                 }
223                 ref_freq = freq->old;
224                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
225                 cpu_khz_ref = cpu_khz;
226         }
227
228         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
229             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
230             (val == CPUFREQ_RESUMECHANGE)) {
231                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
232                         cpu_data[freq->cpu].loops_per_jiffy =
233                                 cpufreq_scale(loops_per_jiffy_ref,
234                                                 ref_freq, freq->new);
235
236                 if (cpu_khz) {
237
238                         if (num_online_cpus() == 1)
239                                 cpu_khz = cpufreq_scale(cpu_khz_ref,
240                                                 ref_freq, freq->new);
241                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
242                                 tsc_khz = cpu_khz;
243                                 set_cyc2ns_scale(cpu_khz);
244                                 /*
245                                  * TSC based sched_clock turns
246                                  * to junk w/ cpufreq
247                                  */
248                                 mark_tsc_unstable("cpufreq changes");
249                         }
250                 }
251         }
252
253         return 0;
254 }
255
256 static struct notifier_block time_cpufreq_notifier_block = {
257         .notifier_call  = time_cpufreq_notifier
258 };
259
260 static int __init cpufreq_tsc(void)
261 {
262         return cpufreq_register_notifier(&time_cpufreq_notifier_block,
263                                          CPUFREQ_TRANSITION_NOTIFIER);
264 }
265 core_initcall(cpufreq_tsc);
266
267 #endif
268
269 /* clock source code */
270
271 static unsigned long current_tsc_khz = 0;
272
273 static cycle_t read_tsc(void)
274 {
275         cycle_t ret;
276
277         rdtscll(ret);
278
279         return ret;
280 }
281
282 static struct clocksource clocksource_tsc = {
283         .name                   = "tsc",
284         .rating                 = 300,
285         .read                   = read_tsc,
286         .mask                   = CLOCKSOURCE_MASK(64),
287         .mult                   = 0, /* to be set */
288         .shift                  = 22,
289         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
290                                   CLOCK_SOURCE_MUST_VERIFY,
291 };
292
293 void mark_tsc_unstable(char *reason)
294 {
295         sched_clock_unstable_event();
296         if (!tsc_unstable) {
297                 tsc_unstable = 1;
298                 tsc_enabled = 0;
299                 printk("Marking TSC unstable due to: %s.\n", reason);
300                 /* Can be called before registration */
301                 if (clocksource_tsc.mult)
302                         clocksource_change_rating(&clocksource_tsc, 0);
303                 else
304                         clocksource_tsc.rating = 0;
305         }
306 }
307 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
308
309 static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d)
310 {
311         printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
312                        d->ident);
313         tsc_unstable = 1;
314         return 0;
315 }
316
317 /* List of systems that have known TSC problems */
318 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
319         {
320          .callback = dmi_mark_tsc_unstable,
321          .ident = "IBM Thinkpad 380XD",
322          .matches = {
323                      DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
324                      DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
325                      },
326          },
327          {}
328 };
329
330 /*
331  * Make an educated guess if the TSC is trustworthy and synchronized
332  * over all CPUs.
333  */
334 __cpuinit int unsynchronized_tsc(void)
335 {
336         if (!cpu_has_tsc || tsc_unstable)
337                 return 1;
338         /*
339          * Intel systems are normally all synchronized.
340          * Exceptions must mark TSC as unstable:
341          */
342         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
343                 /* assume multi socket systems are not synchronized: */
344                 if (num_possible_cpus() > 1)
345                         tsc_unstable = 1;
346         }
347         return tsc_unstable;
348 }
349
350 /*
351  * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
352  */
353 #ifdef CONFIG_MGEODE_LX
354 /* RTSC counts during suspend */
355 #define RTSC_SUSP 0x100
356
357 static void __init check_geode_tsc_reliable(void)
358 {
359         unsigned long val;
360
361         rdmsrl(MSR_GEODE_BUSCONT_CONF0, val);
362         if ((val & RTSC_SUSP))
363                 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
364 }
365 #else
366 static inline void check_geode_tsc_reliable(void) { }
367 #endif
368
369
370 void __init tsc_init(void)
371 {
372         if (!cpu_has_tsc || tsc_disable)
373                 goto out_no_tsc;
374
375         cpu_khz = calculate_cpu_khz();
376         tsc_khz = cpu_khz;
377
378         if (!cpu_khz)
379                 goto out_no_tsc;
380
381         printk("Detected %lu.%03lu MHz processor.\n",
382                                 (unsigned long)cpu_khz / 1000,
383                                 (unsigned long)cpu_khz % 1000);
384
385         set_cyc2ns_scale(cpu_khz);
386         use_tsc_delay();
387
388         /* Check and install the TSC clocksource */
389         dmi_check_system(bad_tsc_dmi_table);
390
391         unsynchronized_tsc();
392         check_geode_tsc_reliable();
393         current_tsc_khz = tsc_khz;
394         clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
395                                                         clocksource_tsc.shift);
396         /* lower the rating if we already know its unstable: */
397         if (check_tsc_unstable()) {
398                 clocksource_tsc.rating = 0;
399                 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
400         } else
401                 tsc_enabled = 1;
402
403         clocksource_register(&clocksource_tsc);
404
405         return;
406
407 out_no_tsc:
408         /*
409          * Set the tsc_disable flag if there's no TSC support, this
410          * makes it a fast flag for the kernel to see whether it
411          * should be using the TSC.
412          */
413         tsc_disable = 1;
414 }