Merge master.kernel.org:/pub/scm/linux/kernel/git/davej/agpgart
[linux-2.6] / arch / i386 / kernel / tsc.c
1 /*
2  * This code largely moved from arch/i386/kernel/timer/timer_tsc.c
3  * which was originally moved from arch/i386/kernel/time.c.
4  * See comments there for proper credits.
5  */
6
7 #include <linux/sched.h>
8 #include <linux/clocksource.h>
9 #include <linux/workqueue.h>
10 #include <linux/cpufreq.h>
11 #include <linux/jiffies.h>
12 #include <linux/init.h>
13 #include <linux/dmi.h>
14
15 #include <asm/delay.h>
16 #include <asm/tsc.h>
17 #include <asm/io.h>
18 #include <asm/timer.h>
19
20 #include "mach_timer.h"
21
22 static int tsc_enabled;
23
24 /*
25  * On some systems the TSC frequency does not
26  * change with the cpu frequency. So we need
27  * an extra value to store the TSC freq
28  */
29 unsigned int tsc_khz;
30
31 int tsc_disable;
32
33 #ifdef CONFIG_X86_TSC
34 static int __init tsc_setup(char *str)
35 {
36         printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
37                                 "cannot disable TSC.\n");
38         return 1;
39 }
40 #else
41 /*
42  * disable flag for tsc. Takes effect by clearing the TSC cpu flag
43  * in cpu/common.c
44  */
45 static int __init tsc_setup(char *str)
46 {
47         tsc_disable = 1;
48
49         return 1;
50 }
51 #endif
52
53 __setup("notsc", tsc_setup);
54
55 /*
56  * code to mark and check if the TSC is unstable
57  * due to cpufreq or due to unsynced TSCs
58  */
59 static int tsc_unstable;
60
61 static inline int check_tsc_unstable(void)
62 {
63         return tsc_unstable;
64 }
65
66 /* Accellerators for sched_clock()
67  * convert from cycles(64bits) => nanoseconds (64bits)
68  *  basic equation:
69  *              ns = cycles / (freq / ns_per_sec)
70  *              ns = cycles * (ns_per_sec / freq)
71  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
72  *              ns = cycles * (10^6 / cpu_khz)
73  *
74  *      Then we use scaling math (suggested by george@mvista.com) to get:
75  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
76  *              ns = cycles * cyc2ns_scale / SC
77  *
78  *      And since SC is a constant power of two, we can convert the div
79  *  into a shift.
80  *
81  *  We can use khz divisor instead of mhz to keep a better percision, since
82  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
83  *  (mathieu.desnoyers@polymtl.ca)
84  *
85  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
86  */
87 static unsigned long cyc2ns_scale __read_mostly;
88
89 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
90
91 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
92 {
93         cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
94 }
95
96 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
97 {
98         return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
99 }
100
101 /*
102  * Scheduler clock - returns current time in nanosec units.
103  */
104 unsigned long long sched_clock(void)
105 {
106         unsigned long long this_offset;
107
108         /*
109          * Fall back to jiffies if there's no TSC available:
110          * ( But note that we still use it if the TSC is marked
111          *   unstable. We do this because unlike Time Of Day,
112          *   the scheduler clock tolerates small errors and it's
113          *   very important for it to be as fast as the platform
114          *   can achive it. )
115          */
116         if (unlikely(!tsc_enabled && !tsc_unstable))
117                 /* No locking but a rare wrong value is not a big deal: */
118                 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
119
120         /* read the Time Stamp Counter: */
121         get_scheduled_cycles(this_offset);
122
123         /* return the value in ns */
124         return cycles_2_ns(this_offset);
125 }
126
127 unsigned long native_calculate_cpu_khz(void)
128 {
129         unsigned long long start, end;
130         unsigned long count;
131         u64 delta64;
132         int i;
133         unsigned long flags;
134
135         local_irq_save(flags);
136
137         /* run 3 times to ensure the cache is warm */
138         for (i = 0; i < 3; i++) {
139                 mach_prepare_counter();
140                 rdtscll(start);
141                 mach_countup(&count);
142                 rdtscll(end);
143         }
144         /*
145          * Error: ECTCNEVERSET
146          * The CTC wasn't reliable: we got a hit on the very first read,
147          * or the CPU was so fast/slow that the quotient wouldn't fit in
148          * 32 bits..
149          */
150         if (count <= 1)
151                 goto err;
152
153         delta64 = end - start;
154
155         /* cpu freq too fast: */
156         if (delta64 > (1ULL<<32))
157                 goto err;
158
159         /* cpu freq too slow: */
160         if (delta64 <= CALIBRATE_TIME_MSEC)
161                 goto err;
162
163         delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
164         do_div(delta64,CALIBRATE_TIME_MSEC);
165
166         local_irq_restore(flags);
167         return (unsigned long)delta64;
168 err:
169         local_irq_restore(flags);
170         return 0;
171 }
172
173 int recalibrate_cpu_khz(void)
174 {
175 #ifndef CONFIG_SMP
176         unsigned long cpu_khz_old = cpu_khz;
177
178         if (cpu_has_tsc) {
179                 cpu_khz = calculate_cpu_khz();
180                 tsc_khz = cpu_khz;
181                 cpu_data[0].loops_per_jiffy =
182                         cpufreq_scale(cpu_data[0].loops_per_jiffy,
183                                         cpu_khz_old, cpu_khz);
184                 return 0;
185         } else
186                 return -ENODEV;
187 #else
188         return -ENODEV;
189 #endif
190 }
191
192 EXPORT_SYMBOL(recalibrate_cpu_khz);
193
194 #ifdef CONFIG_CPU_FREQ
195
196 /*
197  * if the CPU frequency is scaled, TSC-based delays will need a different
198  * loops_per_jiffy value to function properly.
199  */
200 static unsigned int ref_freq = 0;
201 static unsigned long loops_per_jiffy_ref = 0;
202 static unsigned long cpu_khz_ref = 0;
203
204 static int
205 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
206 {
207         struct cpufreq_freqs *freq = data;
208
209         if (!ref_freq) {
210                 if (!freq->old){
211                         ref_freq = freq->new;
212                         return 0;
213                 }
214                 ref_freq = freq->old;
215                 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
216                 cpu_khz_ref = cpu_khz;
217         }
218
219         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
220             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
221             (val == CPUFREQ_RESUMECHANGE)) {
222                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
223                         cpu_data[freq->cpu].loops_per_jiffy =
224                                 cpufreq_scale(loops_per_jiffy_ref,
225                                                 ref_freq, freq->new);
226
227                 if (cpu_khz) {
228
229                         if (num_online_cpus() == 1)
230                                 cpu_khz = cpufreq_scale(cpu_khz_ref,
231                                                 ref_freq, freq->new);
232                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
233                                 tsc_khz = cpu_khz;
234                                 set_cyc2ns_scale(cpu_khz);
235                                 /*
236                                  * TSC based sched_clock turns
237                                  * to junk w/ cpufreq
238                                  */
239                                 mark_tsc_unstable("cpufreq changes");
240                         }
241                 }
242         }
243
244         return 0;
245 }
246
247 static struct notifier_block time_cpufreq_notifier_block = {
248         .notifier_call  = time_cpufreq_notifier
249 };
250
251 static int __init cpufreq_tsc(void)
252 {
253         return cpufreq_register_notifier(&time_cpufreq_notifier_block,
254                                          CPUFREQ_TRANSITION_NOTIFIER);
255 }
256 core_initcall(cpufreq_tsc);
257
258 #endif
259
260 /* clock source code */
261
262 static unsigned long current_tsc_khz = 0;
263
264 static cycle_t read_tsc(void)
265 {
266         cycle_t ret;
267
268         rdtscll(ret);
269
270         return ret;
271 }
272
273 static struct clocksource clocksource_tsc = {
274         .name                   = "tsc",
275         .rating                 = 300,
276         .read                   = read_tsc,
277         .mask                   = CLOCKSOURCE_MASK(64),
278         .mult                   = 0, /* to be set */
279         .shift                  = 22,
280         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
281                                   CLOCK_SOURCE_MUST_VERIFY,
282 };
283
284 void mark_tsc_unstable(char *reason)
285 {
286         sched_clock_unstable_event();
287         if (!tsc_unstable) {
288                 tsc_unstable = 1;
289                 tsc_enabled = 0;
290                 printk("Marking TSC unstable due to: %s.\n", reason);
291                 /* Can be called before registration */
292                 if (clocksource_tsc.mult)
293                         clocksource_change_rating(&clocksource_tsc, 0);
294                 else
295                         clocksource_tsc.rating = 0;
296         }
297 }
298 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
299
300 static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d)
301 {
302         printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
303                        d->ident);
304         tsc_unstable = 1;
305         return 0;
306 }
307
308 /* List of systems that have known TSC problems */
309 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
310         {
311          .callback = dmi_mark_tsc_unstable,
312          .ident = "IBM Thinkpad 380XD",
313          .matches = {
314                      DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
315                      DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
316                      },
317          },
318          {}
319 };
320
321 /*
322  * Make an educated guess if the TSC is trustworthy and synchronized
323  * over all CPUs.
324  */
325 __cpuinit int unsynchronized_tsc(void)
326 {
327         if (!cpu_has_tsc || tsc_unstable)
328                 return 1;
329         /*
330          * Intel systems are normally all synchronized.
331          * Exceptions must mark TSC as unstable:
332          */
333         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
334                 /* assume multi socket systems are not synchronized: */
335                 if (num_possible_cpus() > 1)
336                         tsc_unstable = 1;
337         }
338         return tsc_unstable;
339 }
340
341 /*
342  * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
343  */
344 #ifdef CONFIG_MGEODE_LX
345 /* RTSC counts during suspend */
346 #define RTSC_SUSP 0x100
347
348 static void __init check_geode_tsc_reliable(void)
349 {
350         unsigned long val;
351
352         rdmsrl(MSR_GEODE_BUSCONT_CONF0, val);
353         if ((val & RTSC_SUSP))
354                 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
355 }
356 #else
357 static inline void check_geode_tsc_reliable(void) { }
358 #endif
359
360
361 void __init tsc_init(void)
362 {
363         if (!cpu_has_tsc || tsc_disable)
364                 goto out_no_tsc;
365
366         cpu_khz = calculate_cpu_khz();
367         tsc_khz = cpu_khz;
368
369         if (!cpu_khz)
370                 goto out_no_tsc;
371
372         printk("Detected %lu.%03lu MHz processor.\n",
373                                 (unsigned long)cpu_khz / 1000,
374                                 (unsigned long)cpu_khz % 1000);
375
376         set_cyc2ns_scale(cpu_khz);
377         use_tsc_delay();
378
379         /* Check and install the TSC clocksource */
380         dmi_check_system(bad_tsc_dmi_table);
381
382         unsynchronized_tsc();
383         check_geode_tsc_reliable();
384         current_tsc_khz = tsc_khz;
385         clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
386                                                         clocksource_tsc.shift);
387         /* lower the rating if we already know its unstable: */
388         if (check_tsc_unstable()) {
389                 clocksource_tsc.rating = 0;
390                 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
391         } else
392                 tsc_enabled = 1;
393
394         clocksource_register(&clocksource_tsc);
395
396         return;
397
398 out_no_tsc:
399         /*
400          * Set the tsc_disable flag if there's no TSC support, this
401          * makes it a fast flag for the kernel to see whether it
402          * should be using the TSC.
403          */
404         tsc_disable = 1;
405 }