Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux...
[linux-2.6] / arch / x86 / kernel / cpu / perfctr-watchdog.c
1 /* local apic based NMI watchdog for various CPUs.
2    This file also handles reservation of performance counters for coordination
3    with other users (like oprofile).
4
5    Note that these events normally don't tick when the CPU idles. This means
6    the frequency varies with CPU load.
7
8    Original code for K7/P6 written by Keith Owens */
9
10 #include <linux/percpu.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/bitops.h>
14 #include <linux/smp.h>
15 #include <linux/nmi.h>
16 #include <asm/apic.h>
17 #include <asm/intel_arch_perfmon.h>
18
19 struct nmi_watchdog_ctlblk {
20         unsigned int cccr_msr;
21         unsigned int perfctr_msr;  /* the MSR to reset in NMI handler */
22         unsigned int evntsel_msr;  /* the MSR to select the events to handle */
23 };
24
25 /* Interface defining a CPU specific perfctr watchdog */
26 struct wd_ops {
27         int (*reserve)(void);
28         void (*unreserve)(void);
29         int (*setup)(unsigned nmi_hz);
30         void (*rearm)(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz);
31         void (*stop)(void);
32         unsigned perfctr;
33         unsigned evntsel;
34         u64 checkbit;
35 };
36
37 static const struct wd_ops *wd_ops;
38
39 /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
40  * offset from MSR_P4_BSU_ESCR0.  It will be the max for all platforms (for now)
41  */
42 #define NMI_MAX_COUNTER_BITS 66
43
44 /* perfctr_nmi_owner tracks the ownership of the perfctr registers:
45  * evtsel_nmi_owner tracks the ownership of the event selection
46  * - different performance counters/ event selection may be reserved for
47  *   different subsystems this reservation system just tries to coordinate
48  *   things a little
49  */
50 static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
51 static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
52
53 static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk);
54
55 /* converts an msr to an appropriate reservation bit */
56 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
57 {
58         /* returns the bit offset of the performance counter register */
59         switch (boot_cpu_data.x86_vendor) {
60         case X86_VENDOR_AMD:
61                 return (msr - MSR_K7_PERFCTR0);
62         case X86_VENDOR_INTEL:
63                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
64                         return (msr - MSR_ARCH_PERFMON_PERFCTR0);
65
66                 switch (boot_cpu_data.x86) {
67                 case 6:
68                         return (msr - MSR_P6_PERFCTR0);
69                 case 15:
70                         return (msr - MSR_P4_BPU_PERFCTR0);
71                 }
72         }
73         return 0;
74 }
75
76 /* converts an msr to an appropriate reservation bit */
77 /* returns the bit offset of the event selection register */
78 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
79 {
80         /* returns the bit offset of the event selection register */
81         switch (boot_cpu_data.x86_vendor) {
82         case X86_VENDOR_AMD:
83                 return (msr - MSR_K7_EVNTSEL0);
84         case X86_VENDOR_INTEL:
85                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
86                         return (msr - MSR_ARCH_PERFMON_EVENTSEL0);
87
88                 switch (boot_cpu_data.x86) {
89                 case 6:
90                         return (msr - MSR_P6_EVNTSEL0);
91                 case 15:
92                         return (msr - MSR_P4_BSU_ESCR0);
93                 }
94         }
95         return 0;
96
97 }
98
99 /* checks for a bit availability (hack for oprofile) */
100 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
101 {
102         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
103
104         return (!test_bit(counter, perfctr_nmi_owner));
105 }
106
107 /* checks the an msr for availability */
108 int avail_to_resrv_perfctr_nmi(unsigned int msr)
109 {
110         unsigned int counter;
111
112         counter = nmi_perfctr_msr_to_bit(msr);
113         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
114
115         return (!test_bit(counter, perfctr_nmi_owner));
116 }
117
118 int reserve_perfctr_nmi(unsigned int msr)
119 {
120         unsigned int counter;
121
122         counter = nmi_perfctr_msr_to_bit(msr);
123         /* register not managed by the allocator? */
124         if (counter > NMI_MAX_COUNTER_BITS)
125                 return 1;
126
127         if (!test_and_set_bit(counter, perfctr_nmi_owner))
128                 return 1;
129         return 0;
130 }
131
132 void release_perfctr_nmi(unsigned int msr)
133 {
134         unsigned int counter;
135
136         counter = nmi_perfctr_msr_to_bit(msr);
137         /* register not managed by the allocator? */
138         if (counter > NMI_MAX_COUNTER_BITS)
139                 return;
140
141         clear_bit(counter, perfctr_nmi_owner);
142 }
143
144 int reserve_evntsel_nmi(unsigned int msr)
145 {
146         unsigned int counter;
147
148         counter = nmi_evntsel_msr_to_bit(msr);
149         /* register not managed by the allocator? */
150         if (counter > NMI_MAX_COUNTER_BITS)
151                 return 1;
152
153         if (!test_and_set_bit(counter, evntsel_nmi_owner))
154                 return 1;
155         return 0;
156 }
157
158 void release_evntsel_nmi(unsigned int msr)
159 {
160         unsigned int counter;
161
162         counter = nmi_evntsel_msr_to_bit(msr);
163         /* register not managed by the allocator? */
164         if (counter > NMI_MAX_COUNTER_BITS)
165                 return;
166
167         clear_bit(counter, evntsel_nmi_owner);
168 }
169
170 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi);
171 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
172 EXPORT_SYMBOL(reserve_perfctr_nmi);
173 EXPORT_SYMBOL(release_perfctr_nmi);
174 EXPORT_SYMBOL(reserve_evntsel_nmi);
175 EXPORT_SYMBOL(release_evntsel_nmi);
176
177 void disable_lapic_nmi_watchdog(void)
178 {
179         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
180
181         if (atomic_read(&nmi_active) <= 0)
182                 return;
183
184         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
185         wd_ops->unreserve();
186
187         BUG_ON(atomic_read(&nmi_active) != 0);
188 }
189
190 void enable_lapic_nmi_watchdog(void)
191 {
192         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
193
194         /* are we already enabled */
195         if (atomic_read(&nmi_active) != 0)
196                 return;
197
198         /* are we lapic aware */
199         if (!wd_ops)
200                 return;
201         if (!wd_ops->reserve()) {
202                 printk(KERN_ERR "NMI watchdog: cannot reserve perfctrs\n");
203                 return;
204         }
205
206         on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
207         touch_nmi_watchdog();
208 }
209
210 /*
211  * Activate the NMI watchdog via the local APIC.
212  */
213
214 static unsigned int adjust_for_32bit_ctr(unsigned int hz)
215 {
216         u64 counter_val;
217         unsigned int retval = hz;
218
219         /*
220          * On Intel CPUs with P6/ARCH_PERFMON only 32 bits in the counter
221          * are writable, with higher bits sign extending from bit 31.
222          * So, we can only program the counter with 31 bit values and
223          * 32nd bit should be 1, for 33.. to be 1.
224          * Find the appropriate nmi_hz
225          */
226         counter_val = (u64)cpu_khz * 1000;
227         do_div(counter_val, retval);
228         if (counter_val > 0x7fffffffULL) {
229                 u64 count = (u64)cpu_khz * 1000;
230                 do_div(count, 0x7fffffffUL);
231                 retval = count + 1;
232         }
233         return retval;
234 }
235
236 static void
237 write_watchdog_counter(unsigned int perfctr_msr, const char *descr, unsigned nmi_hz)
238 {
239         u64 count = (u64)cpu_khz * 1000;
240
241         do_div(count, nmi_hz);
242         if(descr)
243                 Dprintk("setting %s to -0x%08Lx\n", descr, count);
244         wrmsrl(perfctr_msr, 0 - count);
245 }
246
247 static void write_watchdog_counter32(unsigned int perfctr_msr,
248                 const char *descr, unsigned nmi_hz)
249 {
250         u64 count = (u64)cpu_khz * 1000;
251
252         do_div(count, nmi_hz);
253         if(descr)
254                 Dprintk("setting %s to -0x%08Lx\n", descr, count);
255         wrmsr(perfctr_msr, (u32)(-count), 0);
256 }
257
258 /* AMD K7/K8/Family10h/Family11h support. AMD keeps this interface
259    nicely stable so there is not much variety */
260
261 #define K7_EVNTSEL_ENABLE       (1 << 22)
262 #define K7_EVNTSEL_INT          (1 << 20)
263 #define K7_EVNTSEL_OS           (1 << 17)
264 #define K7_EVNTSEL_USR          (1 << 16)
265 #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING    0x76
266 #define K7_NMI_EVENT            K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING
267
268 static int setup_k7_watchdog(unsigned nmi_hz)
269 {
270         unsigned int perfctr_msr, evntsel_msr;
271         unsigned int evntsel;
272         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
273
274         perfctr_msr = wd_ops->perfctr;
275         evntsel_msr = wd_ops->evntsel;
276
277         wrmsrl(perfctr_msr, 0UL);
278
279         evntsel = K7_EVNTSEL_INT
280                 | K7_EVNTSEL_OS
281                 | K7_EVNTSEL_USR
282                 | K7_NMI_EVENT;
283
284         /* setup the timer */
285         wrmsr(evntsel_msr, evntsel, 0);
286         write_watchdog_counter(perfctr_msr, "K7_PERFCTR0",nmi_hz);
287         apic_write(APIC_LVTPC, APIC_DM_NMI);
288         evntsel |= K7_EVNTSEL_ENABLE;
289         wrmsr(evntsel_msr, evntsel, 0);
290
291         wd->perfctr_msr = perfctr_msr;
292         wd->evntsel_msr = evntsel_msr;
293         wd->cccr_msr = 0;  //unused
294         return 1;
295 }
296
297 static void single_msr_stop_watchdog(void)
298 {
299         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
300
301         wrmsr(wd->evntsel_msr, 0, 0);
302 }
303
304 static int single_msr_reserve(void)
305 {
306         if (!reserve_perfctr_nmi(wd_ops->perfctr))
307                 return 0;
308
309         if (!reserve_evntsel_nmi(wd_ops->evntsel)) {
310                 release_perfctr_nmi(wd_ops->perfctr);
311                 return 0;
312         }
313         return 1;
314 }
315
316 static void single_msr_unreserve(void)
317 {
318         release_evntsel_nmi(wd_ops->evntsel);
319         release_perfctr_nmi(wd_ops->perfctr);
320 }
321
322 static void single_msr_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
323 {
324         /* start the cycle over again */
325         write_watchdog_counter(wd->perfctr_msr, NULL, nmi_hz);
326 }
327
328 static const struct wd_ops k7_wd_ops = {
329         .reserve = single_msr_reserve,
330         .unreserve = single_msr_unreserve,
331         .setup = setup_k7_watchdog,
332         .rearm = single_msr_rearm,
333         .stop = single_msr_stop_watchdog,
334         .perfctr = MSR_K7_PERFCTR0,
335         .evntsel = MSR_K7_EVNTSEL0,
336         .checkbit = 1ULL<<47,
337 };
338
339 /* Intel Model 6 (PPro+,P2,P3,P-M,Core1) */
340
341 #define P6_EVNTSEL0_ENABLE      (1 << 22)
342 #define P6_EVNTSEL_INT          (1 << 20)
343 #define P6_EVNTSEL_OS           (1 << 17)
344 #define P6_EVNTSEL_USR          (1 << 16)
345 #define P6_EVENT_CPU_CLOCKS_NOT_HALTED  0x79
346 #define P6_NMI_EVENT            P6_EVENT_CPU_CLOCKS_NOT_HALTED
347
348 static int setup_p6_watchdog(unsigned nmi_hz)
349 {
350         unsigned int perfctr_msr, evntsel_msr;
351         unsigned int evntsel;
352         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
353
354         perfctr_msr = wd_ops->perfctr;
355         evntsel_msr = wd_ops->evntsel;
356
357         /* KVM doesn't implement this MSR */
358         if (wrmsr_safe(perfctr_msr, 0, 0) < 0)
359                 return 0;
360
361         evntsel = P6_EVNTSEL_INT
362                 | P6_EVNTSEL_OS
363                 | P6_EVNTSEL_USR
364                 | P6_NMI_EVENT;
365
366         /* setup the timer */
367         wrmsr(evntsel_msr, evntsel, 0);
368         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
369         write_watchdog_counter32(perfctr_msr, "P6_PERFCTR0",nmi_hz);
370         apic_write(APIC_LVTPC, APIC_DM_NMI);
371         evntsel |= P6_EVNTSEL0_ENABLE;
372         wrmsr(evntsel_msr, evntsel, 0);
373
374         wd->perfctr_msr = perfctr_msr;
375         wd->evntsel_msr = evntsel_msr;
376         wd->cccr_msr = 0;  //unused
377         return 1;
378 }
379
380 static void p6_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
381 {
382         /* P6 based Pentium M need to re-unmask
383          * the apic vector but it doesn't hurt
384          * other P6 variant.
385          * ArchPerfom/Core Duo also needs this */
386         apic_write(APIC_LVTPC, APIC_DM_NMI);
387         /* P6/ARCH_PERFMON has 32 bit counter write */
388         write_watchdog_counter32(wd->perfctr_msr, NULL,nmi_hz);
389 }
390
391 static const struct wd_ops p6_wd_ops = {
392         .reserve = single_msr_reserve,
393         .unreserve = single_msr_unreserve,
394         .setup = setup_p6_watchdog,
395         .rearm = p6_rearm,
396         .stop = single_msr_stop_watchdog,
397         .perfctr = MSR_P6_PERFCTR0,
398         .evntsel = MSR_P6_EVNTSEL0,
399         .checkbit = 1ULL<<39,
400 };
401
402 /* Intel P4 performance counters. By far the most complicated of all. */
403
404 #define MSR_P4_MISC_ENABLE_PERF_AVAIL   (1<<7)
405 #define P4_ESCR_EVENT_SELECT(N) ((N)<<25)
406 #define P4_ESCR_OS              (1<<3)
407 #define P4_ESCR_USR             (1<<2)
408 #define P4_CCCR_OVF_PMI0        (1<<26)
409 #define P4_CCCR_OVF_PMI1        (1<<27)
410 #define P4_CCCR_THRESHOLD(N)    ((N)<<20)
411 #define P4_CCCR_COMPLEMENT      (1<<19)
412 #define P4_CCCR_COMPARE         (1<<18)
413 #define P4_CCCR_REQUIRED        (3<<16)
414 #define P4_CCCR_ESCR_SELECT(N)  ((N)<<13)
415 #define P4_CCCR_ENABLE          (1<<12)
416 #define P4_CCCR_OVF             (1<<31)
417
418 /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
419    CRU_ESCR0 (with any non-null event selector) through a complemented
420    max threshold. [IA32-Vol3, Section 14.9.9] */
421
422 static int setup_p4_watchdog(unsigned nmi_hz)
423 {
424         unsigned int perfctr_msr, evntsel_msr, cccr_msr;
425         unsigned int evntsel, cccr_val;
426         unsigned int misc_enable, dummy;
427         unsigned int ht_num;
428         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
429
430         rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy);
431         if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL))
432                 return 0;
433
434 #ifdef CONFIG_SMP
435         /* detect which hyperthread we are on */
436         if (smp_num_siblings == 2) {
437                 unsigned int ebx, apicid;
438
439                 ebx = cpuid_ebx(1);
440                 apicid = (ebx >> 24) & 0xff;
441                 ht_num = apicid & 1;
442         } else
443 #endif
444                 ht_num = 0;
445
446         /* performance counters are shared resources
447          * assign each hyperthread its own set
448          * (re-use the ESCR0 register, seems safe
449          * and keeps the cccr_val the same)
450          */
451         if (!ht_num) {
452                 /* logical cpu 0 */
453                 perfctr_msr = MSR_P4_IQ_PERFCTR0;
454                 evntsel_msr = MSR_P4_CRU_ESCR0;
455                 cccr_msr = MSR_P4_IQ_CCCR0;
456                 cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4);
457         } else {
458                 /* logical cpu 1 */
459                 perfctr_msr = MSR_P4_IQ_PERFCTR1;
460                 evntsel_msr = MSR_P4_CRU_ESCR0;
461                 cccr_msr = MSR_P4_IQ_CCCR1;
462                 cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
463         }
464
465         evntsel = P4_ESCR_EVENT_SELECT(0x3F)
466                 | P4_ESCR_OS
467                 | P4_ESCR_USR;
468
469         cccr_val |= P4_CCCR_THRESHOLD(15)
470                  | P4_CCCR_COMPLEMENT
471                  | P4_CCCR_COMPARE
472                  | P4_CCCR_REQUIRED;
473
474         wrmsr(evntsel_msr, evntsel, 0);
475         wrmsr(cccr_msr, cccr_val, 0);
476         write_watchdog_counter(perfctr_msr, "P4_IQ_COUNTER0", nmi_hz);
477         apic_write(APIC_LVTPC, APIC_DM_NMI);
478         cccr_val |= P4_CCCR_ENABLE;
479         wrmsr(cccr_msr, cccr_val, 0);
480         wd->perfctr_msr = perfctr_msr;
481         wd->evntsel_msr = evntsel_msr;
482         wd->cccr_msr = cccr_msr;
483         return 1;
484 }
485
486 static void stop_p4_watchdog(void)
487 {
488         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
489         wrmsr(wd->cccr_msr, 0, 0);
490         wrmsr(wd->evntsel_msr, 0, 0);
491 }
492
493 static int p4_reserve(void)
494 {
495         if (!reserve_perfctr_nmi(MSR_P4_IQ_PERFCTR0))
496                 return 0;
497 #ifdef CONFIG_SMP
498         if (smp_num_siblings > 1 && !reserve_perfctr_nmi(MSR_P4_IQ_PERFCTR1))
499                 goto fail1;
500 #endif
501         if (!reserve_evntsel_nmi(MSR_P4_CRU_ESCR0))
502                 goto fail2;
503         /* RED-PEN why is ESCR1 not reserved here? */
504         return 1;
505  fail2:
506 #ifdef CONFIG_SMP
507         if (smp_num_siblings > 1)
508                 release_perfctr_nmi(MSR_P4_IQ_PERFCTR1);
509  fail1:
510 #endif
511         release_perfctr_nmi(MSR_P4_IQ_PERFCTR0);
512         return 0;
513 }
514
515 static void p4_unreserve(void)
516 {
517 #ifdef CONFIG_SMP
518         if (smp_num_siblings > 1)
519                 release_perfctr_nmi(MSR_P4_IQ_PERFCTR1);
520 #endif
521         release_evntsel_nmi(MSR_P4_CRU_ESCR0);
522         release_perfctr_nmi(MSR_P4_IQ_PERFCTR0);
523 }
524
525 static void p4_rearm(struct nmi_watchdog_ctlblk *wd, unsigned nmi_hz)
526 {
527         unsigned dummy;
528         /*
529          * P4 quirks:
530          * - An overflown perfctr will assert its interrupt
531          *   until the OVF flag in its CCCR is cleared.
532          * - LVTPC is masked on interrupt and must be
533          *   unmasked by the LVTPC handler.
534          */
535         rdmsrl(wd->cccr_msr, dummy);
536         dummy &= ~P4_CCCR_OVF;
537         wrmsrl(wd->cccr_msr, dummy);
538         apic_write(APIC_LVTPC, APIC_DM_NMI);
539         /* start the cycle over again */
540         write_watchdog_counter(wd->perfctr_msr, NULL, nmi_hz);
541 }
542
543 static const struct wd_ops p4_wd_ops = {
544         .reserve = p4_reserve,
545         .unreserve = p4_unreserve,
546         .setup = setup_p4_watchdog,
547         .rearm = p4_rearm,
548         .stop = stop_p4_watchdog,
549         /* RED-PEN this is wrong for the other sibling */
550         .perfctr = MSR_P4_BPU_PERFCTR0,
551         .evntsel = MSR_P4_BSU_ESCR0,
552         .checkbit = 1ULL<<39,
553 };
554
555 /* Watchdog using the Intel architected PerfMon. Used for Core2 and hopefully
556    all future Intel CPUs. */
557
558 #define ARCH_PERFMON_NMI_EVENT_SEL      ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
559 #define ARCH_PERFMON_NMI_EVENT_UMASK    ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK
560
561 static struct wd_ops intel_arch_wd_ops;
562
563 static int setup_intel_arch_watchdog(unsigned nmi_hz)
564 {
565         unsigned int ebx;
566         union cpuid10_eax eax;
567         unsigned int unused;
568         unsigned int perfctr_msr, evntsel_msr;
569         unsigned int evntsel;
570         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
571
572         /*
573          * Check whether the Architectural PerfMon supports
574          * Unhalted Core Cycles Event or not.
575          * NOTE: Corresponding bit = 0 in ebx indicates event present.
576          */
577         cpuid(10, &(eax.full), &ebx, &unused, &unused);
578         if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
579             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
580                 return 0;
581
582         perfctr_msr = wd_ops->perfctr;
583         evntsel_msr = wd_ops->evntsel;
584
585         wrmsrl(perfctr_msr, 0UL);
586
587         evntsel = ARCH_PERFMON_EVENTSEL_INT
588                 | ARCH_PERFMON_EVENTSEL_OS
589                 | ARCH_PERFMON_EVENTSEL_USR
590                 | ARCH_PERFMON_NMI_EVENT_SEL
591                 | ARCH_PERFMON_NMI_EVENT_UMASK;
592
593         /* setup the timer */
594         wrmsr(evntsel_msr, evntsel, 0);
595         nmi_hz = adjust_for_32bit_ctr(nmi_hz);
596         write_watchdog_counter32(perfctr_msr, "INTEL_ARCH_PERFCTR0", nmi_hz);
597         apic_write(APIC_LVTPC, APIC_DM_NMI);
598         evntsel |= ARCH_PERFMON_EVENTSEL0_ENABLE;
599         wrmsr(evntsel_msr, evntsel, 0);
600
601         wd->perfctr_msr = perfctr_msr;
602         wd->evntsel_msr = evntsel_msr;
603         wd->cccr_msr = 0;  //unused
604         intel_arch_wd_ops.checkbit = 1ULL << (eax.split.bit_width - 1);
605         return 1;
606 }
607
608 static struct wd_ops intel_arch_wd_ops __read_mostly = {
609         .reserve = single_msr_reserve,
610         .unreserve = single_msr_unreserve,
611         .setup = setup_intel_arch_watchdog,
612         .rearm = p6_rearm,
613         .stop = single_msr_stop_watchdog,
614         .perfctr = MSR_ARCH_PERFMON_PERFCTR1,
615         .evntsel = MSR_ARCH_PERFMON_EVENTSEL1,
616 };
617
618 static struct wd_ops coreduo_wd_ops = {
619         .reserve = single_msr_reserve,
620         .unreserve = single_msr_unreserve,
621         .setup = setup_intel_arch_watchdog,
622         .rearm = p6_rearm,
623         .stop = single_msr_stop_watchdog,
624         .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
625         .evntsel = MSR_ARCH_PERFMON_EVENTSEL0,
626 };
627
628 static void probe_nmi_watchdog(void)
629 {
630         switch (boot_cpu_data.x86_vendor) {
631         case X86_VENDOR_AMD:
632                 if (boot_cpu_data.x86 != 6 && boot_cpu_data.x86 != 15 &&
633                     boot_cpu_data.x86 != 16)
634                         return;
635                 wd_ops = &k7_wd_ops;
636                 break;
637         case X86_VENDOR_INTEL:
638                 /* Work around Core Duo (Yonah) errata AE49 where perfctr1
639                    doesn't have a working enable bit. */
640                 if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 14) {
641                         wd_ops = &coreduo_wd_ops;
642                         break;
643                 }
644                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
645                         wd_ops = &intel_arch_wd_ops;
646                         break;
647                 }
648                 switch (boot_cpu_data.x86) {
649                 case 6:
650                         if (boot_cpu_data.x86_model > 0xd)
651                                 return;
652
653                         wd_ops = &p6_wd_ops;
654                         break;
655                 case 15:
656                         if (boot_cpu_data.x86_model > 0x4)
657                                 return;
658
659                         wd_ops = &p4_wd_ops;
660                         break;
661                 default:
662                         return;
663                 }
664                 break;
665         }
666 }
667
668 /* Interface to nmi.c */
669
670 int lapic_watchdog_init(unsigned nmi_hz)
671 {
672         if (!wd_ops) {
673                 probe_nmi_watchdog();
674                 if (!wd_ops)
675                         return -1;
676
677                 if (!wd_ops->reserve()) {
678                         printk(KERN_ERR
679                                 "NMI watchdog: cannot reserve perfctrs\n");
680                         return -1;
681                 }
682         }
683
684         if (!(wd_ops->setup(nmi_hz))) {
685                 printk(KERN_ERR "Cannot setup NMI watchdog on CPU %d\n",
686                        raw_smp_processor_id());
687                 return -1;
688         }
689
690         return 0;
691 }
692
693 void lapic_watchdog_stop(void)
694 {
695         if (wd_ops)
696                 wd_ops->stop();
697 }
698
699 unsigned lapic_adjust_nmi_hz(unsigned hz)
700 {
701         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
702         if (wd->perfctr_msr == MSR_P6_PERFCTR0 ||
703             wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR1)
704                 hz = adjust_for_32bit_ctr(hz);
705         return hz;
706 }
707
708 int lapic_wd_event(unsigned nmi_hz)
709 {
710         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
711         u64 ctr;
712         rdmsrl(wd->perfctr_msr, ctr);
713         if (ctr & wd_ops->checkbit) { /* perfctr still running? */
714                 return 0;
715         }
716         wd_ops->rearm(wd, nmi_hz);
717         return 1;
718 }
719
720 int lapic_watchdog_ok(void)
721 {
722         return wd_ops != NULL;
723 }