[PATCH] Use BUILD_BUG_ON in apic.c build sanity checking
[linux-2.6] / arch / x86_64 / kernel / nmi.c
1 /*
2  *  linux/arch/x86_64/nmi.c
3  *
4  *  NMI watchdog support on APIC systems
5  *
6  *  Started by Ingo Molnar <mingo@redhat.com>
7  *
8  *  Fixes:
9  *  Mikael Pettersson   : AMD K7 support for local APIC NMI watchdog.
10  *  Mikael Pettersson   : Power Management for local APIC NMI watchdog.
11  *  Pavel Machek and
12  *  Mikael Pettersson   : PM converted to driver model. Disable/enable API.
13  */
14
15 #include <linux/mm.h>
16 #include <linux/delay.h>
17 #include <linux/interrupt.h>
18 #include <linux/module.h>
19 #include <linux/sysdev.h>
20 #include <linux/nmi.h>
21 #include <linux/sysctl.h>
22 #include <linux/kprobes.h>
23
24 #include <asm/smp.h>
25 #include <asm/nmi.h>
26 #include <asm/proto.h>
27 #include <asm/kdebug.h>
28 #include <asm/mce.h>
29 #include <asm/intel_arch_perfmon.h>
30
31 /* perfctr_nmi_owner tracks the ownership of the perfctr registers:
32  * evtsel_nmi_owner tracks the ownership of the event selection
33  * - different performance counters/ event selection may be reserved for
34  *   different subsystems this reservation system just tries to coordinate
35  *   things a little
36  */
37 static DEFINE_PER_CPU(unsigned, perfctr_nmi_owner);
38 static DEFINE_PER_CPU(unsigned, evntsel_nmi_owner[2]);
39
40 /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
41  * offset from MSR_P4_BSU_ESCR0.  It will be the max for all platforms (for now)
42  */
43 #define NMI_MAX_COUNTER_BITS 66
44
45 /* nmi_active:
46  * >0: the lapic NMI watchdog is active, but can be disabled
47  * <0: the lapic NMI watchdog has not been set up, and cannot
48  *     be enabled
49  *  0: the lapic NMI watchdog is disabled, but can be enabled
50  */
51 atomic_t nmi_active = ATOMIC_INIT(0);           /* oprofile uses this */
52 int panic_on_timeout;
53
54 unsigned int nmi_watchdog = NMI_DEFAULT;
55 static unsigned int nmi_hz = HZ;
56
57 struct nmi_watchdog_ctlblk {
58         int enabled;
59         u64 check_bit;
60         unsigned int cccr_msr;
61         unsigned int perfctr_msr;  /* the MSR to reset in NMI handler */
62         unsigned int evntsel_msr;  /* the MSR to select the events to handle */
63 };
64 static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk);
65
66 /* local prototypes */
67 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu);
68
69 /* converts an msr to an appropriate reservation bit */
70 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
71 {
72         /* returns the bit offset of the performance counter register */
73         switch (boot_cpu_data.x86_vendor) {
74         case X86_VENDOR_AMD:
75                 return (msr - MSR_K7_PERFCTR0);
76         case X86_VENDOR_INTEL:
77                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
78                         return (msr - MSR_ARCH_PERFMON_PERFCTR0);
79                 else
80                         return (msr - MSR_P4_BPU_PERFCTR0);
81         }
82         return 0;
83 }
84
85 /* converts an msr to an appropriate reservation bit */
86 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
87 {
88         /* returns the bit offset of the event selection register */
89         switch (boot_cpu_data.x86_vendor) {
90         case X86_VENDOR_AMD:
91                 return (msr - MSR_K7_EVNTSEL0);
92         case X86_VENDOR_INTEL:
93                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
94                         return (msr - MSR_ARCH_PERFMON_EVENTSEL0);
95                 else
96                         return (msr - MSR_P4_BSU_ESCR0);
97         }
98         return 0;
99 }
100
101 /* checks for a bit availability (hack for oprofile) */
102 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
103 {
104         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
105
106         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
107 }
108
109 /* checks the an msr for availability */
110 int avail_to_resrv_perfctr_nmi(unsigned int msr)
111 {
112         unsigned int counter;
113
114         counter = nmi_perfctr_msr_to_bit(msr);
115         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
116
117         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
118 }
119
120 int reserve_perfctr_nmi(unsigned int msr)
121 {
122         unsigned int counter;
123
124         counter = nmi_perfctr_msr_to_bit(msr);
125         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
126
127         if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner)))
128                 return 1;
129         return 0;
130 }
131
132 void release_perfctr_nmi(unsigned int msr)
133 {
134         unsigned int counter;
135
136         counter = nmi_perfctr_msr_to_bit(msr);
137         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
138
139         clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner));
140 }
141
142 int reserve_evntsel_nmi(unsigned int msr)
143 {
144         unsigned int counter;
145
146         counter = nmi_evntsel_msr_to_bit(msr);
147         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
148
149         if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner)))
150                 return 1;
151         return 0;
152 }
153
154 void release_evntsel_nmi(unsigned int msr)
155 {
156         unsigned int counter;
157
158         counter = nmi_evntsel_msr_to_bit(msr);
159         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
160
161         clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner));
162 }
163
164 static __cpuinit inline int nmi_known_cpu(void)
165 {
166         switch (boot_cpu_data.x86_vendor) {
167         case X86_VENDOR_AMD:
168                 return boot_cpu_data.x86 == 15;
169         case X86_VENDOR_INTEL:
170                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
171                         return 1;
172                 else
173                         return (boot_cpu_data.x86 == 15);
174         }
175         return 0;
176 }
177
178 /* Run after command line and cpu_init init, but before all other checks */
179 void nmi_watchdog_default(void)
180 {
181         if (nmi_watchdog != NMI_DEFAULT)
182                 return;
183         if (nmi_known_cpu())
184                 nmi_watchdog = NMI_LOCAL_APIC;
185         else
186                 nmi_watchdog = NMI_IO_APIC;
187 }
188
189 #ifdef CONFIG_SMP
190 /* The performance counters used by NMI_LOCAL_APIC don't trigger when
191  * the CPU is idle. To make sure the NMI watchdog really ticks on all
192  * CPUs during the test make them busy.
193  */
194 static __init void nmi_cpu_busy(void *data)
195 {
196         volatile int *endflag = data;
197         local_irq_enable_in_hardirq();
198         /* Intentionally don't use cpu_relax here. This is
199            to make sure that the performance counter really ticks,
200            even if there is a simulator or similar that catches the
201            pause instruction. On a real HT machine this is fine because
202            all other CPUs are busy with "useless" delay loops and don't
203            care if they get somewhat less cycles. */
204         while (*endflag == 0)
205                 barrier();
206 }
207 #endif
208
209 int __init check_nmi_watchdog (void)
210 {
211         volatile int endflag = 0;
212         int *counts;
213         int cpu;
214
215         if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DEFAULT))
216                 return 0;
217
218         if (!atomic_read(&nmi_active))
219                 return 0;
220
221         counts = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL);
222         if (!counts)
223                 return -1;
224
225         printk(KERN_INFO "testing NMI watchdog ... ");
226
227 #ifdef CONFIG_SMP
228         if (nmi_watchdog == NMI_LOCAL_APIC)
229                 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0);
230 #endif
231
232         for (cpu = 0; cpu < NR_CPUS; cpu++)
233                 counts[cpu] = cpu_pda(cpu)->__nmi_count;
234         local_irq_enable();
235         mdelay((10*1000)/nmi_hz); // wait 10 ticks
236
237         for_each_online_cpu(cpu) {
238                 if (!per_cpu(nmi_watchdog_ctlblk, cpu).enabled)
239                         continue;
240                 if (cpu_pda(cpu)->__nmi_count - counts[cpu] <= 5) {
241                         printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n",
242                                cpu,
243                                counts[cpu],
244                                cpu_pda(cpu)->__nmi_count);
245                         per_cpu(nmi_watchdog_ctlblk, cpu).enabled = 0;
246                         atomic_dec(&nmi_active);
247                 }
248         }
249         if (!atomic_read(&nmi_active)) {
250                 kfree(counts);
251                 atomic_set(&nmi_active, -1);
252                 return -1;
253         }
254         endflag = 1;
255         printk("OK.\n");
256
257         /* now that we know it works we can reduce NMI frequency to
258            something more reasonable; makes a difference in some configs */
259         if (nmi_watchdog == NMI_LOCAL_APIC) {
260                 struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
261
262                 nmi_hz = 1;
263                 /*
264                  * On Intel CPUs with ARCH_PERFMON only 32 bits in the counter
265                  * are writable, with higher bits sign extending from bit 31.
266                  * So, we can only program the counter with 31 bit values and
267                  * 32nd bit should be 1, for 33.. to be 1.
268                  * Find the appropriate nmi_hz
269                  */
270                 if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0 &&
271                         ((u64)cpu_khz * 1000) > 0x7fffffffULL) {
272                         nmi_hz = ((u64)cpu_khz * 1000) / 0x7fffffffUL + 1;
273                 }
274         }
275
276         kfree(counts);
277         return 0;
278 }
279
280 int __init setup_nmi_watchdog(char *str)
281 {
282         int nmi;
283
284         if (!strncmp(str,"panic",5)) {
285                 panic_on_timeout = 1;
286                 str = strchr(str, ',');
287                 if (!str)
288                         return 1;
289                 ++str;
290         }
291
292         get_option(&str, &nmi);
293
294         if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE))
295                 return 0;
296
297         if ((nmi == NMI_LOCAL_APIC) && (nmi_known_cpu() == 0))
298                 return 0;  /* no lapic support */
299         nmi_watchdog = nmi;
300         return 1;
301 }
302
303 __setup("nmi_watchdog=", setup_nmi_watchdog);
304
305 static void disable_lapic_nmi_watchdog(void)
306 {
307         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
308
309         if (atomic_read(&nmi_active) <= 0)
310                 return;
311
312         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
313
314         BUG_ON(atomic_read(&nmi_active) != 0);
315 }
316
317 static void enable_lapic_nmi_watchdog(void)
318 {
319         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
320
321         /* are we already enabled */
322         if (atomic_read(&nmi_active) != 0)
323                 return;
324
325         /* are we lapic aware */
326         if (nmi_known_cpu() <= 0)
327                 return;
328
329         on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
330         touch_nmi_watchdog();
331 }
332
333 void disable_timer_nmi_watchdog(void)
334 {
335         BUG_ON(nmi_watchdog != NMI_IO_APIC);
336
337         if (atomic_read(&nmi_active) <= 0)
338                 return;
339
340         disable_irq(0);
341         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
342
343         BUG_ON(atomic_read(&nmi_active) != 0);
344 }
345
346 void enable_timer_nmi_watchdog(void)
347 {
348         BUG_ON(nmi_watchdog != NMI_IO_APIC);
349
350         if (atomic_read(&nmi_active) == 0) {
351                 touch_nmi_watchdog();
352                 on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
353                 enable_irq(0);
354         }
355 }
356
357 #ifdef CONFIG_PM
358
359 static int nmi_pm_active; /* nmi_active before suspend */
360
361 static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state)
362 {
363         /* only CPU0 goes here, other CPUs should be offline */
364         nmi_pm_active = atomic_read(&nmi_active);
365         stop_apic_nmi_watchdog(NULL);
366         BUG_ON(atomic_read(&nmi_active) != 0);
367         return 0;
368 }
369
370 static int lapic_nmi_resume(struct sys_device *dev)
371 {
372         /* only CPU0 goes here, other CPUs should be offline */
373         if (nmi_pm_active > 0) {
374                 setup_apic_nmi_watchdog(NULL);
375                 touch_nmi_watchdog();
376         }
377         return 0;
378 }
379
380 static struct sysdev_class nmi_sysclass = {
381         set_kset_name("lapic_nmi"),
382         .resume         = lapic_nmi_resume,
383         .suspend        = lapic_nmi_suspend,
384 };
385
386 static struct sys_device device_lapic_nmi = {
387         .id             = 0,
388         .cls    = &nmi_sysclass,
389 };
390
391 static int __init init_lapic_nmi_sysfs(void)
392 {
393         int error;
394
395         /* should really be a BUG_ON but b/c this is an
396          * init call, it just doesn't work.  -dcz
397          */
398         if (nmi_watchdog != NMI_LOCAL_APIC)
399                 return 0;
400
401         if ( atomic_read(&nmi_active) < 0 )
402                 return 0;
403
404         error = sysdev_class_register(&nmi_sysclass);
405         if (!error)
406                 error = sysdev_register(&device_lapic_nmi);
407         return error;
408 }
409 /* must come after the local APIC's device_initcall() */
410 late_initcall(init_lapic_nmi_sysfs);
411
412 #endif  /* CONFIG_PM */
413
414 /*
415  * Activate the NMI watchdog via the local APIC.
416  * Original code written by Keith Owens.
417  */
418
419 /* Note that these events don't tick when the CPU idles. This means
420    the frequency varies with CPU load. */
421
422 #define K7_EVNTSEL_ENABLE       (1 << 22)
423 #define K7_EVNTSEL_INT          (1 << 20)
424 #define K7_EVNTSEL_OS           (1 << 17)
425 #define K7_EVNTSEL_USR          (1 << 16)
426 #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING    0x76
427 #define K7_NMI_EVENT            K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING
428
429 static int setup_k7_watchdog(void)
430 {
431         unsigned int perfctr_msr, evntsel_msr;
432         unsigned int evntsel;
433         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
434
435         perfctr_msr = MSR_K7_PERFCTR0;
436         evntsel_msr = MSR_K7_EVNTSEL0;
437         if (!reserve_perfctr_nmi(perfctr_msr))
438                 goto fail;
439
440         if (!reserve_evntsel_nmi(evntsel_msr))
441                 goto fail1;
442
443         /* Simulator may not support it */
444         if (checking_wrmsrl(evntsel_msr, 0UL))
445                 goto fail2;
446         wrmsrl(perfctr_msr, 0UL);
447
448         evntsel = K7_EVNTSEL_INT
449                 | K7_EVNTSEL_OS
450                 | K7_EVNTSEL_USR
451                 | K7_NMI_EVENT;
452
453         /* setup the timer */
454         wrmsr(evntsel_msr, evntsel, 0);
455         wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
456         apic_write(APIC_LVTPC, APIC_DM_NMI);
457         evntsel |= K7_EVNTSEL_ENABLE;
458         wrmsr(evntsel_msr, evntsel, 0);
459
460         wd->perfctr_msr = perfctr_msr;
461         wd->evntsel_msr = evntsel_msr;
462         wd->cccr_msr = 0;  //unused
463         wd->check_bit = 1ULL<<63;
464         return 1;
465 fail2:
466         release_evntsel_nmi(evntsel_msr);
467 fail1:
468         release_perfctr_nmi(perfctr_msr);
469 fail:
470         return 0;
471 }
472
473 static void stop_k7_watchdog(void)
474 {
475         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
476
477         wrmsr(wd->evntsel_msr, 0, 0);
478
479         release_evntsel_nmi(wd->evntsel_msr);
480         release_perfctr_nmi(wd->perfctr_msr);
481 }
482
483 /* Note that these events don't tick when the CPU idles. This means
484    the frequency varies with CPU load. */
485
486 #define MSR_P4_MISC_ENABLE_PERF_AVAIL   (1<<7)
487 #define P4_ESCR_EVENT_SELECT(N) ((N)<<25)
488 #define P4_ESCR_OS              (1<<3)
489 #define P4_ESCR_USR             (1<<2)
490 #define P4_CCCR_OVF_PMI0        (1<<26)
491 #define P4_CCCR_OVF_PMI1        (1<<27)
492 #define P4_CCCR_THRESHOLD(N)    ((N)<<20)
493 #define P4_CCCR_COMPLEMENT      (1<<19)
494 #define P4_CCCR_COMPARE         (1<<18)
495 #define P4_CCCR_REQUIRED        (3<<16)
496 #define P4_CCCR_ESCR_SELECT(N)  ((N)<<13)
497 #define P4_CCCR_ENABLE          (1<<12)
498 #define P4_CCCR_OVF             (1<<31)
499 /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
500    CRU_ESCR0 (with any non-null event selector) through a complemented
501    max threshold. [IA32-Vol3, Section 14.9.9] */
502
503 static int setup_p4_watchdog(void)
504 {
505         unsigned int perfctr_msr, evntsel_msr, cccr_msr;
506         unsigned int evntsel, cccr_val;
507         unsigned int misc_enable, dummy;
508         unsigned int ht_num;
509         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
510
511         rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy);
512         if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL))
513                 return 0;
514
515 #ifdef CONFIG_SMP
516         /* detect which hyperthread we are on */
517         if (smp_num_siblings == 2) {
518                 unsigned int ebx, apicid;
519
520                 ebx = cpuid_ebx(1);
521                 apicid = (ebx >> 24) & 0xff;
522                 ht_num = apicid & 1;
523         } else
524 #endif
525                 ht_num = 0;
526
527         /* performance counters are shared resources
528          * assign each hyperthread its own set
529          * (re-use the ESCR0 register, seems safe
530          * and keeps the cccr_val the same)
531          */
532         if (!ht_num) {
533                 /* logical cpu 0 */
534                 perfctr_msr = MSR_P4_IQ_PERFCTR0;
535                 evntsel_msr = MSR_P4_CRU_ESCR0;
536                 cccr_msr = MSR_P4_IQ_CCCR0;
537                 cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4);
538         } else {
539                 /* logical cpu 1 */
540                 perfctr_msr = MSR_P4_IQ_PERFCTR1;
541                 evntsel_msr = MSR_P4_CRU_ESCR0;
542                 cccr_msr = MSR_P4_IQ_CCCR1;
543                 cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
544         }
545
546         if (!reserve_perfctr_nmi(perfctr_msr))
547                 goto fail;
548
549         if (!reserve_evntsel_nmi(evntsel_msr))
550                 goto fail1;
551
552         evntsel = P4_ESCR_EVENT_SELECT(0x3F)
553                 | P4_ESCR_OS
554                 | P4_ESCR_USR;
555
556         cccr_val |= P4_CCCR_THRESHOLD(15)
557                  | P4_CCCR_COMPLEMENT
558                  | P4_CCCR_COMPARE
559                  | P4_CCCR_REQUIRED;
560
561         wrmsr(evntsel_msr, evntsel, 0);
562         wrmsr(cccr_msr, cccr_val, 0);
563         wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
564         apic_write(APIC_LVTPC, APIC_DM_NMI);
565         cccr_val |= P4_CCCR_ENABLE;
566         wrmsr(cccr_msr, cccr_val, 0);
567
568         wd->perfctr_msr = perfctr_msr;
569         wd->evntsel_msr = evntsel_msr;
570         wd->cccr_msr = cccr_msr;
571         wd->check_bit = 1ULL<<39;
572         return 1;
573 fail1:
574         release_perfctr_nmi(perfctr_msr);
575 fail:
576         return 0;
577 }
578
579 static void stop_p4_watchdog(void)
580 {
581         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
582
583         wrmsr(wd->cccr_msr, 0, 0);
584         wrmsr(wd->evntsel_msr, 0, 0);
585
586         release_evntsel_nmi(wd->evntsel_msr);
587         release_perfctr_nmi(wd->perfctr_msr);
588 }
589
590 #define ARCH_PERFMON_NMI_EVENT_SEL      ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
591 #define ARCH_PERFMON_NMI_EVENT_UMASK    ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK
592
593 static int setup_intel_arch_watchdog(void)
594 {
595         unsigned int ebx;
596         union cpuid10_eax eax;
597         unsigned int unused;
598         unsigned int perfctr_msr, evntsel_msr;
599         unsigned int evntsel;
600         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
601
602         /*
603          * Check whether the Architectural PerfMon supports
604          * Unhalted Core Cycles Event or not.
605          * NOTE: Corresponding bit = 0 in ebx indicates event present.
606          */
607         cpuid(10, &(eax.full), &ebx, &unused, &unused);
608         if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
609             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
610                 goto fail;
611
612         perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0;
613         evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0;
614
615         if (!reserve_perfctr_nmi(perfctr_msr))
616                 goto fail;
617
618         if (!reserve_evntsel_nmi(evntsel_msr))
619                 goto fail1;
620
621         wrmsrl(perfctr_msr, 0UL);
622
623         evntsel = ARCH_PERFMON_EVENTSEL_INT
624                 | ARCH_PERFMON_EVENTSEL_OS
625                 | ARCH_PERFMON_EVENTSEL_USR
626                 | ARCH_PERFMON_NMI_EVENT_SEL
627                 | ARCH_PERFMON_NMI_EVENT_UMASK;
628
629         /* setup the timer */
630         wrmsr(evntsel_msr, evntsel, 0);
631         wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
632
633         apic_write(APIC_LVTPC, APIC_DM_NMI);
634         evntsel |= ARCH_PERFMON_EVENTSEL0_ENABLE;
635         wrmsr(evntsel_msr, evntsel, 0);
636
637         wd->perfctr_msr = perfctr_msr;
638         wd->evntsel_msr = evntsel_msr;
639         wd->cccr_msr = 0;  //unused
640         wd->check_bit = 1ULL << (eax.split.bit_width - 1);
641         return 1;
642 fail1:
643         release_perfctr_nmi(perfctr_msr);
644 fail:
645         return 0;
646 }
647
648 static void stop_intel_arch_watchdog(void)
649 {
650         unsigned int ebx;
651         union cpuid10_eax eax;
652         unsigned int unused;
653         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
654
655         /*
656          * Check whether the Architectural PerfMon supports
657          * Unhalted Core Cycles Event or not.
658          * NOTE: Corresponding bit = 0 in ebx indicates event present.
659          */
660         cpuid(10, &(eax.full), &ebx, &unused, &unused);
661         if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) ||
662             (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT))
663                 return;
664
665         wrmsr(wd->evntsel_msr, 0, 0);
666
667         release_evntsel_nmi(wd->evntsel_msr);
668         release_perfctr_nmi(wd->perfctr_msr);
669 }
670
671 void setup_apic_nmi_watchdog(void *unused)
672 {
673         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
674
675         /* only support LOCAL and IO APICs for now */
676         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
677             (nmi_watchdog != NMI_IO_APIC))
678                 return;
679
680         if (wd->enabled == 1)
681                 return;
682
683         /* cheap hack to support suspend/resume */
684         /* if cpu0 is not active neither should the other cpus */
685         if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0))
686                 return;
687
688         if (nmi_watchdog == NMI_LOCAL_APIC) {
689                 switch (boot_cpu_data.x86_vendor) {
690                 case X86_VENDOR_AMD:
691                         if (strstr(boot_cpu_data.x86_model_id, "Screwdriver"))
692                                 return;
693                         if (!setup_k7_watchdog())
694                                 return;
695                         break;
696                 case X86_VENDOR_INTEL:
697                         if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
698                                 if (!setup_intel_arch_watchdog())
699                                         return;
700                                 break;
701                         }
702                         if (!setup_p4_watchdog())
703                                 return;
704                         break;
705                 default:
706                         return;
707                 }
708         }
709         wd->enabled = 1;
710         atomic_inc(&nmi_active);
711 }
712
713 void stop_apic_nmi_watchdog(void *unused)
714 {
715         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
716
717         /* only support LOCAL and IO APICs for now */
718         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
719             (nmi_watchdog != NMI_IO_APIC))
720                 return;
721
722         if (wd->enabled == 0)
723                 return;
724
725         if (nmi_watchdog == NMI_LOCAL_APIC) {
726                 switch (boot_cpu_data.x86_vendor) {
727                 case X86_VENDOR_AMD:
728                         if (strstr(boot_cpu_data.x86_model_id, "Screwdriver"))
729                                 return;
730                         stop_k7_watchdog();
731                         break;
732                 case X86_VENDOR_INTEL:
733                         if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
734                                 stop_intel_arch_watchdog();
735                                 break;
736                         }
737                         stop_p4_watchdog();
738                         break;
739                 default:
740                         return;
741                 }
742         }
743         wd->enabled = 0;
744         atomic_dec(&nmi_active);
745 }
746
747 /*
748  * the best way to detect whether a CPU has a 'hard lockup' problem
749  * is to check it's local APIC timer IRQ counts. If they are not
750  * changing then that CPU has some problem.
751  *
752  * as these watchdog NMI IRQs are generated on every CPU, we only
753  * have to check the current processor.
754  */
755
756 static DEFINE_PER_CPU(unsigned, last_irq_sum);
757 static DEFINE_PER_CPU(local_t, alert_counter);
758 static DEFINE_PER_CPU(int, nmi_touch);
759
760 void touch_nmi_watchdog (void)
761 {
762         if (nmi_watchdog > 0) {
763                 unsigned cpu;
764
765                 /*
766                  * Tell other CPUs to reset their alert counters. We cannot
767                  * do it ourselves because the alert count increase is not
768                  * atomic.
769                  */
770                 for_each_present_cpu (cpu)
771                         per_cpu(nmi_touch, cpu) = 1;
772         }
773
774         touch_softlockup_watchdog();
775 }
776
777 int __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason)
778 {
779         int sum;
780         int touched = 0;
781         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
782         u64 dummy;
783         int rc=0;
784
785         /* check for other users first */
786         if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT)
787                         == NOTIFY_STOP) {
788                 rc = 1;
789                 touched = 1;
790         }
791
792         sum = read_pda(apic_timer_irqs);
793         if (__get_cpu_var(nmi_touch)) {
794                 __get_cpu_var(nmi_touch) = 0;
795                 touched = 1;
796         }
797
798 #ifdef CONFIG_X86_MCE
799         /* Could check oops_in_progress here too, but it's safer
800            not too */
801         if (atomic_read(&mce_entry) > 0)
802                 touched = 1;
803 #endif
804         /* if the apic timer isn't firing, this cpu isn't doing much */
805         if (!touched && __get_cpu_var(last_irq_sum) == sum) {
806                 /*
807                  * Ayiee, looks like this CPU is stuck ...
808                  * wait a few IRQs (5 seconds) before doing the oops ...
809                  */
810                 local_inc(&__get_cpu_var(alert_counter));
811                 if (local_read(&__get_cpu_var(alert_counter)) == 5*nmi_hz)
812                         die_nmi("NMI Watchdog detected LOCKUP on CPU %d\n", regs,
813                                 panic_on_timeout);
814         } else {
815                 __get_cpu_var(last_irq_sum) = sum;
816                 local_set(&__get_cpu_var(alert_counter), 0);
817         }
818
819         /* see if the nmi watchdog went off */
820         if (wd->enabled) {
821                 if (nmi_watchdog == NMI_LOCAL_APIC) {
822                         rdmsrl(wd->perfctr_msr, dummy);
823                         if (dummy & wd->check_bit){
824                                 /* this wasn't a watchdog timer interrupt */
825                                 goto done;
826                         }
827
828                         /* only Intel uses the cccr msr */
829                         if (wd->cccr_msr != 0) {
830                                 /*
831                                  * P4 quirks:
832                                  * - An overflown perfctr will assert its interrupt
833                                  *   until the OVF flag in its CCCR is cleared.
834                                  * - LVTPC is masked on interrupt and must be
835                                  *   unmasked by the LVTPC handler.
836                                  */
837                                 rdmsrl(wd->cccr_msr, dummy);
838                                 dummy &= ~P4_CCCR_OVF;
839                                 wrmsrl(wd->cccr_msr, dummy);
840                                 apic_write(APIC_LVTPC, APIC_DM_NMI);
841                         } else if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0) {
842                                 /*
843                                  * ArchPerfom/Core Duo needs to re-unmask
844                                  * the apic vector
845                                  */
846                                 apic_write(APIC_LVTPC, APIC_DM_NMI);
847                         }
848                         /* start the cycle over again */
849                         wrmsrl(wd->perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
850                         rc = 1;
851                 } else  if (nmi_watchdog == NMI_IO_APIC) {
852                         /* don't know how to accurately check for this.
853                          * just assume it was a watchdog timer interrupt
854                          * This matches the old behaviour.
855                          */
856                         rc = 1;
857                 } else
858                         printk(KERN_WARNING "Unknown enabled NMI hardware?!\n");
859         }
860 done:
861         return rc;
862 }
863
864 asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code)
865 {
866         nmi_enter();
867         add_pda(__nmi_count,1);
868         default_do_nmi(regs);
869         nmi_exit();
870 }
871
872 int do_nmi_callback(struct pt_regs * regs, int cpu)
873 {
874 #ifdef CONFIG_SYSCTL
875         if (unknown_nmi_panic)
876                 return unknown_nmi_panic_callback(regs, cpu);
877 #endif
878         return 0;
879 }
880
881 #ifdef CONFIG_SYSCTL
882
883 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
884 {
885         unsigned char reason = get_nmi_reason();
886         char buf[64];
887
888         sprintf(buf, "NMI received for unknown reason %02x\n", reason);
889         die_nmi(buf, regs, 1);  /* Always panic here */
890         return 0;
891 }
892
893 /*
894  * proc handler for /proc/sys/kernel/nmi
895  */
896 int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
897                         void __user *buffer, size_t *length, loff_t *ppos)
898 {
899         int old_state;
900
901         nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0;
902         old_state = nmi_watchdog_enabled;
903         proc_dointvec(table, write, file, buffer, length, ppos);
904         if (!!old_state == !!nmi_watchdog_enabled)
905                 return 0;
906
907         if (atomic_read(&nmi_active) < 0) {
908                 printk( KERN_WARNING "NMI watchdog is permanently disabled\n");
909                 return -EIO;
910         }
911
912         /* if nmi_watchdog is not set yet, then set it */
913         nmi_watchdog_default();
914
915         if (nmi_watchdog == NMI_LOCAL_APIC) {
916                 if (nmi_watchdog_enabled)
917                         enable_lapic_nmi_watchdog();
918                 else
919                         disable_lapic_nmi_watchdog();
920         } else {
921                 printk( KERN_WARNING
922                         "NMI watchdog doesn't know what hardware to touch\n");
923                 return -EIO;
924         }
925         return 0;
926 }
927
928 #endif
929
930 EXPORT_SYMBOL(nmi_active);
931 EXPORT_SYMBOL(nmi_watchdog);
932 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi);
933 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
934 EXPORT_SYMBOL(reserve_perfctr_nmi);
935 EXPORT_SYMBOL(release_perfctr_nmi);
936 EXPORT_SYMBOL(reserve_evntsel_nmi);
937 EXPORT_SYMBOL(release_evntsel_nmi);
938 EXPORT_SYMBOL(disable_timer_nmi_watchdog);
939 EXPORT_SYMBOL(enable_timer_nmi_watchdog);
940 EXPORT_SYMBOL(touch_nmi_watchdog);