git.oblomov.eu Git - linux-2.6/blob - arch/x86_64/kernel/nmi.c

   1 /*
   2  *  linux/arch/x86_64/nmi.c
   3  *
   4  *  NMI watchdog support on APIC systems
   5  *
   6  *  Started by Ingo Molnar <mingo@redhat.com>
   7  *
   8  *  Fixes:
   9  *  Mikael Pettersson   : AMD K7 support for local APIC NMI watchdog.
  10  *  Mikael Pettersson   : Power Management for local APIC NMI watchdog.
  11  *  Pavel Machek and
  12  *  Mikael Pettersson   : PM converted to driver model. Disable/enable API.
  13  */
  14
  15 #include <linux/mm.h>
  16 #include <linux/delay.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/module.h>
  19 #include <linux/sysdev.h>
  20 #include <linux/nmi.h>
  21 #include <linux/sysctl.h>
  22 #include <linux/kprobes.h>
  23
  24 #include <asm/smp.h>
  25 #include <asm/nmi.h>
  26 #include <asm/proto.h>
  27 #include <asm/kdebug.h>
  28 #include <asm/mce.h>
  29
  30 /* perfctr_nmi_owner tracks the ownership of the perfctr registers:
  31  * evtsel_nmi_owner tracks the ownership of the event selection
  32  * - different performance counters/ event selection may be reserved for
  33  *   different subsystems this reservation system just tries to coordinate
  34  *   things a little
  35  */
  36 static DEFINE_PER_CPU(unsigned, perfctr_nmi_owner);
  37 static DEFINE_PER_CPU(unsigned, evntsel_nmi_owner[2]);
  38
  39 /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
  40  * offset from MSR_P4_BSU_ESCR0.  It will be the max for all platforms (for now)
  41  */
  42 #define NMI_MAX_COUNTER_BITS 66
  43
  44 /* nmi_active:
  45  * >0: the lapic NMI watchdog is active, but can be disabled
  46  * <0: the lapic NMI watchdog has not been set up, and cannot
  47  *     be enabled
  48  *  0: the lapic NMI watchdog is disabled, but can be enabled
  49  */
  50 atomic_t nmi_active = ATOMIC_INIT(0);           /* oprofile uses this */
  51 int panic_on_timeout;
  52
  53 unsigned int nmi_watchdog = NMI_DEFAULT;
  54 static unsigned int nmi_hz = HZ;
  55
  56 struct nmi_watchdog_ctlblk {
  57         int enabled;
  58         u64 check_bit;
  59         unsigned int cccr_msr;
  60         unsigned int perfctr_msr;  /* the MSR to reset in NMI handler */
  61         unsigned int evntsel_msr;  /* the MSR to select the events to handle */
  62 };
  63 static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk);
  64
  65 /* local prototypes */
  66 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu);
  67
  68 /* converts an msr to an appropriate reservation bit */
  69 static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
  70 {
  71         /* returns the bit offset of the performance counter register */
  72         switch (boot_cpu_data.x86_vendor) {
  73         case X86_VENDOR_AMD:
  74                 return (msr - MSR_K7_PERFCTR0);
  75         case X86_VENDOR_INTEL:
  76                 return (msr - MSR_P4_BPU_PERFCTR0);
  77         }
  78         return 0;
  79 }
  80
  81 /* converts an msr to an appropriate reservation bit */
  82 static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
  83 {
  84         /* returns the bit offset of the event selection register */
  85         switch (boot_cpu_data.x86_vendor) {
  86         case X86_VENDOR_AMD:
  87                 return (msr - MSR_K7_EVNTSEL0);
  88         case X86_VENDOR_INTEL:
  89                 return (msr - MSR_P4_BSU_ESCR0);
  90         }
  91         return 0;
  92 }
  93
  94 /* checks for a bit availability (hack for oprofile) */
  95 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
  96 {
  97         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
  98
  99         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
 100 }
 101
 102 /* checks the an msr for availability */
 103 int avail_to_resrv_perfctr_nmi(unsigned int msr)
 104 {
 105         unsigned int counter;
 106
 107         counter = nmi_perfctr_msr_to_bit(msr);
 108         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 109
 110         return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
 111 }
 112
 113 int reserve_perfctr_nmi(unsigned int msr)
 114 {
 115         unsigned int counter;
 116
 117         counter = nmi_perfctr_msr_to_bit(msr);
 118         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 119
 120         if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner)))
 121                 return 1;
 122         return 0;
 123 }
 124
 125 void release_perfctr_nmi(unsigned int msr)
 126 {
 127         unsigned int counter;
 128
 129         counter = nmi_perfctr_msr_to_bit(msr);
 130         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 131
 132         clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner));
 133 }
 134
 135 int reserve_evntsel_nmi(unsigned int msr)
 136 {
 137         unsigned int counter;
 138
 139         counter = nmi_evntsel_msr_to_bit(msr);
 140         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 141
 142         if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner)))
 143                 return 1;
 144         return 0;
 145 }
 146
 147 void release_evntsel_nmi(unsigned int msr)
 148 {
 149         unsigned int counter;
 150
 151         counter = nmi_evntsel_msr_to_bit(msr);
 152         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 153
 154         clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner));
 155 }
 156
 157 static __cpuinit inline int nmi_known_cpu(void)
 158 {
 159         switch (boot_cpu_data.x86_vendor) {
 160         case X86_VENDOR_AMD:
 161                 return boot_cpu_data.x86 == 15;
 162         case X86_VENDOR_INTEL:
 163                 return boot_cpu_data.x86 == 15;
 164         }
 165         return 0;
 166 }
 167
 168 /* Run after command line and cpu_init init, but before all other checks */
 169 void nmi_watchdog_default(void)
 170 {
 171         if (nmi_watchdog != NMI_DEFAULT)
 172                 return;
 173         if (nmi_known_cpu())
 174                 nmi_watchdog = NMI_LOCAL_APIC;
 175         else
 176                 nmi_watchdog = NMI_IO_APIC;
 177 }
 178
 179 #ifdef CONFIG_SMP
 180 /* The performance counters used by NMI_LOCAL_APIC don't trigger when
 181  * the CPU is idle. To make sure the NMI watchdog really ticks on all
 182  * CPUs during the test make them busy.
 183  */
 184 static __init void nmi_cpu_busy(void *data)
 185 {
 186         volatile int *endflag = data;
 187         local_irq_enable_in_hardirq();
 188         /* Intentionally don't use cpu_relax here. This is
 189            to make sure that the performance counter really ticks,
 190            even if there is a simulator or similar that catches the
 191            pause instruction. On a real HT machine this is fine because
 192            all other CPUs are busy with "useless" delay loops and don't
 193            care if they get somewhat less cycles. */
 194         while (*endflag == 0)
 195                 barrier();
 196 }
 197 #endif
 198
 199 int __init check_nmi_watchdog (void)
 200 {
 201         volatile int endflag = 0;
 202         int *counts;
 203         int cpu;
 204
 205         if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DEFAULT))
 206                 return 0;
 207
 208         if (!atomic_read(&nmi_active))
 209                 return 0;
 210
 211         counts = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL);
 212         if (!counts)
 213                 return -1;
 214
 215         printk(KERN_INFO "testing NMI watchdog ... ");
 216
 217 #ifdef CONFIG_SMP
 218         if (nmi_watchdog == NMI_LOCAL_APIC)
 219                 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0);
 220 #endif
 221
 222         for (cpu = 0; cpu < NR_CPUS; cpu++)
 223                 counts[cpu] = cpu_pda(cpu)->__nmi_count;
 224         local_irq_enable();
 225         mdelay((10*1000)/nmi_hz); // wait 10 ticks
 226
 227         for_each_online_cpu(cpu) {
 228                 if (!per_cpu(nmi_watchdog_ctlblk, cpu).enabled)
 229                         continue;
 230                 if (cpu_pda(cpu)->__nmi_count - counts[cpu] <= 5) {
 231                         printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n",
 232                                cpu,
 233                                counts[cpu],
 234                                cpu_pda(cpu)->__nmi_count);
 235                         per_cpu(nmi_watchdog_ctlblk, cpu).enabled = 0;
 236                         atomic_dec(&nmi_active);
 237                 }
 238         }
 239         if (!atomic_read(&nmi_active)) {
 240                 kfree(counts);
 241                 atomic_set(&nmi_active, -1);
 242                 return -1;
 243         }
 244         endflag = 1;
 245         printk("OK.\n");
 246
 247         /* now that we know it works we can reduce NMI frequency to
 248            something more reasonable; makes a difference in some configs */
 249         if (nmi_watchdog == NMI_LOCAL_APIC)
 250                 nmi_hz = 1;
 251
 252         kfree(counts);
 253         return 0;
 254 }
 255
 256 int __init setup_nmi_watchdog(char *str)
 257 {
 258         int nmi;
 259
 260         if (!strncmp(str,"panic",5)) {
 261                 panic_on_timeout = 1;
 262                 str = strchr(str, ',');
 263                 if (!str)
 264                         return 1;
 265                 ++str;
 266         }
 267
 268         get_option(&str, &nmi);
 269
 270         if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE))
 271                 return 0;
 272
 273         if ((nmi == NMI_LOCAL_APIC) && (nmi_known_cpu() == 0))
 274                 return 0;  /* no lapic support */
 275         nmi_watchdog = nmi;
 276         return 1;
 277 }
 278
 279 __setup("nmi_watchdog=", setup_nmi_watchdog);
 280
 281 static void disable_lapic_nmi_watchdog(void)
 282 {
 283         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
 284
 285         if (atomic_read(&nmi_active) <= 0)
 286                 return;
 287
 288         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
 289
 290         BUG_ON(atomic_read(&nmi_active) != 0);
 291 }
 292
 293 static void enable_lapic_nmi_watchdog(void)
 294 {
 295         BUG_ON(nmi_watchdog != NMI_LOCAL_APIC);
 296
 297         /* are we already enabled */
 298         if (atomic_read(&nmi_active) != 0)
 299                 return;
 300
 301         /* are we lapic aware */
 302         if (nmi_known_cpu() <= 0)
 303                 return;
 304
 305         on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
 306         touch_nmi_watchdog();
 307 }
 308
 309 void disable_timer_nmi_watchdog(void)
 310 {
 311         BUG_ON(nmi_watchdog != NMI_IO_APIC);
 312
 313         if (atomic_read(&nmi_active) <= 0)
 314                 return;
 315
 316         disable_irq(0);
 317         on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1);
 318
 319         BUG_ON(atomic_read(&nmi_active) != 0);
 320 }
 321
 322 void enable_timer_nmi_watchdog(void)
 323 {
 324         BUG_ON(nmi_watchdog != NMI_IO_APIC);
 325
 326         if (atomic_read(&nmi_active) == 0) {
 327                 touch_nmi_watchdog();
 328                 on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1);
 329                 enable_irq(0);
 330         }
 331 }
 332
 333 #ifdef CONFIG_PM
 334
 335 static int nmi_pm_active; /* nmi_active before suspend */
 336
 337 static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state)
 338 {
 339         /* only CPU0 goes here, other CPUs should be offline */
 340         nmi_pm_active = atomic_read(&nmi_active);
 341         stop_apic_nmi_watchdog(NULL);
 342         BUG_ON(atomic_read(&nmi_active) != 0);
 343         return 0;
 344 }
 345
 346 static int lapic_nmi_resume(struct sys_device *dev)
 347 {
 348         /* only CPU0 goes here, other CPUs should be offline */
 349         if (nmi_pm_active > 0) {
 350                 setup_apic_nmi_watchdog(NULL);
 351                 touch_nmi_watchdog();
 352         }
 353         return 0;
 354 }
 355
 356 static struct sysdev_class nmi_sysclass = {
 357         set_kset_name("lapic_nmi"),
 358         .resume         = lapic_nmi_resume,
 359         .suspend        = lapic_nmi_suspend,
 360 };
 361
 362 static struct sys_device device_lapic_nmi = {
 363         .id             = 0,
 364         .cls    = &nmi_sysclass,
 365 };
 366
 367 static int __init init_lapic_nmi_sysfs(void)
 368 {
 369         int error;
 370
 371         /* should really be a BUG_ON but b/c this is an
 372          * init call, it just doesn't work.  -dcz
 373          */
 374         if (nmi_watchdog != NMI_LOCAL_APIC)
 375                 return 0;
 376
 377         if ( atomic_read(&nmi_active) < 0 )
 378                 return 0;
 379
 380         error = sysdev_class_register(&nmi_sysclass);
 381         if (!error)
 382                 error = sysdev_register(&device_lapic_nmi);
 383         return error;
 384 }
 385 /* must come after the local APIC's device_initcall() */
 386 late_initcall(init_lapic_nmi_sysfs);
 387
 388 #endif  /* CONFIG_PM */
 389
 390 /*
 391  * Activate the NMI watchdog via the local APIC.
 392  * Original code written by Keith Owens.
 393  */
 394
 395 /* Note that these events don't tick when the CPU idles. This means
 396    the frequency varies with CPU load. */
 397
 398 #define K7_EVNTSEL_ENABLE       (1 << 22)
 399 #define K7_EVNTSEL_INT          (1 << 20)
 400 #define K7_EVNTSEL_OS           (1 << 17)
 401 #define K7_EVNTSEL_USR          (1 << 16)
 402 #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING    0x76
 403 #define K7_NMI_EVENT            K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING
 404
 405 static int setup_k7_watchdog(void)
 406 {
 407         unsigned int perfctr_msr, evntsel_msr;
 408         unsigned int evntsel;
 409         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 410
 411         perfctr_msr = MSR_K7_PERFCTR0;
 412         evntsel_msr = MSR_K7_EVNTSEL0;
 413         if (!reserve_perfctr_nmi(perfctr_msr))
 414                 goto fail;
 415
 416         if (!reserve_evntsel_nmi(evntsel_msr))
 417                 goto fail1;
 418
 419         /* Simulator may not support it */
 420         if (checking_wrmsrl(evntsel_msr, 0UL))
 421                 goto fail2;
 422         wrmsrl(perfctr_msr, 0UL);
 423
 424         evntsel = K7_EVNTSEL_INT
 425                 | K7_EVNTSEL_OS
 426                 | K7_EVNTSEL_USR
 427                 | K7_NMI_EVENT;
 428
 429         /* setup the timer */
 430         wrmsr(evntsel_msr, evntsel, 0);
 431         wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
 432         apic_write(APIC_LVTPC, APIC_DM_NMI);
 433         evntsel |= K7_EVNTSEL_ENABLE;
 434         wrmsr(evntsel_msr, evntsel, 0);
 435
 436         wd->perfctr_msr = perfctr_msr;
 437         wd->evntsel_msr = evntsel_msr;
 438         wd->cccr_msr = 0;  //unused
 439         wd->check_bit = 1ULL<<63;
 440         return 1;
 441 fail2:
 442         release_evntsel_nmi(evntsel_msr);
 443 fail1:
 444         release_perfctr_nmi(perfctr_msr);
 445 fail:
 446         return 0;
 447 }
 448
 449 static void stop_k7_watchdog(void)
 450 {
 451         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 452
 453         wrmsr(wd->evntsel_msr, 0, 0);
 454
 455         release_evntsel_nmi(wd->evntsel_msr);
 456         release_perfctr_nmi(wd->perfctr_msr);
 457 }
 458
 459 /* Note that these events don't tick when the CPU idles. This means
 460    the frequency varies with CPU load. */
 461
 462 #define MSR_P4_MISC_ENABLE_PERF_AVAIL   (1<<7)
 463 #define P4_ESCR_EVENT_SELECT(N) ((N)<<25)
 464 #define P4_ESCR_OS              (1<<3)
 465 #define P4_ESCR_USR             (1<<2)
 466 #define P4_CCCR_OVF_PMI0        (1<<26)
 467 #define P4_CCCR_OVF_PMI1        (1<<27)
 468 #define P4_CCCR_THRESHOLD(N)    ((N)<<20)
 469 #define P4_CCCR_COMPLEMENT      (1<<19)
 470 #define P4_CCCR_COMPARE         (1<<18)
 471 #define P4_CCCR_REQUIRED        (3<<16)
 472 #define P4_CCCR_ESCR_SELECT(N)  ((N)<<13)
 473 #define P4_CCCR_ENABLE          (1<<12)
 474 #define P4_CCCR_OVF             (1<<31)
 475 /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter
 476    CRU_ESCR0 (with any non-null event selector) through a complemented
 477    max threshold. [IA32-Vol3, Section 14.9.9] */
 478
 479 static int setup_p4_watchdog(void)
 480 {
 481         unsigned int perfctr_msr, evntsel_msr, cccr_msr;
 482         unsigned int evntsel, cccr_val;
 483         unsigned int misc_enable, dummy;
 484         unsigned int ht_num;
 485         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 486
 487         rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy);
 488         if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL))
 489                 return 0;
 490
 491 #ifdef CONFIG_SMP
 492         /* detect which hyperthread we are on */
 493         if (smp_num_siblings == 2) {
 494                 unsigned int ebx, apicid;
 495
 496                 ebx = cpuid_ebx(1);
 497                 apicid = (ebx >> 24) & 0xff;
 498                 ht_num = apicid & 1;
 499         } else
 500 #endif
 501                 ht_num = 0;
 502
 503         /* performance counters are shared resources
 504          * assign each hyperthread its own set
 505          * (re-use the ESCR0 register, seems safe
 506          * and keeps the cccr_val the same)
 507          */
 508         if (!ht_num) {
 509                 /* logical cpu 0 */
 510                 perfctr_msr = MSR_P4_IQ_PERFCTR0;
 511                 evntsel_msr = MSR_P4_CRU_ESCR0;
 512                 cccr_msr = MSR_P4_IQ_CCCR0;
 513                 cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4);
 514         } else {
 515                 /* logical cpu 1 */
 516                 perfctr_msr = MSR_P4_IQ_PERFCTR1;
 517                 evntsel_msr = MSR_P4_CRU_ESCR0;
 518                 cccr_msr = MSR_P4_IQ_CCCR1;
 519                 cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
 520         }
 521
 522         if (!reserve_perfctr_nmi(perfctr_msr))
 523                 goto fail;
 524
 525         if (!reserve_evntsel_nmi(evntsel_msr))
 526                 goto fail1;
 527
 528         evntsel = P4_ESCR_EVENT_SELECT(0x3F)
 529                 | P4_ESCR_OS
 530                 | P4_ESCR_USR;
 531
 532         cccr_val |= P4_CCCR_THRESHOLD(15)
 533                  | P4_CCCR_COMPLEMENT
 534                  | P4_CCCR_COMPARE
 535                  | P4_CCCR_REQUIRED;
 536
 537         wrmsr(evntsel_msr, evntsel, 0);
 538         wrmsr(cccr_msr, cccr_val, 0);
 539         wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
 540         apic_write(APIC_LVTPC, APIC_DM_NMI);
 541         cccr_val |= P4_CCCR_ENABLE;
 542         wrmsr(cccr_msr, cccr_val, 0);
 543
 544         wd->perfctr_msr = perfctr_msr;
 545         wd->evntsel_msr = evntsel_msr;
 546         wd->cccr_msr = cccr_msr;
 547         wd->check_bit = 1ULL<<39;
 548         return 1;
 549 fail1:
 550         release_perfctr_nmi(perfctr_msr);
 551 fail:
 552         return 0;
 553 }
 554
 555 static void stop_p4_watchdog(void)
 556 {
 557         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 558
 559         wrmsr(wd->cccr_msr, 0, 0);
 560         wrmsr(wd->evntsel_msr, 0, 0);
 561
 562         release_evntsel_nmi(wd->evntsel_msr);
 563         release_perfctr_nmi(wd->perfctr_msr);
 564 }
 565
 566 void setup_apic_nmi_watchdog(void *unused)
 567 {
 568         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 569
 570         /* only support LOCAL and IO APICs for now */
 571         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
 572             (nmi_watchdog != NMI_IO_APIC))
 573                 return;
 574
 575         if (wd->enabled == 1)
 576                 return;
 577
 578         /* cheap hack to support suspend/resume */
 579         /* if cpu0 is not active neither should the other cpus */
 580         if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0))
 581                 return;
 582
 583         if (nmi_watchdog == NMI_LOCAL_APIC) {
 584                 switch (boot_cpu_data.x86_vendor) {
 585                 case X86_VENDOR_AMD:
 586                         if (strstr(boot_cpu_data.x86_model_id, "Screwdriver"))
 587                                 return;
 588                         if (!setup_k7_watchdog())
 589                                 return;
 590                         break;
 591                 case X86_VENDOR_INTEL:
 592                         if (!setup_p4_watchdog())
 593                                 return;
 594                         break;
 595                 default:
 596                         return;
 597                 }
 598         }
 599         wd->enabled = 1;
 600         atomic_inc(&nmi_active);
 601 }
 602
 603 void stop_apic_nmi_watchdog(void *unused)
 604 {
 605         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 606
 607         /* only support LOCAL and IO APICs for now */
 608         if ((nmi_watchdog != NMI_LOCAL_APIC) &&
 609             (nmi_watchdog != NMI_IO_APIC))
 610                 return;
 611
 612         if (wd->enabled == 0)
 613                 return;
 614
 615         if (nmi_watchdog == NMI_LOCAL_APIC) {
 616                 switch (boot_cpu_data.x86_vendor) {
 617                 case X86_VENDOR_AMD:
 618                         if (strstr(boot_cpu_data.x86_model_id, "Screwdriver"))
 619                                 return;
 620                         stop_k7_watchdog();
 621                         break;
 622                 case X86_VENDOR_INTEL:
 623                         stop_p4_watchdog();
 624                         break;
 625                 default:
 626                         return;
 627                 }
 628         }
 629         wd->enabled = 0;
 630         atomic_dec(&nmi_active);
 631 }
 632
 633 /*
 634  * the best way to detect whether a CPU has a 'hard lockup' problem
 635  * is to check it's local APIC timer IRQ counts. If they are not
 636  * changing then that CPU has some problem.
 637  *
 638  * as these watchdog NMI IRQs are generated on every CPU, we only
 639  * have to check the current processor.
 640  */
 641
 642 static DEFINE_PER_CPU(unsigned, last_irq_sum);
 643 static DEFINE_PER_CPU(local_t, alert_counter);
 644 static DEFINE_PER_CPU(int, nmi_touch);
 645
 646 void touch_nmi_watchdog (void)
 647 {
 648         if (nmi_watchdog > 0) {
 649                 unsigned cpu;
 650
 651                 /*
 652                  * Tell other CPUs to reset their alert counters. We cannot
 653                  * do it ourselves because the alert count increase is not
 654                  * atomic.
 655                  */
 656                 for_each_present_cpu (cpu)
 657                         per_cpu(nmi_touch, cpu) = 1;
 658         }
 659
 660         touch_softlockup_watchdog();
 661 }
 662
 663 int __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason)
 664 {
 665         int sum;
 666         int touched = 0;
 667         struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk);
 668         u64 dummy;
 669         int rc=0;
 670
 671         /* check for other users first */
 672         if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT)
 673                         == NOTIFY_STOP) {
 674                 rc = 1;
 675                 touched = 1;
 676         }
 677
 678         sum = read_pda(apic_timer_irqs);
 679         if (__get_cpu_var(nmi_touch)) {
 680                 __get_cpu_var(nmi_touch) = 0;
 681                 touched = 1;
 682         }
 683
 684 #ifdef CONFIG_X86_MCE
 685         /* Could check oops_in_progress here too, but it's safer
 686            not too */
 687         if (atomic_read(&mce_entry) > 0)
 688                 touched = 1;
 689 #endif
 690         /* if the apic timer isn't firing, this cpu isn't doing much */
 691         if (!touched && __get_cpu_var(last_irq_sum) == sum) {
 692                 /*
 693                  * Ayiee, looks like this CPU is stuck ...
 694                  * wait a few IRQs (5 seconds) before doing the oops ...
 695                  */
 696                 local_inc(&__get_cpu_var(alert_counter));
 697                 if (local_read(&__get_cpu_var(alert_counter)) == 5*nmi_hz)
 698                         die_nmi("NMI Watchdog detected LOCKUP on CPU %d\n", regs,
 699                                 panic_on_timeout);
 700         } else {
 701                 __get_cpu_var(last_irq_sum) = sum;
 702                 local_set(&__get_cpu_var(alert_counter), 0);
 703         }
 704
 705         /* see if the nmi watchdog went off */
 706         if (wd->enabled) {
 707                 if (nmi_watchdog == NMI_LOCAL_APIC) {
 708                         rdmsrl(wd->perfctr_msr, dummy);
 709                         if (dummy & wd->check_bit){
 710                                 /* this wasn't a watchdog timer interrupt */
 711                                 goto done;
 712                         }
 713
 714                         /* only Intel uses the cccr msr */
 715                         if (wd->cccr_msr != 0) {
 716                                 /*
 717                                  * P4 quirks:
 718                                  * - An overflown perfctr will assert its interrupt
 719                                  *   until the OVF flag in its CCCR is cleared.
 720                                  * - LVTPC is masked on interrupt and must be
 721                                  *   unmasked by the LVTPC handler.
 722                                  */
 723                                 rdmsrl(wd->cccr_msr, dummy);
 724                                 dummy &= ~P4_CCCR_OVF;
 725                                 wrmsrl(wd->cccr_msr, dummy);
 726                                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 727                         }
 728                         /* start the cycle over again */
 729                         wrmsrl(wd->perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz));
 730                         rc = 1;
 731                 } else  if (nmi_watchdog == NMI_IO_APIC) {
 732                         /* don't know how to accurately check for this.
 733                          * just assume it was a watchdog timer interrupt
 734                          * This matches the old behaviour.
 735                          */
 736                         rc = 1;
 737                 } else
 738                         printk(KERN_WARNING "Unknown enabled NMI hardware?!\n");
 739         }
 740 done:
 741         return rc;
 742 }
 743
 744 asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code)
 745 {
 746         nmi_enter();
 747         add_pda(__nmi_count,1);
 748         default_do_nmi(regs);
 749         nmi_exit();
 750 }
 751
 752 int do_nmi_callback(struct pt_regs * regs, int cpu)
 753 {
 754 #ifdef CONFIG_SYSCTL
 755         if (unknown_nmi_panic)
 756                 return unknown_nmi_panic_callback(regs, cpu);
 757 #endif
 758         return 0;
 759 }
 760
 761 #ifdef CONFIG_SYSCTL
 762
 763 static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
 764 {
 765         unsigned char reason = get_nmi_reason();
 766         char buf[64];
 767
 768         sprintf(buf, "NMI received for unknown reason %02x\n", reason);
 769         die_nmi(buf, regs, 1);  /* Always panic here */
 770         return 0;
 771 }
 772
 773 /*
 774  * proc handler for /proc/sys/kernel/nmi
 775  */
 776 int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file,
 777                         void __user *buffer, size_t *length, loff_t *ppos)
 778 {
 779         int old_state;
 780
 781         nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0;
 782         old_state = nmi_watchdog_enabled;
 783         proc_dointvec(table, write, file, buffer, length, ppos);
 784         if (!!old_state == !!nmi_watchdog_enabled)
 785                 return 0;
 786
 787         if (atomic_read(&nmi_active) < 0) {
 788                 printk( KERN_WARNING "NMI watchdog is permanently disabled\n");
 789                 return -EIO;
 790         }
 791
 792         /* if nmi_watchdog is not set yet, then set it */
 793         nmi_watchdog_default();
 794
 795         if (nmi_watchdog == NMI_LOCAL_APIC) {
 796                 if (nmi_watchdog_enabled)
 797                         enable_lapic_nmi_watchdog();
 798                 else
 799                         disable_lapic_nmi_watchdog();
 800         } else {
 801                 printk( KERN_WARNING
 802                         "NMI watchdog doesn't know what hardware to touch\n");
 803                 return -EIO;
 804         }
 805         return 0;
 806 }
 807
 808 #endif
 809
 810 EXPORT_SYMBOL(nmi_active);
 811 EXPORT_SYMBOL(nmi_watchdog);
 812 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi);
 813 EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
 814 EXPORT_SYMBOL(reserve_perfctr_nmi);
 815 EXPORT_SYMBOL(release_perfctr_nmi);
 816 EXPORT_SYMBOL(reserve_evntsel_nmi);
 817 EXPORT_SYMBOL(release_evntsel_nmi);
 818 EXPORT_SYMBOL(disable_timer_nmi_watchdog);
 819 EXPORT_SYMBOL(enable_timer_nmi_watchdog);
 820 EXPORT_SYMBOL(touch_nmi_watchdog);