git.oblomov.eu Git - linux-2.6/blob - arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *  Copyright(C) 2009 Jaswinder Singh Rajput
   7  *
   8  *  For licencing details see kernel-base/COPYING
   9  */
  10
  11 #include <linux/perf_counter.h>
  12 #include <linux/capability.h>
  13 #include <linux/notifier.h>
  14 #include <linux/hardirq.h>
  15 #include <linux/kprobes.h>
  16 #include <linux/module.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/sched.h>
  19
  20 #include <asm/perf_counter.h>
  21 #include <asm/apic.h>
  22
  23 static bool perf_counters_initialized __read_mostly;
  24
  25 /*
  26  * Number of (generic) HW counters:
  27  */
  28 static int nr_counters_generic __read_mostly;
  29 static u64 perf_counter_mask __read_mostly;
  30 static u64 counter_value_mask __read_mostly;
  31 static int counter_value_bits __read_mostly;
  32
  33 static int nr_counters_fixed __read_mostly;
  34
  35 struct cpu_hw_counters {
  36         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  37         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  38         unsigned long           interrupts;
  39         u64                     throttle_ctrl;
  40         u64                     active_mask;
  41         int                     enabled;
  42 };
  43
  44 /*
  45  * struct pmc_x86_ops - performance counter x86 ops
  46  */
  47 struct pmc_x86_ops {
  48         u64             (*save_disable_all)(void);
  49         void            (*restore_all)(u64);
  50         u64             (*get_status)(u64);
  51         void            (*ack_status)(u64);
  52         void            (*enable)(int, u64);
  53         void            (*disable)(int, u64);
  54         unsigned        eventsel;
  55         unsigned        perfctr;
  56         u64             (*event_map)(int);
  57         u64             (*raw_event)(u64);
  58         int             max_events;
  59 };
  60
  61 static struct pmc_x86_ops *pmc_ops;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  73   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  74   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  75   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  76   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  77   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  78   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  79 };
  80
  81 static u64 pmc_intel_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 static u64 pmc_intel_raw_event(u64 event)
  87 {
  88 #define CORE_EVNTSEL_EVENT_MASK         0x000000FF
  89 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00
  90 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000
  91
  92 #define CORE_EVNTSEL_MASK               \
  93         (CORE_EVNTSEL_EVENT_MASK |      \
  94          CORE_EVNTSEL_UNIT_MASK  |      \
  95          CORE_EVNTSEL_COUNTER_MASK)
  96
  97         return event & CORE_EVNTSEL_MASK;
  98 }
  99
 100 /*
 101  * AMD Performance Monitor K7 and later.
 102  */
 103 static const u64 amd_perfmon_event_map[] =
 104 {
 105   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 106   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 107   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 108   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 109   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 110   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 111 };
 112
 113 static u64 pmc_amd_event_map(int event)
 114 {
 115         return amd_perfmon_event_map[event];
 116 }
 117
 118 static u64 pmc_amd_raw_event(u64 event)
 119 {
 120 #define K7_EVNTSEL_EVENT_MASK   0x7000000FF
 121 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00
 122 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000
 123
 124 #define K7_EVNTSEL_MASK                 \
 125         (K7_EVNTSEL_EVENT_MASK |        \
 126          K7_EVNTSEL_UNIT_MASK  |        \
 127          K7_EVNTSEL_COUNTER_MASK)
 128
 129         return event & K7_EVNTSEL_MASK;
 130 }
 131
 132 /*
 133  * Propagate counter elapsed time into the generic counter.
 134  * Can only be executed on the CPU where the counter is active.
 135  * Returns the delta events processed.
 136  */
 137 static void
 138 x86_perf_counter_update(struct perf_counter *counter,
 139                         struct hw_perf_counter *hwc, int idx)
 140 {
 141         u64 prev_raw_count, new_raw_count, delta;
 142
 143         /*
 144          * Careful: an NMI might modify the previous counter value.
 145          *
 146          * Our tactic to handle this is to first atomically read and
 147          * exchange a new raw count - then add that new-prev delta
 148          * count to the generic counter atomically:
 149          */
 150 again:
 151         prev_raw_count = atomic64_read(&hwc->prev_count);
 152         rdmsrl(hwc->counter_base + idx, new_raw_count);
 153
 154         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 155                                         new_raw_count) != prev_raw_count)
 156                 goto again;
 157
 158         /*
 159          * Now we have the new raw value and have updated the prev
 160          * timestamp already. We can now calculate the elapsed delta
 161          * (counter-)time and add that to the generic counter.
 162          *
 163          * Careful, not all hw sign-extends above the physical width
 164          * of the count, so we do that by clipping the delta to 32 bits:
 165          */
 166         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
 167
 168         atomic64_add(delta, &counter->count);
 169         atomic64_sub(delta, &hwc->period_left);
 170 }
 171
 172 /*
 173  * Setup the hardware configuration for a given hw_event_type
 174  */
 175 static int __hw_perf_counter_init(struct perf_counter *counter)
 176 {
 177         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 178         struct hw_perf_counter *hwc = &counter->hw;
 179
 180         if (unlikely(!perf_counters_initialized))
 181                 return -EINVAL;
 182
 183         /*
 184          * Generate PMC IRQs:
 185          * (keep 'enabled' bit clear for now)
 186          */
 187         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 188
 189         /*
 190          * Count user and OS events unless requested not to.
 191          */
 192         if (!hw_event->exclude_user)
 193                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 194         if (!hw_event->exclude_kernel)
 195                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 196
 197         /*
 198          * If privileged enough, allow NMI events:
 199          */
 200         hwc->nmi = 0;
 201         if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
 202                 hwc->nmi = 1;
 203
 204         hwc->irq_period         = hw_event->irq_period;
 205         /*
 206          * Intel PMCs cannot be accessed sanely above 32 bit width,
 207          * so we install an artificial 1<<31 period regardless of
 208          * the generic counter period:
 209          */
 210         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 211                 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
 212                         hwc->irq_period = 0x7FFFFFFF;
 213
 214         atomic64_set(&hwc->period_left, hwc->irq_period);
 215
 216         /*
 217          * Raw event type provide the config in the event structure
 218          */
 219         if (hw_event->raw) {
 220                 hwc->config |= pmc_ops->raw_event(hw_event->type);
 221         } else {
 222                 if (hw_event->type >= pmc_ops->max_events)
 223                         return -EINVAL;
 224                 /*
 225                  * The generic map:
 226                  */
 227                 hwc->config |= pmc_ops->event_map(hw_event->type);
 228         }
 229         counter->wakeup_pending = 0;
 230
 231         return 0;
 232 }
 233
 234 static u64 pmc_intel_save_disable_all(void)
 235 {
 236         u64 ctrl;
 237
 238         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 239         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 240
 241         return ctrl;
 242 }
 243
 244 static u64 pmc_amd_save_disable_all(void)
 245 {
 246         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 247         int enabled, idx;
 248
 249         enabled = cpuc->enabled;
 250         cpuc->enabled = 0;
 251         barrier();
 252
 253         for (idx = 0; idx < nr_counters_generic; idx++) {
 254                 u64 val;
 255
 256                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 257                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) {
 258                         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 259                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 260                 }
 261         }
 262
 263         return enabled;
 264 }
 265
 266 u64 hw_perf_save_disable(void)
 267 {
 268         if (unlikely(!perf_counters_initialized))
 269                 return 0;
 270
 271         return pmc_ops->save_disable_all();
 272 }
 273 /*
 274  * Exported because of ACPI idle
 275  */
 276 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
 277
 278 static void pmc_intel_restore_all(u64 ctrl)
 279 {
 280         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 281 }
 282
 283 static void pmc_amd_restore_all(u64 ctrl)
 284 {
 285         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 286         int idx;
 287
 288         cpuc->enabled = ctrl;
 289         barrier();
 290         if (!ctrl)
 291                 return;
 292
 293         for (idx = 0; idx < nr_counters_generic; idx++) {
 294                 if (test_bit(idx, (unsigned long *)&cpuc->active_mask)) {
 295                         u64 val;
 296
 297                         rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 298                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 299                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 300                 }
 301         }
 302 }
 303
 304 void hw_perf_restore(u64 ctrl)
 305 {
 306         if (unlikely(!perf_counters_initialized))
 307                 return;
 308
 309         pmc_ops->restore_all(ctrl);
 310 }
 311 /*
 312  * Exported because of ACPI idle
 313  */
 314 EXPORT_SYMBOL_GPL(hw_perf_restore);
 315
 316 static u64 pmc_intel_get_status(u64 mask)
 317 {
 318         u64 status;
 319
 320         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 321
 322         return status;
 323 }
 324
 325 static u64 pmc_amd_get_status(u64 mask)
 326 {
 327         u64 status = 0;
 328         int idx;
 329
 330         for (idx = 0; idx < nr_counters_generic; idx++) {
 331                 s64 val;
 332
 333                 if (!(mask & (1 << idx)))
 334                         continue;
 335
 336                 rdmsrl(MSR_K7_PERFCTR0 + idx, val);
 337                 val <<= (64 - counter_value_bits);
 338                 if (val >= 0)
 339                         status |= (1 << idx);
 340         }
 341
 342         return status;
 343 }
 344
 345 static u64 hw_perf_get_status(u64 mask)
 346 {
 347         if (unlikely(!perf_counters_initialized))
 348                 return 0;
 349
 350         return pmc_ops->get_status(mask);
 351 }
 352
 353 static void pmc_intel_ack_status(u64 ack)
 354 {
 355         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 356 }
 357
 358 static void pmc_amd_ack_status(u64 ack)
 359 {
 360 }
 361
 362 static void hw_perf_ack_status(u64 ack)
 363 {
 364         if (unlikely(!perf_counters_initialized))
 365                 return;
 366
 367         pmc_ops->ack_status(ack);
 368 }
 369
 370 static void pmc_intel_enable(int idx, u64 config)
 371 {
 372         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
 373                         config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 374 }
 375
 376 static void pmc_amd_enable(int idx, u64 config)
 377 {
 378         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 379
 380         set_bit(idx, (unsigned long *)&cpuc->active_mask);
 381         if (cpuc->enabled)
 382                 config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 383
 384         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 385 }
 386
 387 static void hw_perf_enable(int idx, u64 config)
 388 {
 389         if (unlikely(!perf_counters_initialized))
 390                 return;
 391
 392         pmc_ops->enable(idx, config);
 393 }
 394
 395 static void pmc_intel_disable(int idx, u64 config)
 396 {
 397         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
 398 }
 399
 400 static void pmc_amd_disable(int idx, u64 config)
 401 {
 402         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 403
 404         clear_bit(idx, (unsigned long *)&cpuc->active_mask);
 405         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 406
 407 }
 408
 409 static void hw_perf_disable(int idx, u64 config)
 410 {
 411         if (unlikely(!perf_counters_initialized))
 412                 return;
 413
 414         pmc_ops->disable(idx, config);
 415 }
 416
 417 static inline void
 418 __pmc_fixed_disable(struct perf_counter *counter,
 419                     struct hw_perf_counter *hwc, unsigned int __idx)
 420 {
 421         int idx = __idx - X86_PMC_IDX_FIXED;
 422         u64 ctrl_val, mask;
 423         int err;
 424
 425         mask = 0xfULL << (idx * 4);
 426
 427         rdmsrl(hwc->config_base, ctrl_val);
 428         ctrl_val &= ~mask;
 429         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 430 }
 431
 432 static inline void
 433 __pmc_generic_disable(struct perf_counter *counter,
 434                            struct hw_perf_counter *hwc, unsigned int idx)
 435 {
 436         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 437                 __pmc_fixed_disable(counter, hwc, idx);
 438         else
 439                 hw_perf_disable(idx, hwc->config);
 440 }
 441
 442 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 443
 444 /*
 445  * Set the next IRQ period, based on the hwc->period_left value.
 446  * To be called with the counter disabled in hw:
 447  */
 448 static void
 449 __hw_perf_counter_set_period(struct perf_counter *counter,
 450                              struct hw_perf_counter *hwc, int idx)
 451 {
 452         s64 left = atomic64_read(&hwc->period_left);
 453         s32 period = hwc->irq_period;
 454         int err;
 455
 456         /*
 457          * If we are way outside a reasoable range then just skip forward:
 458          */
 459         if (unlikely(left <= -period)) {
 460                 left = period;
 461                 atomic64_set(&hwc->period_left, left);
 462         }
 463
 464         if (unlikely(left <= 0)) {
 465                 left += period;
 466                 atomic64_set(&hwc->period_left, left);
 467         }
 468
 469         per_cpu(prev_left[idx], smp_processor_id()) = left;
 470
 471         /*
 472          * The hw counter starts counting from this counter offset,
 473          * mark it to be able to extra future deltas:
 474          */
 475         atomic64_set(&hwc->prev_count, (u64)-left);
 476
 477         err = checking_wrmsrl(hwc->counter_base + idx,
 478                              (u64)(-left) & counter_value_mask);
 479 }
 480
 481 static inline void
 482 __pmc_fixed_enable(struct perf_counter *counter,
 483                    struct hw_perf_counter *hwc, unsigned int __idx)
 484 {
 485         int idx = __idx - X86_PMC_IDX_FIXED;
 486         u64 ctrl_val, bits, mask;
 487         int err;
 488
 489         /*
 490          * Enable IRQ generation (0x8),
 491          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 492          * if requested:
 493          */
 494         bits = 0x8ULL;
 495         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 496                 bits |= 0x2;
 497         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 498                 bits |= 0x1;
 499         bits <<= (idx * 4);
 500         mask = 0xfULL << (idx * 4);
 501
 502         rdmsrl(hwc->config_base, ctrl_val);
 503         ctrl_val &= ~mask;
 504         ctrl_val |= bits;
 505         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 506 }
 507
 508 static void
 509 __pmc_generic_enable(struct perf_counter *counter,
 510                           struct hw_perf_counter *hwc, int idx)
 511 {
 512         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 513                 __pmc_fixed_enable(counter, hwc, idx);
 514         else
 515                 hw_perf_enable(idx, hwc->config);
 516 }
 517
 518 static int
 519 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 520 {
 521         unsigned int event;
 522
 523         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 524                 return -1;
 525
 526         if (unlikely(hwc->nmi))
 527                 return -1;
 528
 529         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 530
 531         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_INSTRUCTIONS)))
 532                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 533         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_CPU_CYCLES)))
 534                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 535         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_BUS_CYCLES)))
 536                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 537
 538         return -1;
 539 }
 540
 541 /*
 542  * Find a PMC slot for the freshly enabled / scheduled in counter:
 543  */
 544 static int pmc_generic_enable(struct perf_counter *counter)
 545 {
 546         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 547         struct hw_perf_counter *hwc = &counter->hw;
 548         int idx;
 549
 550         idx = fixed_mode_idx(counter, hwc);
 551         if (idx >= 0) {
 552                 /*
 553                  * Try to get the fixed counter, if that is already taken
 554                  * then try to get a generic counter:
 555                  */
 556                 if (test_and_set_bit(idx, cpuc->used))
 557                         goto try_generic;
 558
 559                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 560                 /*
 561                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 562                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 563                  */
 564                 hwc->counter_base =
 565                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 566                 hwc->idx = idx;
 567         } else {
 568                 idx = hwc->idx;
 569                 /* Try to get the previous generic counter again */
 570                 if (test_and_set_bit(idx, cpuc->used)) {
 571 try_generic:
 572                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
 573                         if (idx == nr_counters_generic)
 574                                 return -EAGAIN;
 575
 576                         set_bit(idx, cpuc->used);
 577                         hwc->idx = idx;
 578                 }
 579                 hwc->config_base  = pmc_ops->eventsel;
 580                 hwc->counter_base = pmc_ops->perfctr;
 581         }
 582
 583         perf_counters_lapic_init(hwc->nmi);
 584
 585         __pmc_generic_disable(counter, hwc, idx);
 586
 587         cpuc->counters[idx] = counter;
 588         /*
 589          * Make it visible before enabling the hw:
 590          */
 591         smp_wmb();
 592
 593         __hw_perf_counter_set_period(counter, hwc, idx);
 594         __pmc_generic_enable(counter, hwc, idx);
 595
 596         return 0;
 597 }
 598
 599 void perf_counter_print_debug(void)
 600 {
 601         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 602         struct cpu_hw_counters *cpuc;
 603         int cpu, idx;
 604
 605         if (!nr_counters_generic)
 606                 return;
 607
 608         local_irq_disable();
 609
 610         cpu = smp_processor_id();
 611         cpuc = &per_cpu(cpu_hw_counters, cpu);
 612
 613         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
 614                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 615                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 616                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 617                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 618
 619                 pr_info("\n");
 620                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 621                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
 622                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 623                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 624         }
 625         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
 626
 627         for (idx = 0; idx < nr_counters_generic; idx++) {
 628                 rdmsrl(pmc_ops->eventsel + idx, pmc_ctrl);
 629                 rdmsrl(pmc_ops->perfctr  + idx, pmc_count);
 630
 631                 prev_left = per_cpu(prev_left[idx], cpu);
 632
 633                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 634                         cpu, idx, pmc_ctrl);
 635                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
 636                         cpu, idx, pmc_count);
 637                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
 638                         cpu, idx, prev_left);
 639         }
 640         for (idx = 0; idx < nr_counters_fixed; idx++) {
 641                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 642
 643                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 644                         cpu, idx, pmc_count);
 645         }
 646         local_irq_enable();
 647 }
 648
 649 static void pmc_generic_disable(struct perf_counter *counter)
 650 {
 651         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 652         struct hw_perf_counter *hwc = &counter->hw;
 653         unsigned int idx = hwc->idx;
 654
 655         __pmc_generic_disable(counter, hwc, idx);
 656
 657         clear_bit(idx, cpuc->used);
 658         cpuc->counters[idx] = NULL;
 659         /*
 660          * Make sure the cleared pointer becomes visible before we
 661          * (potentially) free the counter:
 662          */
 663         smp_wmb();
 664
 665         /*
 666          * Drain the remaining delta count out of a counter
 667          * that we are disabling:
 668          */
 669         x86_perf_counter_update(counter, hwc, idx);
 670 }
 671
 672 static void perf_store_irq_data(struct perf_counter *counter, u64 data)
 673 {
 674         struct perf_data *irqdata = counter->irqdata;
 675
 676         if (irqdata->len > PERF_DATA_BUFLEN - sizeof(u64)) {
 677                 irqdata->overrun++;
 678         } else {
 679                 u64 *p = (u64 *) &irqdata->data[irqdata->len];
 680
 681                 *p = data;
 682                 irqdata->len += sizeof(u64);
 683         }
 684 }
 685
 686 /*
 687  * Save and restart an expired counter. Called by NMI contexts,
 688  * so it has to be careful about preempting normal counter ops:
 689  */
 690 static void perf_save_and_restart(struct perf_counter *counter)
 691 {
 692         struct hw_perf_counter *hwc = &counter->hw;
 693         int idx = hwc->idx;
 694
 695         x86_perf_counter_update(counter, hwc, idx);
 696         __hw_perf_counter_set_period(counter, hwc, idx);
 697
 698         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 699                 __pmc_generic_enable(counter, hwc, idx);
 700 }
 701
 702 static void
 703 perf_handle_group(struct perf_counter *sibling, u64 *status, u64 *overflown)
 704 {
 705         struct perf_counter *counter, *group_leader = sibling->group_leader;
 706
 707         /*
 708          * Store sibling timestamps (if any):
 709          */
 710         list_for_each_entry(counter, &group_leader->sibling_list, list_entry) {
 711
 712                 x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
 713                 perf_store_irq_data(sibling, counter->hw_event.type);
 714                 perf_store_irq_data(sibling, atomic64_read(&counter->count));
 715         }
 716 }
 717
 718 /*
 719  * Maximum interrupt frequency of 100KHz per CPU
 720  */
 721 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
 722
 723 /*
 724  * This handler is triggered by the local APIC, so the APIC IRQ handling
 725  * rules apply:
 726  */
 727 static int __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
 728 {
 729         int bit, cpu = smp_processor_id();
 730         u64 ack, status;
 731         struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
 732         int ret = 0;
 733
 734         cpuc->throttle_ctrl = hw_perf_save_disable();
 735
 736         status = hw_perf_get_status(cpuc->throttle_ctrl);
 737         if (!status)
 738                 goto out;
 739
 740         ret = 1;
 741 again:
 742         inc_irq_stat(apic_perf_irqs);
 743         ack = status;
 744         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 745                 struct perf_counter *counter = cpuc->counters[bit];
 746
 747                 clear_bit(bit, (unsigned long *) &status);
 748                 if (!counter)
 749                         continue;
 750
 751                 perf_save_and_restart(counter);
 752
 753                 switch (counter->hw_event.record_type) {
 754                 case PERF_RECORD_SIMPLE:
 755                         continue;
 756                 case PERF_RECORD_IRQ:
 757                         perf_store_irq_data(counter, instruction_pointer(regs));
 758                         break;
 759                 case PERF_RECORD_GROUP:
 760                         perf_handle_group(counter, &status, &ack);
 761                         break;
 762                 }
 763                 /*
 764                  * From NMI context we cannot call into the scheduler to
 765                  * do a task wakeup - but we mark these generic as
 766                  * wakeup_pending and initate a wakeup callback:
 767                  */
 768                 if (nmi) {
 769                         counter->wakeup_pending = 1;
 770                         set_tsk_thread_flag(current, TIF_PERF_COUNTERS);
 771                 } else {
 772                         wake_up(&counter->waitq);
 773                 }
 774         }
 775
 776         hw_perf_ack_status(ack);
 777
 778         /*
 779          * Repeat if there is more work to be done:
 780          */
 781         status = hw_perf_get_status(cpuc->throttle_ctrl);
 782         if (status)
 783                 goto again;
 784 out:
 785         /*
 786          * Restore - do not reenable when global enable is off or throttled:
 787          */
 788         if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
 789                 hw_perf_restore(cpuc->throttle_ctrl);
 790
 791         return ret;
 792 }
 793
 794 void perf_counter_unthrottle(void)
 795 {
 796         struct cpu_hw_counters *cpuc;
 797
 798         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 799                 return;
 800
 801         if (unlikely(!perf_counters_initialized))
 802                 return;
 803
 804         cpuc = &__get_cpu_var(cpu_hw_counters);
 805         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
 806                 if (printk_ratelimit())
 807                         printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
 808                 hw_perf_restore(cpuc->throttle_ctrl);
 809         }
 810         cpuc->interrupts = 0;
 811 }
 812
 813 void smp_perf_counter_interrupt(struct pt_regs *regs)
 814 {
 815         irq_enter();
 816         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 817         ack_APIC_irq();
 818         __smp_perf_counter_interrupt(regs, 0);
 819         irq_exit();
 820 }
 821
 822 /*
 823  * This handler is triggered by NMI contexts:
 824  */
 825 void perf_counter_notify(struct pt_regs *regs)
 826 {
 827         struct cpu_hw_counters *cpuc;
 828         unsigned long flags;
 829         int bit, cpu;
 830
 831         local_irq_save(flags);
 832         cpu = smp_processor_id();
 833         cpuc = &per_cpu(cpu_hw_counters, cpu);
 834
 835         for_each_bit(bit, cpuc->used, X86_PMC_IDX_MAX) {
 836                 struct perf_counter *counter = cpuc->counters[bit];
 837
 838                 if (!counter)
 839                         continue;
 840
 841                 if (counter->wakeup_pending) {
 842                         counter->wakeup_pending = 0;
 843                         wake_up(&counter->waitq);
 844                 }
 845         }
 846
 847         local_irq_restore(flags);
 848 }
 849
 850 void perf_counters_lapic_init(int nmi)
 851 {
 852         u32 apic_val;
 853
 854         if (!perf_counters_initialized)
 855                 return;
 856         /*
 857          * Enable the performance counter vector in the APIC LVT:
 858          */
 859         apic_val = apic_read(APIC_LVTERR);
 860
 861         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 862         if (nmi)
 863                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 864         else
 865                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 866         apic_write(APIC_LVTERR, apic_val);
 867 }
 868
 869 static int __kprobes
 870 perf_counter_nmi_handler(struct notifier_block *self,
 871                          unsigned long cmd, void *__args)
 872 {
 873         struct die_args *args = __args;
 874         struct pt_regs *regs;
 875         int ret;
 876
 877         switch (cmd) {
 878         case DIE_NMI:
 879         case DIE_NMI_IPI:
 880                 break;
 881
 882         default:
 883                 return NOTIFY_DONE;
 884         }
 885
 886         regs = args->regs;
 887
 888         apic_write(APIC_LVTPC, APIC_DM_NMI);
 889         ret = __smp_perf_counter_interrupt(regs, 1);
 890
 891         return ret ? NOTIFY_STOP : NOTIFY_OK;
 892 }
 893
 894 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 895         .notifier_call          = perf_counter_nmi_handler,
 896         .next                   = NULL,
 897         .priority               = 1
 898 };
 899
 900 static struct pmc_x86_ops pmc_intel_ops = {
 901         .save_disable_all       = pmc_intel_save_disable_all,
 902         .restore_all            = pmc_intel_restore_all,
 903         .get_status             = pmc_intel_get_status,
 904         .ack_status             = pmc_intel_ack_status,
 905         .enable                 = pmc_intel_enable,
 906         .disable                = pmc_intel_disable,
 907         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
 908         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
 909         .event_map              = pmc_intel_event_map,
 910         .raw_event              = pmc_intel_raw_event,
 911         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
 912 };
 913
 914 static struct pmc_x86_ops pmc_amd_ops = {
 915         .save_disable_all       = pmc_amd_save_disable_all,
 916         .restore_all            = pmc_amd_restore_all,
 917         .get_status             = pmc_amd_get_status,
 918         .ack_status             = pmc_amd_ack_status,
 919         .enable                 = pmc_amd_enable,
 920         .disable                = pmc_amd_disable,
 921         .eventsel               = MSR_K7_EVNTSEL0,
 922         .perfctr                = MSR_K7_PERFCTR0,
 923         .event_map              = pmc_amd_event_map,
 924         .raw_event              = pmc_amd_raw_event,
 925         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
 926 };
 927
 928 static struct pmc_x86_ops *pmc_intel_init(void)
 929 {
 930         union cpuid10_eax eax;
 931         unsigned int ebx;
 932         unsigned int unused;
 933         union cpuid10_edx edx;
 934
 935         /*
 936          * Check whether the Architectural PerfMon supports
 937          * Branch Misses Retired Event or not.
 938          */
 939         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 940         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 941                 return NULL;
 942
 943         pr_info("Intel Performance Monitoring support detected.\n");
 944         pr_info("... version:         %d\n", eax.split.version_id);
 945         pr_info("... bit width:       %d\n", eax.split.bit_width);
 946         pr_info("... mask length:     %d\n", eax.split.mask_length);
 947
 948         nr_counters_generic = eax.split.num_counters;
 949         nr_counters_fixed = edx.split.num_counters_fixed;
 950         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
 951
 952         return &pmc_intel_ops;
 953 }
 954
 955 static struct pmc_x86_ops *pmc_amd_init(void)
 956 {
 957         u64 old;
 958         int bits;
 959
 960         nr_counters_generic = 4;
 961         nr_counters_fixed = 0;
 962         counter_value_mask = ~0ULL;
 963
 964         rdmsrl(MSR_K7_PERFCTR0, old);
 965         wrmsrl(MSR_K7_PERFCTR0, counter_value_mask);
 966         /*
 967          * read the truncated mask
 968          */
 969         rdmsrl(MSR_K7_PERFCTR0, counter_value_mask);
 970         wrmsrl(MSR_K7_PERFCTR0, old);
 971
 972         bits = 32 + fls(counter_value_mask >> 32);
 973         if (bits == 32)
 974                 bits = fls((u32)counter_value_mask);
 975         counter_value_bits = bits;
 976
 977         pr_info("AMD Performance Monitoring support detected.\n");
 978
 979         return &pmc_amd_ops;
 980 }
 981
 982 void __init init_hw_perf_counters(void)
 983 {
 984         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 985                 return;
 986
 987         switch (boot_cpu_data.x86_vendor) {
 988         case X86_VENDOR_INTEL:
 989                 pmc_ops = pmc_intel_init();
 990                 break;
 991         case X86_VENDOR_AMD:
 992                 pmc_ops = pmc_amd_init();
 993                 break;
 994         }
 995         if (!pmc_ops)
 996                 return;
 997
 998         pr_info("... num counters:    %d\n", nr_counters_generic);
 999         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
1000                 nr_counters_generic = X86_PMC_MAX_GENERIC;
1001                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1002                         nr_counters_generic, X86_PMC_MAX_GENERIC);
1003         }
1004         perf_counter_mask = (1 << nr_counters_generic) - 1;
1005         perf_max_counters = nr_counters_generic;
1006
1007         pr_info("... value mask:      %016Lx\n", counter_value_mask);
1008
1009         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
1010                 nr_counters_fixed = X86_PMC_MAX_FIXED;
1011                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1012                         nr_counters_fixed, X86_PMC_MAX_FIXED);
1013         }
1014         pr_info("... fixed counters:  %d\n", nr_counters_fixed);
1015
1016         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1017
1018         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1019         perf_counters_initialized = true;
1020
1021         perf_counters_lapic_init(0);
1022         register_die_notifier(&perf_counter_nmi_notifier);
1023 }
1024
1025 static void pmc_generic_read(struct perf_counter *counter)
1026 {
1027         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1028 }
1029
1030 static const struct hw_perf_counter_ops x86_perf_counter_ops = {
1031         .enable         = pmc_generic_enable,
1032         .disable        = pmc_generic_disable,
1033         .read           = pmc_generic_read,
1034 };
1035
1036 const struct hw_perf_counter_ops *
1037 hw_perf_counter_init(struct perf_counter *counter)
1038 {
1039         int err;
1040
1041         err = __hw_perf_counter_init(counter);
1042         if (err)
1043                 return NULL;
1044
1045         return &x86_perf_counter_ops;
1046 }