git.oblomov.eu Git - linux-2.6/blob - arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *
  10  *  For licencing details see kernel-base/COPYING
  11  */
  12
  13 #include <linux/perf_counter.h>
  14 #include <linux/capability.h>
  15 #include <linux/notifier.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/module.h>
  19 #include <linux/kdebug.h>
  20 #include <linux/sched.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <asm/apic.h>
  24 #include <asm/stacktrace.h>
  25 #include <asm/nmi.h>
  26
  27 static u64 perf_counter_mask __read_mostly;
  28
  29 struct cpu_hw_counters {
  30         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  31         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  32         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  33         unsigned long           interrupts;
  34         int                     enabled;
  35 };
  36
  37 /*
  38  * struct x86_pmu - generic x86 pmu
  39  */
  40 struct x86_pmu {
  41         const char      *name;
  42         int             version;
  43         int             (*handle_irq)(struct pt_regs *);
  44         void            (*disable_all)(void);
  45         void            (*enable_all)(void);
  46         void            (*enable)(struct hw_perf_counter *, int);
  47         void            (*disable)(struct hw_perf_counter *, int);
  48         unsigned        eventsel;
  49         unsigned        perfctr;
  50         u64             (*event_map)(int);
  51         u64             (*raw_event)(u64);
  52         int             max_events;
  53         int             num_counters;
  54         int             num_counters_fixed;
  55         int             counter_bits;
  56         u64             counter_mask;
  57         u64             max_period;
  58         u64             intel_ctrl;
  59 };
  60
  61 static struct x86_pmu x86_pmu __read_mostly;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  73   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  74   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  75   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  76   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  77   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  78   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  79 };
  80
  81 static u64 intel_pmu_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 /*
  87  * Generalized hw caching related event table, filled
  88  * in on a per model basis. A value of 0 means
  89  * 'not supported', -1 means 'event makes no sense on
  90  * this CPU', any other value means the raw event
  91  * ID.
  92  */
  93
  94 #define C(x) PERF_COUNT_HW_CACHE_##x
  95
  96 static u64 __read_mostly hw_cache_event_ids
  97                                 [PERF_COUNT_HW_CACHE_MAX]
  98                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  99                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 100
 101 static const u64 nehalem_hw_cache_event_ids
 102                                 [PERF_COUNT_HW_CACHE_MAX]
 103                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 104                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 105 {
 106  [ C(L1D) ] = {
 107         [ C(OP_READ) ] = {
 108                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 109                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 110         },
 111         [ C(OP_WRITE) ] = {
 112                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 113                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 114         },
 115         [ C(OP_PREFETCH) ] = {
 116                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 117                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 118         },
 119  },
 120  [ C(L1I ) ] = {
 121         [ C(OP_READ) ] = {
 122                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 123                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 124         },
 125         [ C(OP_WRITE) ] = {
 126                 [ C(RESULT_ACCESS) ] = -1,
 127                 [ C(RESULT_MISS)   ] = -1,
 128         },
 129         [ C(OP_PREFETCH) ] = {
 130                 [ C(RESULT_ACCESS) ] = 0x0,
 131                 [ C(RESULT_MISS)   ] = 0x0,
 132         },
 133  },
 134  [ C(L2  ) ] = {
 135         [ C(OP_READ) ] = {
 136                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 137                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 138         },
 139         [ C(OP_WRITE) ] = {
 140                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 141                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 142         },
 143         [ C(OP_PREFETCH) ] = {
 144                 [ C(RESULT_ACCESS) ] = 0xc024, /* L2_RQSTS.PREFETCHES          */
 145                 [ C(RESULT_MISS)   ] = 0x8024, /* L2_RQSTS.PREFETCH_MISS       */
 146         },
 147  },
 148  [ C(DTLB) ] = {
 149         [ C(OP_READ) ] = {
 150                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 151                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 152         },
 153         [ C(OP_WRITE) ] = {
 154                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 155                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 156         },
 157         [ C(OP_PREFETCH) ] = {
 158                 [ C(RESULT_ACCESS) ] = 0x0,
 159                 [ C(RESULT_MISS)   ] = 0x0,
 160         },
 161  },
 162  [ C(ITLB) ] = {
 163         [ C(OP_READ) ] = {
 164                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 165                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 166         },
 167         [ C(OP_WRITE) ] = {
 168                 [ C(RESULT_ACCESS) ] = -1,
 169                 [ C(RESULT_MISS)   ] = -1,
 170         },
 171         [ C(OP_PREFETCH) ] = {
 172                 [ C(RESULT_ACCESS) ] = -1,
 173                 [ C(RESULT_MISS)   ] = -1,
 174         },
 175  },
 176  [ C(BPU ) ] = {
 177         [ C(OP_READ) ] = {
 178                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 179                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 180         },
 181         [ C(OP_WRITE) ] = {
 182                 [ C(RESULT_ACCESS) ] = -1,
 183                 [ C(RESULT_MISS)   ] = -1,
 184         },
 185         [ C(OP_PREFETCH) ] = {
 186                 [ C(RESULT_ACCESS) ] = -1,
 187                 [ C(RESULT_MISS)   ] = -1,
 188         },
 189  },
 190 };
 191
 192 static const u64 core2_hw_cache_event_ids
 193                                 [PERF_COUNT_HW_CACHE_MAX]
 194                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 195                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 196 {
 197  [ C(L1D) ] = {
 198         [ C(OP_READ) ] = {
 199                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 200                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 201         },
 202         [ C(OP_WRITE) ] = {
 203                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 204                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 205         },
 206         [ C(OP_PREFETCH) ] = {
 207                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 208                 [ C(RESULT_MISS)   ] = 0,
 209         },
 210  },
 211  [ C(L1I ) ] = {
 212         [ C(OP_READ) ] = {
 213                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 214                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 215         },
 216         [ C(OP_WRITE) ] = {
 217                 [ C(RESULT_ACCESS) ] = -1,
 218                 [ C(RESULT_MISS)   ] = -1,
 219         },
 220         [ C(OP_PREFETCH) ] = {
 221                 [ C(RESULT_ACCESS) ] = 0,
 222                 [ C(RESULT_MISS)   ] = 0,
 223         },
 224  },
 225  [ C(L2  ) ] = {
 226         [ C(OP_READ) ] = {
 227                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 228                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 229         },
 230         [ C(OP_WRITE) ] = {
 231                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 232                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 233         },
 234         [ C(OP_PREFETCH) ] = {
 235                 [ C(RESULT_ACCESS) ] = 0,
 236                 [ C(RESULT_MISS)   ] = 0,
 237         },
 238  },
 239  [ C(DTLB) ] = {
 240         [ C(OP_READ) ] = {
 241                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 242                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 243         },
 244         [ C(OP_WRITE) ] = {
 245                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 246                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 247         },
 248         [ C(OP_PREFETCH) ] = {
 249                 [ C(RESULT_ACCESS) ] = 0,
 250                 [ C(RESULT_MISS)   ] = 0,
 251         },
 252  },
 253  [ C(ITLB) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 256                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = -1,
 260                 [ C(RESULT_MISS)   ] = -1,
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = -1,
 264                 [ C(RESULT_MISS)   ] = -1,
 265         },
 266  },
 267  [ C(BPU ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 270                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = -1,
 278                 [ C(RESULT_MISS)   ] = -1,
 279         },
 280  },
 281 };
 282
 283 static const u64 atom_hw_cache_event_ids
 284                                 [PERF_COUNT_HW_CACHE_MAX]
 285                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 286                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 287 {
 288  [ C(L1D) ] = {
 289         [ C(OP_READ) ] = {
 290                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 291                 [ C(RESULT_MISS)   ] = 0,
 292         },
 293         [ C(OP_WRITE) ] = {
 294                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 295                 [ C(RESULT_MISS)   ] = 0,
 296         },
 297         [ C(OP_PREFETCH) ] = {
 298                 [ C(RESULT_ACCESS) ] = 0x0,
 299                 [ C(RESULT_MISS)   ] = 0,
 300         },
 301  },
 302  [ C(L1I ) ] = {
 303         [ C(OP_READ) ] = {
 304                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 305                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 306         },
 307         [ C(OP_WRITE) ] = {
 308                 [ C(RESULT_ACCESS) ] = -1,
 309                 [ C(RESULT_MISS)   ] = -1,
 310         },
 311         [ C(OP_PREFETCH) ] = {
 312                 [ C(RESULT_ACCESS) ] = 0,
 313                 [ C(RESULT_MISS)   ] = 0,
 314         },
 315  },
 316  [ C(L2  ) ] = {
 317         [ C(OP_READ) ] = {
 318                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 319                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 320         },
 321         [ C(OP_WRITE) ] = {
 322                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 323                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 324         },
 325         [ C(OP_PREFETCH) ] = {
 326                 [ C(RESULT_ACCESS) ] = 0,
 327                 [ C(RESULT_MISS)   ] = 0,
 328         },
 329  },
 330  [ C(DTLB) ] = {
 331         [ C(OP_READ) ] = {
 332                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 333                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 334         },
 335         [ C(OP_WRITE) ] = {
 336                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 337                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 338         },
 339         [ C(OP_PREFETCH) ] = {
 340                 [ C(RESULT_ACCESS) ] = 0,
 341                 [ C(RESULT_MISS)   ] = 0,
 342         },
 343  },
 344  [ C(ITLB) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 347                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = -1,
 351                 [ C(RESULT_MISS)   ] = -1,
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = -1,
 355                 [ C(RESULT_MISS)   ] = -1,
 356         },
 357  },
 358  [ C(BPU ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 361                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = -1,
 369                 [ C(RESULT_MISS)   ] = -1,
 370         },
 371  },
 372 };
 373
 374 static u64 intel_pmu_raw_event(u64 event)
 375 {
 376 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 377 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 378 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 379 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 380 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
 381
 382 #define CORE_EVNTSEL_MASK               \
 383         (CORE_EVNTSEL_EVENT_MASK |      \
 384          CORE_EVNTSEL_UNIT_MASK  |      \
 385          CORE_EVNTSEL_EDGE_MASK  |      \
 386          CORE_EVNTSEL_INV_MASK  |       \
 387          CORE_EVNTSEL_COUNTER_MASK)
 388
 389         return event & CORE_EVNTSEL_MASK;
 390 }
 391
 392 static const u64 amd_0f_hw_cache_event_ids
 393                                 [PERF_COUNT_HW_CACHE_MAX]
 394                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 395                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 396 {
 397  [ C(L1D) ] = {
 398         [ C(OP_READ) ] = {
 399                 [ C(RESULT_ACCESS) ] = 0,
 400                 [ C(RESULT_MISS)   ] = 0,
 401         },
 402         [ C(OP_WRITE) ] = {
 403                 [ C(RESULT_ACCESS) ] = 0,
 404                 [ C(RESULT_MISS)   ] = 0,
 405         },
 406         [ C(OP_PREFETCH) ] = {
 407                 [ C(RESULT_ACCESS) ] = 0,
 408                 [ C(RESULT_MISS)   ] = 0,
 409         },
 410  },
 411  [ C(L1I ) ] = {
 412         [ C(OP_READ) ] = {
 413                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 414                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 415         },
 416         [ C(OP_WRITE) ] = {
 417                 [ C(RESULT_ACCESS) ] = -1,
 418                 [ C(RESULT_MISS)   ] = -1,
 419         },
 420         [ C(OP_PREFETCH) ] = {
 421                 [ C(RESULT_ACCESS) ] = 0,
 422                 [ C(RESULT_MISS)   ] = 0,
 423         },
 424  },
 425  [ C(L2  ) ] = {
 426         [ C(OP_READ) ] = {
 427                 [ C(RESULT_ACCESS) ] = 0,
 428                 [ C(RESULT_MISS)   ] = 0,
 429         },
 430         [ C(OP_WRITE) ] = {
 431                 [ C(RESULT_ACCESS) ] = 0,
 432                 [ C(RESULT_MISS)   ] = 0,
 433         },
 434         [ C(OP_PREFETCH) ] = {
 435                 [ C(RESULT_ACCESS) ] = 0,
 436                 [ C(RESULT_MISS)   ] = 0,
 437         },
 438  },
 439  [ C(DTLB) ] = {
 440         [ C(OP_READ) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0,
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_WRITE) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448         [ C(OP_PREFETCH) ] = {
 449                 [ C(RESULT_ACCESS) ] = 0,
 450                 [ C(RESULT_MISS)   ] = 0,
 451         },
 452  },
 453  [ C(ITLB) ] = {
 454         [ C(OP_READ) ] = {
 455                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 456                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 457         },
 458         [ C(OP_WRITE) ] = {
 459                 [ C(RESULT_ACCESS) ] = -1,
 460                 [ C(RESULT_MISS)   ] = -1,
 461         },
 462         [ C(OP_PREFETCH) ] = {
 463                 [ C(RESULT_ACCESS) ] = -1,
 464                 [ C(RESULT_MISS)   ] = -1,
 465         },
 466  },
 467  [ C(BPU ) ] = {
 468         [ C(OP_READ) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 470                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 471         },
 472         [ C(OP_WRITE) ] = {
 473                 [ C(RESULT_ACCESS) ] = -1,
 474                 [ C(RESULT_MISS)   ] = -1,
 475         },
 476         [ C(OP_PREFETCH) ] = {
 477                 [ C(RESULT_ACCESS) ] = -1,
 478                 [ C(RESULT_MISS)   ] = -1,
 479         },
 480  },
 481 };
 482
 483 /*
 484  * AMD Performance Monitor K7 and later.
 485  */
 486 static const u64 amd_perfmon_event_map[] =
 487 {
 488   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 489   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 490   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 491   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 492   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 493   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 494 };
 495
 496 static u64 amd_pmu_event_map(int event)
 497 {
 498         return amd_perfmon_event_map[event];
 499 }
 500
 501 static u64 amd_pmu_raw_event(u64 event)
 502 {
 503 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 504 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 505 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 506 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 507 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 508
 509 #define K7_EVNTSEL_MASK                 \
 510         (K7_EVNTSEL_EVENT_MASK |        \
 511          K7_EVNTSEL_UNIT_MASK  |        \
 512          K7_EVNTSEL_EDGE_MASK  |        \
 513          K7_EVNTSEL_INV_MASK   |        \
 514          K7_EVNTSEL_COUNTER_MASK)
 515
 516         return event & K7_EVNTSEL_MASK;
 517 }
 518
 519 /*
 520  * Propagate counter elapsed time into the generic counter.
 521  * Can only be executed on the CPU where the counter is active.
 522  * Returns the delta events processed.
 523  */
 524 static u64
 525 x86_perf_counter_update(struct perf_counter *counter,
 526                         struct hw_perf_counter *hwc, int idx)
 527 {
 528         int shift = 64 - x86_pmu.counter_bits;
 529         u64 prev_raw_count, new_raw_count;
 530         s64 delta;
 531
 532         /*
 533          * Careful: an NMI might modify the previous counter value.
 534          *
 535          * Our tactic to handle this is to first atomically read and
 536          * exchange a new raw count - then add that new-prev delta
 537          * count to the generic counter atomically:
 538          */
 539 again:
 540         prev_raw_count = atomic64_read(&hwc->prev_count);
 541         rdmsrl(hwc->counter_base + idx, new_raw_count);
 542
 543         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 544                                         new_raw_count) != prev_raw_count)
 545                 goto again;
 546
 547         /*
 548          * Now we have the new raw value and have updated the prev
 549          * timestamp already. We can now calculate the elapsed delta
 550          * (counter-)time and add that to the generic counter.
 551          *
 552          * Careful, not all hw sign-extends above the physical width
 553          * of the count.
 554          */
 555         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 556         delta >>= shift;
 557
 558         atomic64_add(delta, &counter->count);
 559         atomic64_sub(delta, &hwc->period_left);
 560
 561         return new_raw_count;
 562 }
 563
 564 static atomic_t active_counters;
 565 static DEFINE_MUTEX(pmc_reserve_mutex);
 566
 567 static bool reserve_pmc_hardware(void)
 568 {
 569         int i;
 570
 571         if (nmi_watchdog == NMI_LOCAL_APIC)
 572                 disable_lapic_nmi_watchdog();
 573
 574         for (i = 0; i < x86_pmu.num_counters; i++) {
 575                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 576                         goto perfctr_fail;
 577         }
 578
 579         for (i = 0; i < x86_pmu.num_counters; i++) {
 580                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 581                         goto eventsel_fail;
 582         }
 583
 584         return true;
 585
 586 eventsel_fail:
 587         for (i--; i >= 0; i--)
 588                 release_evntsel_nmi(x86_pmu.eventsel + i);
 589
 590         i = x86_pmu.num_counters;
 591
 592 perfctr_fail:
 593         for (i--; i >= 0; i--)
 594                 release_perfctr_nmi(x86_pmu.perfctr + i);
 595
 596         if (nmi_watchdog == NMI_LOCAL_APIC)
 597                 enable_lapic_nmi_watchdog();
 598
 599         return false;
 600 }
 601
 602 static void release_pmc_hardware(void)
 603 {
 604         int i;
 605
 606         for (i = 0; i < x86_pmu.num_counters; i++) {
 607                 release_perfctr_nmi(x86_pmu.perfctr + i);
 608                 release_evntsel_nmi(x86_pmu.eventsel + i);
 609         }
 610
 611         if (nmi_watchdog == NMI_LOCAL_APIC)
 612                 enable_lapic_nmi_watchdog();
 613 }
 614
 615 static void hw_perf_counter_destroy(struct perf_counter *counter)
 616 {
 617         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
 618                 release_pmc_hardware();
 619                 mutex_unlock(&pmc_reserve_mutex);
 620         }
 621 }
 622
 623 static inline int x86_pmu_initialized(void)
 624 {
 625         return x86_pmu.handle_irq != NULL;
 626 }
 627
 628 static inline int
 629 set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
 630 {
 631         unsigned int cache_type, cache_op, cache_result;
 632         u64 config, val;
 633
 634         config = attr->config;
 635
 636         cache_type = (config >>  0) & 0xff;
 637         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 638                 return -EINVAL;
 639
 640         cache_op = (config >>  8) & 0xff;
 641         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 642                 return -EINVAL;
 643
 644         cache_result = (config >> 16) & 0xff;
 645         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 646                 return -EINVAL;
 647
 648         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 649
 650         if (val == 0)
 651                 return -ENOENT;
 652
 653         if (val == -1)
 654                 return -EINVAL;
 655
 656         hwc->config |= val;
 657
 658         return 0;
 659 }
 660
 661 /*
 662  * Setup the hardware configuration for a given attr_type
 663  */
 664 static int __hw_perf_counter_init(struct perf_counter *counter)
 665 {
 666         struct perf_counter_attr *attr = &counter->attr;
 667         struct hw_perf_counter *hwc = &counter->hw;
 668         int err;
 669
 670         if (!x86_pmu_initialized())
 671                 return -ENODEV;
 672
 673         err = 0;
 674         if (!atomic_inc_not_zero(&active_counters)) {
 675                 mutex_lock(&pmc_reserve_mutex);
 676                 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
 677                         err = -EBUSY;
 678                 else
 679                         atomic_inc(&active_counters);
 680                 mutex_unlock(&pmc_reserve_mutex);
 681         }
 682         if (err)
 683                 return err;
 684
 685         /*
 686          * Generate PMC IRQs:
 687          * (keep 'enabled' bit clear for now)
 688          */
 689         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 690
 691         /*
 692          * Count user and OS events unless requested not to.
 693          */
 694         if (!attr->exclude_user)
 695                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 696         if (!attr->exclude_kernel)
 697                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 698
 699         if (!hwc->sample_period) {
 700                 hwc->sample_period = x86_pmu.max_period;
 701                 atomic64_set(&hwc->period_left, hwc->sample_period);
 702         }
 703
 704         counter->destroy = hw_perf_counter_destroy;
 705
 706         /*
 707          * Raw event type provide the config in the event structure
 708          */
 709         if (attr->type == PERF_TYPE_RAW) {
 710                 hwc->config |= x86_pmu.raw_event(attr->config);
 711                 return 0;
 712         }
 713
 714         if (attr->type == PERF_TYPE_HW_CACHE)
 715                 return set_ext_hw_attr(hwc, attr);
 716
 717         if (attr->config >= x86_pmu.max_events)
 718                 return -EINVAL;
 719         /*
 720          * The generic map:
 721          */
 722         hwc->config |= x86_pmu.event_map(attr->config);
 723
 724         return 0;
 725 }
 726
 727 static void intel_pmu_disable_all(void)
 728 {
 729         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 730 }
 731
 732 static void amd_pmu_disable_all(void)
 733 {
 734         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 735         int idx;
 736
 737         if (!cpuc->enabled)
 738                 return;
 739
 740         cpuc->enabled = 0;
 741         /*
 742          * ensure we write the disable before we start disabling the
 743          * counters proper, so that amd_pmu_enable_counter() does the
 744          * right thing.
 745          */
 746         barrier();
 747
 748         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 749                 u64 val;
 750
 751                 if (!test_bit(idx, cpuc->active_mask))
 752                         continue;
 753                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 754                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 755                         continue;
 756                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 757                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 758         }
 759 }
 760
 761 void hw_perf_disable(void)
 762 {
 763         if (!x86_pmu_initialized())
 764                 return;
 765         return x86_pmu.disable_all();
 766 }
 767
 768 static void intel_pmu_enable_all(void)
 769 {
 770         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 771 }
 772
 773 static void amd_pmu_enable_all(void)
 774 {
 775         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 776         int idx;
 777
 778         if (cpuc->enabled)
 779                 return;
 780
 781         cpuc->enabled = 1;
 782         barrier();
 783
 784         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 785                 u64 val;
 786
 787                 if (!test_bit(idx, cpuc->active_mask))
 788                         continue;
 789                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 790                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 791                         continue;
 792                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 793                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 794         }
 795 }
 796
 797 void hw_perf_enable(void)
 798 {
 799         if (!x86_pmu_initialized())
 800                 return;
 801         x86_pmu.enable_all();
 802 }
 803
 804 static inline u64 intel_pmu_get_status(void)
 805 {
 806         u64 status;
 807
 808         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 809
 810         return status;
 811 }
 812
 813 static inline void intel_pmu_ack_status(u64 ack)
 814 {
 815         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 816 }
 817
 818 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 819 {
 820         int err;
 821         err = checking_wrmsrl(hwc->config_base + idx,
 822                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 823 }
 824
 825 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 826 {
 827         int err;
 828         err = checking_wrmsrl(hwc->config_base + idx,
 829                               hwc->config);
 830 }
 831
 832 static inline void
 833 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 834 {
 835         int idx = __idx - X86_PMC_IDX_FIXED;
 836         u64 ctrl_val, mask;
 837         int err;
 838
 839         mask = 0xfULL << (idx * 4);
 840
 841         rdmsrl(hwc->config_base, ctrl_val);
 842         ctrl_val &= ~mask;
 843         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 844 }
 845
 846 static inline void
 847 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 848 {
 849         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 850                 intel_pmu_disable_fixed(hwc, idx);
 851                 return;
 852         }
 853
 854         x86_pmu_disable_counter(hwc, idx);
 855 }
 856
 857 static inline void
 858 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 859 {
 860         x86_pmu_disable_counter(hwc, idx);
 861 }
 862
 863 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 864
 865 /*
 866  * Set the next IRQ period, based on the hwc->period_left value.
 867  * To be called with the counter disabled in hw:
 868  */
 869 static int
 870 x86_perf_counter_set_period(struct perf_counter *counter,
 871                              struct hw_perf_counter *hwc, int idx)
 872 {
 873         s64 left = atomic64_read(&hwc->period_left);
 874         s64 period = hwc->sample_period;
 875         int err, ret = 0;
 876
 877         /*
 878          * If we are way outside a reasoable range then just skip forward:
 879          */
 880         if (unlikely(left <= -period)) {
 881                 left = period;
 882                 atomic64_set(&hwc->period_left, left);
 883                 ret = 1;
 884         }
 885
 886         if (unlikely(left <= 0)) {
 887                 left += period;
 888                 atomic64_set(&hwc->period_left, left);
 889                 ret = 1;
 890         }
 891         /*
 892          * Quirk: certain CPUs dont like it if just 1 event is left:
 893          */
 894         if (unlikely(left < 2))
 895                 left = 2;
 896
 897         if (left > x86_pmu.max_period)
 898                 left = x86_pmu.max_period;
 899
 900         per_cpu(prev_left[idx], smp_processor_id()) = left;
 901
 902         /*
 903          * The hw counter starts counting from this counter offset,
 904          * mark it to be able to extra future deltas:
 905          */
 906         atomic64_set(&hwc->prev_count, (u64)-left);
 907
 908         err = checking_wrmsrl(hwc->counter_base + idx,
 909                              (u64)(-left) & x86_pmu.counter_mask);
 910
 911         return ret;
 912 }
 913
 914 static inline void
 915 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
 916 {
 917         int idx = __idx - X86_PMC_IDX_FIXED;
 918         u64 ctrl_val, bits, mask;
 919         int err;
 920
 921         /*
 922          * Enable IRQ generation (0x8),
 923          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 924          * if requested:
 925          */
 926         bits = 0x8ULL;
 927         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 928                 bits |= 0x2;
 929         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 930                 bits |= 0x1;
 931         bits <<= (idx * 4);
 932         mask = 0xfULL << (idx * 4);
 933
 934         rdmsrl(hwc->config_base, ctrl_val);
 935         ctrl_val &= ~mask;
 936         ctrl_val |= bits;
 937         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 938 }
 939
 940 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 941 {
 942         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 943                 intel_pmu_enable_fixed(hwc, idx);
 944                 return;
 945         }
 946
 947         x86_pmu_enable_counter(hwc, idx);
 948 }
 949
 950 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 951 {
 952         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 953
 954         if (cpuc->enabled)
 955                 x86_pmu_enable_counter(hwc, idx);
 956         else
 957                 x86_pmu_disable_counter(hwc, idx);
 958 }
 959
 960 static int
 961 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 962 {
 963         unsigned int event;
 964
 965         if (!x86_pmu.num_counters_fixed)
 966                 return -1;
 967
 968         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 969
 970         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
 971                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 972         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
 973                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 974         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
 975                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 976
 977         return -1;
 978 }
 979
 980 /*
 981  * Find a PMC slot for the freshly enabled / scheduled in counter:
 982  */
 983 static int x86_pmu_enable(struct perf_counter *counter)
 984 {
 985         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 986         struct hw_perf_counter *hwc = &counter->hw;
 987         int idx;
 988
 989         idx = fixed_mode_idx(counter, hwc);
 990         if (idx >= 0) {
 991                 /*
 992                  * Try to get the fixed counter, if that is already taken
 993                  * then try to get a generic counter:
 994                  */
 995                 if (test_and_set_bit(idx, cpuc->used_mask))
 996                         goto try_generic;
 997
 998                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 999                 /*
1000                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
1001                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1002                  */
1003                 hwc->counter_base =
1004                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1005                 hwc->idx = idx;
1006         } else {
1007                 idx = hwc->idx;
1008                 /* Try to get the previous generic counter again */
1009                 if (test_and_set_bit(idx, cpuc->used_mask)) {
1010 try_generic:
1011                         idx = find_first_zero_bit(cpuc->used_mask,
1012                                                   x86_pmu.num_counters);
1013                         if (idx == x86_pmu.num_counters)
1014                                 return -EAGAIN;
1015
1016                         set_bit(idx, cpuc->used_mask);
1017                         hwc->idx = idx;
1018                 }
1019                 hwc->config_base  = x86_pmu.eventsel;
1020                 hwc->counter_base = x86_pmu.perfctr;
1021         }
1022
1023         perf_counters_lapic_init();
1024
1025         x86_pmu.disable(hwc, idx);
1026
1027         cpuc->counters[idx] = counter;
1028         set_bit(idx, cpuc->active_mask);
1029
1030         x86_perf_counter_set_period(counter, hwc, idx);
1031         x86_pmu.enable(hwc, idx);
1032
1033         return 0;
1034 }
1035
1036 static void x86_pmu_unthrottle(struct perf_counter *counter)
1037 {
1038         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1039         struct hw_perf_counter *hwc = &counter->hw;
1040
1041         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1042                                 cpuc->counters[hwc->idx] != counter))
1043                 return;
1044
1045         x86_pmu.enable(hwc, hwc->idx);
1046 }
1047
1048 void perf_counter_print_debug(void)
1049 {
1050         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1051         struct cpu_hw_counters *cpuc;
1052         unsigned long flags;
1053         int cpu, idx;
1054
1055         if (!x86_pmu.num_counters)
1056                 return;
1057
1058         local_irq_save(flags);
1059
1060         cpu = smp_processor_id();
1061         cpuc = &per_cpu(cpu_hw_counters, cpu);
1062
1063         if (x86_pmu.version >= 2) {
1064                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1065                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1066                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1067                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1068
1069                 pr_info("\n");
1070                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1071                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1072                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1073                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1074         }
1075         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1076
1077         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1078                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1079                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1080
1081                 prev_left = per_cpu(prev_left[idx], cpu);
1082
1083                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1084                         cpu, idx, pmc_ctrl);
1085                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1086                         cpu, idx, pmc_count);
1087                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1088                         cpu, idx, prev_left);
1089         }
1090         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1091                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1092
1093                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1094                         cpu, idx, pmc_count);
1095         }
1096         local_irq_restore(flags);
1097 }
1098
1099 static void x86_pmu_disable(struct perf_counter *counter)
1100 {
1101         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1102         struct hw_perf_counter *hwc = &counter->hw;
1103         int idx = hwc->idx;
1104
1105         /*
1106          * Must be done before we disable, otherwise the nmi handler
1107          * could reenable again:
1108          */
1109         clear_bit(idx, cpuc->active_mask);
1110         x86_pmu.disable(hwc, idx);
1111
1112         /*
1113          * Make sure the cleared pointer becomes visible before we
1114          * (potentially) free the counter:
1115          */
1116         barrier();
1117
1118         /*
1119          * Drain the remaining delta count out of a counter
1120          * that we are disabling:
1121          */
1122         x86_perf_counter_update(counter, hwc, idx);
1123         cpuc->counters[idx] = NULL;
1124         clear_bit(idx, cpuc->used_mask);
1125 }
1126
1127 /*
1128  * Save and restart an expired counter. Called by NMI contexts,
1129  * so it has to be careful about preempting normal counter ops:
1130  */
1131 static int intel_pmu_save_and_restart(struct perf_counter *counter)
1132 {
1133         struct hw_perf_counter *hwc = &counter->hw;
1134         int idx = hwc->idx;
1135         int ret;
1136
1137         x86_perf_counter_update(counter, hwc, idx);
1138         ret = x86_perf_counter_set_period(counter, hwc, idx);
1139
1140         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1141                 intel_pmu_enable_counter(hwc, idx);
1142
1143         return ret;
1144 }
1145
1146 static void intel_pmu_reset(void)
1147 {
1148         unsigned long flags;
1149         int idx;
1150
1151         if (!x86_pmu.num_counters)
1152                 return;
1153
1154         local_irq_save(flags);
1155
1156         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1157
1158         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1159                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1160                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1161         }
1162         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1163                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1164         }
1165
1166         local_irq_restore(flags);
1167 }
1168
1169
1170 /*
1171  * This handler is triggered by the local APIC, so the APIC IRQ handling
1172  * rules apply:
1173  */
1174 static int intel_pmu_handle_irq(struct pt_regs *regs)
1175 {
1176         struct cpu_hw_counters *cpuc;
1177         struct cpu_hw_counters;
1178         int bit, cpu, loops;
1179         u64 ack, status;
1180
1181         cpu = smp_processor_id();
1182         cpuc = &per_cpu(cpu_hw_counters, cpu);
1183
1184         perf_disable();
1185         status = intel_pmu_get_status();
1186         if (!status) {
1187                 perf_enable();
1188                 return 0;
1189         }
1190
1191         loops = 0;
1192 again:
1193         if (++loops > 100) {
1194                 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
1195                 perf_counter_print_debug();
1196                 intel_pmu_reset();
1197                 perf_enable();
1198                 return 1;
1199         }
1200
1201         inc_irq_stat(apic_perf_irqs);
1202         ack = status;
1203         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1204                 struct perf_counter *counter = cpuc->counters[bit];
1205
1206                 clear_bit(bit, (unsigned long *) &status);
1207                 if (!test_bit(bit, cpuc->active_mask))
1208                         continue;
1209
1210                 if (!intel_pmu_save_and_restart(counter))
1211                         continue;
1212
1213                 if (perf_counter_overflow(counter, 1, regs, 0))
1214                         intel_pmu_disable_counter(&counter->hw, bit);
1215         }
1216
1217         intel_pmu_ack_status(ack);
1218
1219         /*
1220          * Repeat if there is more work to be done:
1221          */
1222         status = intel_pmu_get_status();
1223         if (status)
1224                 goto again;
1225
1226         perf_enable();
1227
1228         return 1;
1229 }
1230
1231 static int amd_pmu_handle_irq(struct pt_regs *regs)
1232 {
1233         int cpu, idx, handled = 0;
1234         struct cpu_hw_counters *cpuc;
1235         struct perf_counter *counter;
1236         struct hw_perf_counter *hwc;
1237         u64 val;
1238
1239         cpu = smp_processor_id();
1240         cpuc = &per_cpu(cpu_hw_counters, cpu);
1241
1242         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1243                 if (!test_bit(idx, cpuc->active_mask))
1244                         continue;
1245
1246                 counter = cpuc->counters[idx];
1247                 hwc = &counter->hw;
1248
1249                 val = x86_perf_counter_update(counter, hwc, idx);
1250                 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
1251                         continue;
1252
1253                 /* counter overflow */
1254                 handled = 1;
1255                 inc_irq_stat(apic_perf_irqs);
1256                 if (!x86_perf_counter_set_period(counter, hwc, idx))
1257                         continue;
1258
1259                 if (perf_counter_overflow(counter, 1, regs, 0))
1260                         amd_pmu_disable_counter(hwc, idx);
1261         }
1262
1263         return handled;
1264 }
1265
1266 void smp_perf_pending_interrupt(struct pt_regs *regs)
1267 {
1268         irq_enter();
1269         ack_APIC_irq();
1270         inc_irq_stat(apic_pending_irqs);
1271         perf_counter_do_pending();
1272         irq_exit();
1273 }
1274
1275 void set_perf_counter_pending(void)
1276 {
1277         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1278 }
1279
1280 void perf_counters_lapic_init(void)
1281 {
1282         if (!x86_pmu_initialized())
1283                 return;
1284
1285         /*
1286          * Always use NMI for PMU
1287          */
1288         apic_write(APIC_LVTPC, APIC_DM_NMI);
1289 }
1290
1291 static int __kprobes
1292 perf_counter_nmi_handler(struct notifier_block *self,
1293                          unsigned long cmd, void *__args)
1294 {
1295         struct die_args *args = __args;
1296         struct pt_regs *regs;
1297
1298         if (!atomic_read(&active_counters))
1299                 return NOTIFY_DONE;
1300
1301         switch (cmd) {
1302         case DIE_NMI:
1303         case DIE_NMI_IPI:
1304                 break;
1305
1306         default:
1307                 return NOTIFY_DONE;
1308         }
1309
1310         regs = args->regs;
1311
1312         apic_write(APIC_LVTPC, APIC_DM_NMI);
1313         /*
1314          * Can't rely on the handled return value to say it was our NMI, two
1315          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
1316          *
1317          * If the first NMI handles both, the latter will be empty and daze
1318          * the CPU.
1319          */
1320         x86_pmu.handle_irq(regs);
1321
1322         return NOTIFY_STOP;
1323 }
1324
1325 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
1326         .notifier_call          = perf_counter_nmi_handler,
1327         .next                   = NULL,
1328         .priority               = 1
1329 };
1330
1331 static struct x86_pmu intel_pmu = {
1332         .name                   = "Intel",
1333         .handle_irq             = intel_pmu_handle_irq,
1334         .disable_all            = intel_pmu_disable_all,
1335         .enable_all             = intel_pmu_enable_all,
1336         .enable                 = intel_pmu_enable_counter,
1337         .disable                = intel_pmu_disable_counter,
1338         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1339         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1340         .event_map              = intel_pmu_event_map,
1341         .raw_event              = intel_pmu_raw_event,
1342         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1343         /*
1344          * Intel PMCs cannot be accessed sanely above 32 bit width,
1345          * so we install an artificial 1<<31 period regardless of
1346          * the generic counter period:
1347          */
1348         .max_period             = (1ULL << 31) - 1,
1349 };
1350
1351 static struct x86_pmu amd_pmu = {
1352         .name                   = "AMD",
1353         .handle_irq             = amd_pmu_handle_irq,
1354         .disable_all            = amd_pmu_disable_all,
1355         .enable_all             = amd_pmu_enable_all,
1356         .enable                 = amd_pmu_enable_counter,
1357         .disable                = amd_pmu_disable_counter,
1358         .eventsel               = MSR_K7_EVNTSEL0,
1359         .perfctr                = MSR_K7_PERFCTR0,
1360         .event_map              = amd_pmu_event_map,
1361         .raw_event              = amd_pmu_raw_event,
1362         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1363         .num_counters           = 4,
1364         .counter_bits           = 48,
1365         .counter_mask           = (1ULL << 48) - 1,
1366         /* use highest bit to detect overflow */
1367         .max_period             = (1ULL << 47) - 1,
1368 };
1369
1370 static int intel_pmu_init(void)
1371 {
1372         union cpuid10_edx edx;
1373         union cpuid10_eax eax;
1374         unsigned int unused;
1375         unsigned int ebx;
1376         int version;
1377
1378         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
1379                 return -ENODEV;
1380
1381         /*
1382          * Check whether the Architectural PerfMon supports
1383          * Branch Misses Retired Event or not.
1384          */
1385         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
1386         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
1387                 return -ENODEV;
1388
1389         version = eax.split.version_id;
1390         if (version < 2)
1391                 return -ENODEV;
1392
1393         x86_pmu                         = intel_pmu;
1394         x86_pmu.version                 = version;
1395         x86_pmu.num_counters            = eax.split.num_counters;
1396         x86_pmu.counter_bits            = eax.split.bit_width;
1397         x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
1398
1399         /*
1400          * Quirk: v2 perfmon does not report fixed-purpose counters, so
1401          * assume at least 3 counters:
1402          */
1403         x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
1404
1405         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1406
1407         /*
1408          * Install the hw-cache-events table:
1409          */
1410         switch (boot_cpu_data.x86_model) {
1411         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1412         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1413         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1414         case 29: /* six-core 45 nm xeon "Dunnington" */
1415                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
1416                        sizeof(hw_cache_event_ids));
1417
1418                 pr_cont("Core2 events, ");
1419                 break;
1420         default:
1421         case 26:
1422                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
1423                        sizeof(hw_cache_event_ids));
1424
1425                 pr_cont("Nehalem/Corei7 events, ");
1426                 break;
1427         case 28:
1428                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
1429                        sizeof(hw_cache_event_ids));
1430
1431                 pr_cont("Atom events, ");
1432                 break;
1433         }
1434         return 0;
1435 }
1436
1437 static int amd_pmu_init(void)
1438 {
1439         x86_pmu = amd_pmu;
1440
1441         switch (boot_cpu_data.x86) {
1442         case 0x0f:
1443         case 0x10:
1444         case 0x11:
1445                 memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
1446                        sizeof(hw_cache_event_ids));
1447
1448                 pr_cont("AMD Family 0f/10/11 events, ");
1449                 break;
1450         }
1451         return 0;
1452 }
1453
1454 void __init init_hw_perf_counters(void)
1455 {
1456         int err;
1457
1458         pr_info("Performance Counters: ");
1459
1460         switch (boot_cpu_data.x86_vendor) {
1461         case X86_VENDOR_INTEL:
1462                 err = intel_pmu_init();
1463                 break;
1464         case X86_VENDOR_AMD:
1465                 err = amd_pmu_init();
1466                 break;
1467         default:
1468                 return;
1469         }
1470         if (err != 0) {
1471                 pr_cont("no PMU driver, software counters only.\n");
1472                 return;
1473         }
1474
1475         pr_cont("%s PMU driver.\n", x86_pmu.name);
1476
1477         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1478                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1479                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1480                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1481         }
1482         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1483         perf_max_counters = x86_pmu.num_counters;
1484
1485         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1486                 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1487                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1488                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1489         }
1490
1491         perf_counter_mask |=
1492                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1493
1494         perf_counters_lapic_init();
1495         register_die_notifier(&perf_counter_nmi_notifier);
1496
1497         pr_info("... version:                 %d\n",     x86_pmu.version);
1498         pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
1499         pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
1500         pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
1501         pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
1502         pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
1503         pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
1504 }
1505
1506 static inline void x86_pmu_read(struct perf_counter *counter)
1507 {
1508         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1509 }
1510
1511 static const struct pmu pmu = {
1512         .enable         = x86_pmu_enable,
1513         .disable        = x86_pmu_disable,
1514         .read           = x86_pmu_read,
1515         .unthrottle     = x86_pmu_unthrottle,
1516 };
1517
1518 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1519 {
1520         int err;
1521
1522         err = __hw_perf_counter_init(counter);
1523         if (err)
1524                 return ERR_PTR(err);
1525
1526         return &pmu;
1527 }
1528
1529 /*
1530  * callchain support
1531  */
1532
1533 static inline
1534 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1535 {
1536         if (entry->nr < MAX_STACK_DEPTH)
1537                 entry->ip[entry->nr++] = ip;
1538 }
1539
1540 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1541 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1542
1543
1544 static void
1545 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1546 {
1547         /* Ignore warnings */
1548 }
1549
1550 static void backtrace_warning(void *data, char *msg)
1551 {
1552         /* Ignore warnings */
1553 }
1554
1555 static int backtrace_stack(void *data, char *name)
1556 {
1557         /* Don't bother with IRQ stacks for now */
1558         return -1;
1559 }
1560
1561 static void backtrace_address(void *data, unsigned long addr, int reliable)
1562 {
1563         struct perf_callchain_entry *entry = data;
1564
1565         if (reliable)
1566                 callchain_store(entry, addr);
1567 }
1568
1569 static const struct stacktrace_ops backtrace_ops = {
1570         .warning                = backtrace_warning,
1571         .warning_symbol         = backtrace_warning_symbol,
1572         .stack                  = backtrace_stack,
1573         .address                = backtrace_address,
1574 };
1575
1576 static void
1577 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1578 {
1579         unsigned long bp;
1580         char *stack;
1581         int nr = entry->nr;
1582
1583         callchain_store(entry, instruction_pointer(regs));
1584
1585         stack = ((char *)regs + sizeof(struct pt_regs));
1586 #ifdef CONFIG_FRAME_POINTER
1587         bp = frame_pointer(regs);
1588 #else
1589         bp = 0;
1590 #endif
1591
1592         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1593
1594         entry->kernel = entry->nr - nr;
1595 }
1596
1597
1598 struct stack_frame {
1599         const void __user       *next_fp;
1600         unsigned long           return_address;
1601 };
1602
1603 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1604 {
1605         int ret;
1606
1607         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1608                 return 0;
1609
1610         ret = 1;
1611         pagefault_disable();
1612         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1613                 ret = 0;
1614         pagefault_enable();
1615
1616         return ret;
1617 }
1618
1619 static void
1620 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1621 {
1622         struct stack_frame frame;
1623         const void __user *fp;
1624         int nr = entry->nr;
1625
1626         regs = (struct pt_regs *)current->thread.sp0 - 1;
1627         fp   = (void __user *)regs->bp;
1628
1629         callchain_store(entry, regs->ip);
1630
1631         while (entry->nr < MAX_STACK_DEPTH) {
1632                 frame.next_fp        = NULL;
1633                 frame.return_address = 0;
1634
1635                 if (!copy_stack_frame(fp, &frame))
1636                         break;
1637
1638                 if ((unsigned long)fp < user_stack_pointer(regs))
1639                         break;
1640
1641                 callchain_store(entry, frame.return_address);
1642                 fp = frame.next_fp;
1643         }
1644
1645         entry->user = entry->nr - nr;
1646 }
1647
1648 static void
1649 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1650 {
1651         int is_user;
1652
1653         if (!regs)
1654                 return;
1655
1656         is_user = user_mode(regs);
1657
1658         if (!current || current->pid == 0)
1659                 return;
1660
1661         if (is_user && current->state != TASK_RUNNING)
1662                 return;
1663
1664         if (!is_user)
1665                 perf_callchain_kernel(regs, entry);
1666
1667         if (current->mm)
1668                 perf_callchain_user(regs, entry);
1669 }
1670
1671 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1672 {
1673         struct perf_callchain_entry *entry;
1674
1675         if (in_nmi())
1676                 entry = &__get_cpu_var(nmi_entry);
1677         else
1678                 entry = &__get_cpu_var(irq_entry);
1679
1680         entry->nr = 0;
1681         entry->hv = 0;
1682         entry->kernel = 0;
1683         entry->user = 0;
1684
1685         perf_do_callchain(regs, entry);
1686
1687         return entry;
1688 }