git.oblomov.eu Git - linux-2.6/blob - arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *
  10  *  For licencing details see kernel-base/COPYING
  11  */
  12
  13 #include <linux/perf_counter.h>
  14 #include <linux/capability.h>
  15 #include <linux/notifier.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/module.h>
  19 #include <linux/kdebug.h>
  20 #include <linux/sched.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <asm/apic.h>
  24 #include <asm/stacktrace.h>
  25 #include <asm/nmi.h>
  26
  27 static u64 perf_counter_mask __read_mostly;
  28
  29 struct cpu_hw_counters {
  30         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  31         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  32         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  33         unsigned long           interrupts;
  34         int                     enabled;
  35 };
  36
  37 /*
  38  * struct x86_pmu - generic x86 pmu
  39  */
  40 struct x86_pmu {
  41         const char      *name;
  42         int             version;
  43         int             (*handle_irq)(struct pt_regs *);
  44         void            (*disable_all)(void);
  45         void            (*enable_all)(void);
  46         void            (*enable)(struct hw_perf_counter *, int);
  47         void            (*disable)(struct hw_perf_counter *, int);
  48         unsigned        eventsel;
  49         unsigned        perfctr;
  50         u64             (*event_map)(int);
  51         u64             (*raw_event)(u64);
  52         int             max_events;
  53         int             num_counters;
  54         int             num_counters_fixed;
  55         int             counter_bits;
  56         u64             counter_mask;
  57         u64             max_period;
  58         u64             intel_ctrl;
  59 };
  60
  61 static struct x86_pmu x86_pmu __read_mostly;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  73   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  74   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  75   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  76   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  77   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  78   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  79 };
  80
  81 static u64 intel_pmu_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 /*
  87  * Generalized hw caching related event table, filled
  88  * in on a per model basis. A value of 0 means
  89  * 'not supported', -1 means 'event makes no sense on
  90  * this CPU', any other value means the raw event
  91  * ID.
  92  */
  93
  94 #define C(x) PERF_COUNT_HW_CACHE_##x
  95
  96 static u64 __read_mostly hw_cache_event_ids
  97                                 [PERF_COUNT_HW_CACHE_MAX]
  98                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  99                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 100
 101 static const u64 nehalem_hw_cache_event_ids
 102                                 [PERF_COUNT_HW_CACHE_MAX]
 103                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 104                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 105 {
 106  [ C(L1D) ] = {
 107         [ C(OP_READ) ] = {
 108                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 109                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 110         },
 111         [ C(OP_WRITE) ] = {
 112                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 113                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 114         },
 115         [ C(OP_PREFETCH) ] = {
 116                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 117                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 118         },
 119  },
 120  [ C(L1I ) ] = {
 121         [ C(OP_READ) ] = {
 122                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 123                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 124         },
 125         [ C(OP_WRITE) ] = {
 126                 [ C(RESULT_ACCESS) ] = -1,
 127                 [ C(RESULT_MISS)   ] = -1,
 128         },
 129         [ C(OP_PREFETCH) ] = {
 130                 [ C(RESULT_ACCESS) ] = 0x0,
 131                 [ C(RESULT_MISS)   ] = 0x0,
 132         },
 133  },
 134  [ C(L2  ) ] = {
 135         [ C(OP_READ) ] = {
 136                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 137                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 138         },
 139         [ C(OP_WRITE) ] = {
 140                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 141                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 142         },
 143         [ C(OP_PREFETCH) ] = {
 144                 [ C(RESULT_ACCESS) ] = 0xc024, /* L2_RQSTS.PREFETCHES          */
 145                 [ C(RESULT_MISS)   ] = 0x8024, /* L2_RQSTS.PREFETCH_MISS       */
 146         },
 147  },
 148  [ C(DTLB) ] = {
 149         [ C(OP_READ) ] = {
 150                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 151                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 152         },
 153         [ C(OP_WRITE) ] = {
 154                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 155                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 156         },
 157         [ C(OP_PREFETCH) ] = {
 158                 [ C(RESULT_ACCESS) ] = 0x0,
 159                 [ C(RESULT_MISS)   ] = 0x0,
 160         },
 161  },
 162  [ C(ITLB) ] = {
 163         [ C(OP_READ) ] = {
 164                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 165                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 166         },
 167         [ C(OP_WRITE) ] = {
 168                 [ C(RESULT_ACCESS) ] = -1,
 169                 [ C(RESULT_MISS)   ] = -1,
 170         },
 171         [ C(OP_PREFETCH) ] = {
 172                 [ C(RESULT_ACCESS) ] = -1,
 173                 [ C(RESULT_MISS)   ] = -1,
 174         },
 175  },
 176  [ C(BPU ) ] = {
 177         [ C(OP_READ) ] = {
 178                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 179                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 180         },
 181         [ C(OP_WRITE) ] = {
 182                 [ C(RESULT_ACCESS) ] = -1,
 183                 [ C(RESULT_MISS)   ] = -1,
 184         },
 185         [ C(OP_PREFETCH) ] = {
 186                 [ C(RESULT_ACCESS) ] = -1,
 187                 [ C(RESULT_MISS)   ] = -1,
 188         },
 189  },
 190 };
 191
 192 static const u64 core2_hw_cache_event_ids
 193                                 [PERF_COUNT_HW_CACHE_MAX]
 194                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 195                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 196 {
 197  [ C(L1D) ] = {
 198         [ C(OP_READ) ] = {
 199                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 200                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 201         },
 202         [ C(OP_WRITE) ] = {
 203                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 204                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 205         },
 206         [ C(OP_PREFETCH) ] = {
 207                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 208                 [ C(RESULT_MISS)   ] = 0,
 209         },
 210  },
 211  [ C(L1I ) ] = {
 212         [ C(OP_READ) ] = {
 213                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 214                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 215         },
 216         [ C(OP_WRITE) ] = {
 217                 [ C(RESULT_ACCESS) ] = -1,
 218                 [ C(RESULT_MISS)   ] = -1,
 219         },
 220         [ C(OP_PREFETCH) ] = {
 221                 [ C(RESULT_ACCESS) ] = 0,
 222                 [ C(RESULT_MISS)   ] = 0,
 223         },
 224  },
 225  [ C(L2  ) ] = {
 226         [ C(OP_READ) ] = {
 227                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 228                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 229         },
 230         [ C(OP_WRITE) ] = {
 231                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 232                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 233         },
 234         [ C(OP_PREFETCH) ] = {
 235                 [ C(RESULT_ACCESS) ] = 0,
 236                 [ C(RESULT_MISS)   ] = 0,
 237         },
 238  },
 239  [ C(DTLB) ] = {
 240         [ C(OP_READ) ] = {
 241                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 242                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 243         },
 244         [ C(OP_WRITE) ] = {
 245                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 246                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 247         },
 248         [ C(OP_PREFETCH) ] = {
 249                 [ C(RESULT_ACCESS) ] = 0,
 250                 [ C(RESULT_MISS)   ] = 0,
 251         },
 252  },
 253  [ C(ITLB) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 256                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = -1,
 260                 [ C(RESULT_MISS)   ] = -1,
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = -1,
 264                 [ C(RESULT_MISS)   ] = -1,
 265         },
 266  },
 267  [ C(BPU ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 270                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = -1,
 278                 [ C(RESULT_MISS)   ] = -1,
 279         },
 280  },
 281 };
 282
 283 static const u64 atom_hw_cache_event_ids
 284                                 [PERF_COUNT_HW_CACHE_MAX]
 285                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 286                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 287 {
 288  [ C(L1D) ] = {
 289         [ C(OP_READ) ] = {
 290                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 291                 [ C(RESULT_MISS)   ] = 0,
 292         },
 293         [ C(OP_WRITE) ] = {
 294                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 295                 [ C(RESULT_MISS)   ] = 0,
 296         },
 297         [ C(OP_PREFETCH) ] = {
 298                 [ C(RESULT_ACCESS) ] = 0x0,
 299                 [ C(RESULT_MISS)   ] = 0,
 300         },
 301  },
 302  [ C(L1I ) ] = {
 303         [ C(OP_READ) ] = {
 304                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 305                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 306         },
 307         [ C(OP_WRITE) ] = {
 308                 [ C(RESULT_ACCESS) ] = -1,
 309                 [ C(RESULT_MISS)   ] = -1,
 310         },
 311         [ C(OP_PREFETCH) ] = {
 312                 [ C(RESULT_ACCESS) ] = 0,
 313                 [ C(RESULT_MISS)   ] = 0,
 314         },
 315  },
 316  [ C(L2  ) ] = {
 317         [ C(OP_READ) ] = {
 318                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 319                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 320         },
 321         [ C(OP_WRITE) ] = {
 322                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 323                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 324         },
 325         [ C(OP_PREFETCH) ] = {
 326                 [ C(RESULT_ACCESS) ] = 0,
 327                 [ C(RESULT_MISS)   ] = 0,
 328         },
 329  },
 330  [ C(DTLB) ] = {
 331         [ C(OP_READ) ] = {
 332                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 333                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 334         },
 335         [ C(OP_WRITE) ] = {
 336                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 337                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 338         },
 339         [ C(OP_PREFETCH) ] = {
 340                 [ C(RESULT_ACCESS) ] = 0,
 341                 [ C(RESULT_MISS)   ] = 0,
 342         },
 343  },
 344  [ C(ITLB) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 347                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = -1,
 351                 [ C(RESULT_MISS)   ] = -1,
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = -1,
 355                 [ C(RESULT_MISS)   ] = -1,
 356         },
 357  },
 358  [ C(BPU ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 361                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = -1,
 369                 [ C(RESULT_MISS)   ] = -1,
 370         },
 371  },
 372 };
 373
 374 static u64 intel_pmu_raw_event(u64 event)
 375 {
 376 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 377 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 378 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 379 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 380 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
 381
 382 #define CORE_EVNTSEL_MASK               \
 383         (CORE_EVNTSEL_EVENT_MASK |      \
 384          CORE_EVNTSEL_UNIT_MASK  |      \
 385          CORE_EVNTSEL_EDGE_MASK  |      \
 386          CORE_EVNTSEL_INV_MASK  |       \
 387          CORE_EVNTSEL_COUNTER_MASK)
 388
 389         return event & CORE_EVNTSEL_MASK;
 390 }
 391
 392 static const u64 amd_0f_hw_cache_event_ids
 393                                 [PERF_COUNT_HW_CACHE_MAX]
 394                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 395                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 396 {
 397  [ C(L1D) ] = {
 398         [ C(OP_READ) ] = {
 399                 [ C(RESULT_ACCESS) ] = 0,
 400                 [ C(RESULT_MISS)   ] = 0,
 401         },
 402         [ C(OP_WRITE) ] = {
 403                 [ C(RESULT_ACCESS) ] = 0,
 404                 [ C(RESULT_MISS)   ] = 0,
 405         },
 406         [ C(OP_PREFETCH) ] = {
 407                 [ C(RESULT_ACCESS) ] = 0,
 408                 [ C(RESULT_MISS)   ] = 0,
 409         },
 410  },
 411  [ C(L1I ) ] = {
 412         [ C(OP_READ) ] = {
 413                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 414                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 415         },
 416         [ C(OP_WRITE) ] = {
 417                 [ C(RESULT_ACCESS) ] = -1,
 418                 [ C(RESULT_MISS)   ] = -1,
 419         },
 420         [ C(OP_PREFETCH) ] = {
 421                 [ C(RESULT_ACCESS) ] = 0,
 422                 [ C(RESULT_MISS)   ] = 0,
 423         },
 424  },
 425  [ C(L2  ) ] = {
 426         [ C(OP_READ) ] = {
 427                 [ C(RESULT_ACCESS) ] = 0,
 428                 [ C(RESULT_MISS)   ] = 0,
 429         },
 430         [ C(OP_WRITE) ] = {
 431                 [ C(RESULT_ACCESS) ] = 0,
 432                 [ C(RESULT_MISS)   ] = 0,
 433         },
 434         [ C(OP_PREFETCH) ] = {
 435                 [ C(RESULT_ACCESS) ] = 0,
 436                 [ C(RESULT_MISS)   ] = 0,
 437         },
 438  },
 439  [ C(DTLB) ] = {
 440         [ C(OP_READ) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0,
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_WRITE) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448         [ C(OP_PREFETCH) ] = {
 449                 [ C(RESULT_ACCESS) ] = 0,
 450                 [ C(RESULT_MISS)   ] = 0,
 451         },
 452  },
 453  [ C(ITLB) ] = {
 454         [ C(OP_READ) ] = {
 455                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 456                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 457         },
 458         [ C(OP_WRITE) ] = {
 459                 [ C(RESULT_ACCESS) ] = -1,
 460                 [ C(RESULT_MISS)   ] = -1,
 461         },
 462         [ C(OP_PREFETCH) ] = {
 463                 [ C(RESULT_ACCESS) ] = -1,
 464                 [ C(RESULT_MISS)   ] = -1,
 465         },
 466  },
 467  [ C(BPU ) ] = {
 468         [ C(OP_READ) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 470                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 471         },
 472         [ C(OP_WRITE) ] = {
 473                 [ C(RESULT_ACCESS) ] = -1,
 474                 [ C(RESULT_MISS)   ] = -1,
 475         },
 476         [ C(OP_PREFETCH) ] = {
 477                 [ C(RESULT_ACCESS) ] = -1,
 478                 [ C(RESULT_MISS)   ] = -1,
 479         },
 480  },
 481 };
 482
 483 /*
 484  * AMD Performance Monitor K7 and later.
 485  */
 486 static const u64 amd_perfmon_event_map[] =
 487 {
 488   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 489   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 490   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 491   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 492   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 493   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 494 };
 495
 496 static u64 amd_pmu_event_map(int event)
 497 {
 498         return amd_perfmon_event_map[event];
 499 }
 500
 501 static u64 amd_pmu_raw_event(u64 event)
 502 {
 503 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 504 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 505 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 506 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 507 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 508
 509 #define K7_EVNTSEL_MASK                 \
 510         (K7_EVNTSEL_EVENT_MASK |        \
 511          K7_EVNTSEL_UNIT_MASK  |        \
 512          K7_EVNTSEL_EDGE_MASK  |        \
 513          K7_EVNTSEL_INV_MASK   |        \
 514          K7_EVNTSEL_COUNTER_MASK)
 515
 516         return event & K7_EVNTSEL_MASK;
 517 }
 518
 519 /*
 520  * Propagate counter elapsed time into the generic counter.
 521  * Can only be executed on the CPU where the counter is active.
 522  * Returns the delta events processed.
 523  */
 524 static u64
 525 x86_perf_counter_update(struct perf_counter *counter,
 526                         struct hw_perf_counter *hwc, int idx)
 527 {
 528         int shift = 64 - x86_pmu.counter_bits;
 529         u64 prev_raw_count, new_raw_count;
 530         s64 delta;
 531
 532         /*
 533          * Careful: an NMI might modify the previous counter value.
 534          *
 535          * Our tactic to handle this is to first atomically read and
 536          * exchange a new raw count - then add that new-prev delta
 537          * count to the generic counter atomically:
 538          */
 539 again:
 540         prev_raw_count = atomic64_read(&hwc->prev_count);
 541         rdmsrl(hwc->counter_base + idx, new_raw_count);
 542
 543         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 544                                         new_raw_count) != prev_raw_count)
 545                 goto again;
 546
 547         /*
 548          * Now we have the new raw value and have updated the prev
 549          * timestamp already. We can now calculate the elapsed delta
 550          * (counter-)time and add that to the generic counter.
 551          *
 552          * Careful, not all hw sign-extends above the physical width
 553          * of the count.
 554          */
 555         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 556         delta >>= shift;
 557
 558         atomic64_add(delta, &counter->count);
 559         atomic64_sub(delta, &hwc->period_left);
 560
 561         return new_raw_count;
 562 }
 563
 564 static atomic_t active_counters;
 565 static DEFINE_MUTEX(pmc_reserve_mutex);
 566
 567 static bool reserve_pmc_hardware(void)
 568 {
 569         int i;
 570
 571         if (nmi_watchdog == NMI_LOCAL_APIC)
 572                 disable_lapic_nmi_watchdog();
 573
 574         for (i = 0; i < x86_pmu.num_counters; i++) {
 575                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 576                         goto perfctr_fail;
 577         }
 578
 579         for (i = 0; i < x86_pmu.num_counters; i++) {
 580                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 581                         goto eventsel_fail;
 582         }
 583
 584         return true;
 585
 586 eventsel_fail:
 587         for (i--; i >= 0; i--)
 588                 release_evntsel_nmi(x86_pmu.eventsel + i);
 589
 590         i = x86_pmu.num_counters;
 591
 592 perfctr_fail:
 593         for (i--; i >= 0; i--)
 594                 release_perfctr_nmi(x86_pmu.perfctr + i);
 595
 596         if (nmi_watchdog == NMI_LOCAL_APIC)
 597                 enable_lapic_nmi_watchdog();
 598
 599         return false;
 600 }
 601
 602 static void release_pmc_hardware(void)
 603 {
 604         int i;
 605
 606         for (i = 0; i < x86_pmu.num_counters; i++) {
 607                 release_perfctr_nmi(x86_pmu.perfctr + i);
 608                 release_evntsel_nmi(x86_pmu.eventsel + i);
 609         }
 610
 611         if (nmi_watchdog == NMI_LOCAL_APIC)
 612                 enable_lapic_nmi_watchdog();
 613 }
 614
 615 static void hw_perf_counter_destroy(struct perf_counter *counter)
 616 {
 617         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
 618                 release_pmc_hardware();
 619                 mutex_unlock(&pmc_reserve_mutex);
 620         }
 621 }
 622
 623 static inline int x86_pmu_initialized(void)
 624 {
 625         return x86_pmu.handle_irq != NULL;
 626 }
 627
 628 static inline int
 629 set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
 630 {
 631         unsigned int cache_type, cache_op, cache_result;
 632         u64 config, val;
 633
 634         config = attr->config;
 635
 636         cache_type = (config >>  0) & 0xff;
 637         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 638                 return -EINVAL;
 639
 640         cache_op = (config >>  8) & 0xff;
 641         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 642                 return -EINVAL;
 643
 644         cache_result = (config >> 16) & 0xff;
 645         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 646                 return -EINVAL;
 647
 648         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 649
 650         if (val == 0)
 651                 return -ENOENT;
 652
 653         if (val == -1)
 654                 return -EINVAL;
 655
 656         hwc->config |= val;
 657
 658         return 0;
 659 }
 660
 661 /*
 662  * Setup the hardware configuration for a given attr_type
 663  */
 664 static int __hw_perf_counter_init(struct perf_counter *counter)
 665 {
 666         struct perf_counter_attr *attr = &counter->attr;
 667         struct hw_perf_counter *hwc = &counter->hw;
 668         int err;
 669
 670         if (!x86_pmu_initialized())
 671                 return -ENODEV;
 672
 673         err = 0;
 674         if (!atomic_inc_not_zero(&active_counters)) {
 675                 mutex_lock(&pmc_reserve_mutex);
 676                 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
 677                         err = -EBUSY;
 678                 else
 679                         atomic_inc(&active_counters);
 680                 mutex_unlock(&pmc_reserve_mutex);
 681         }
 682         if (err)
 683                 return err;
 684
 685         /*
 686          * Generate PMC IRQs:
 687          * (keep 'enabled' bit clear for now)
 688          */
 689         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 690
 691         /*
 692          * Count user and OS events unless requested not to.
 693          */
 694         if (!attr->exclude_user)
 695                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 696         if (!attr->exclude_kernel)
 697                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 698
 699         if (!hwc->sample_period)
 700                 hwc->sample_period = x86_pmu.max_period;
 701
 702         atomic64_set(&hwc->period_left, hwc->sample_period);
 703         counter->destroy = hw_perf_counter_destroy;
 704
 705         /*
 706          * Raw event type provide the config in the event structure
 707          */
 708         if (attr->type == PERF_TYPE_RAW) {
 709                 hwc->config |= x86_pmu.raw_event(attr->config);
 710                 return 0;
 711         }
 712
 713         if (attr->type == PERF_TYPE_HW_CACHE)
 714                 return set_ext_hw_attr(hwc, attr);
 715
 716         if (attr->config >= x86_pmu.max_events)
 717                 return -EINVAL;
 718         /*
 719          * The generic map:
 720          */
 721         hwc->config |= x86_pmu.event_map(attr->config);
 722
 723         return 0;
 724 }
 725
 726 static void intel_pmu_disable_all(void)
 727 {
 728         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 729 }
 730
 731 static void amd_pmu_disable_all(void)
 732 {
 733         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 734         int idx;
 735
 736         if (!cpuc->enabled)
 737                 return;
 738
 739         cpuc->enabled = 0;
 740         /*
 741          * ensure we write the disable before we start disabling the
 742          * counters proper, so that amd_pmu_enable_counter() does the
 743          * right thing.
 744          */
 745         barrier();
 746
 747         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 748                 u64 val;
 749
 750                 if (!test_bit(idx, cpuc->active_mask))
 751                         continue;
 752                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 753                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 754                         continue;
 755                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 756                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 757         }
 758 }
 759
 760 void hw_perf_disable(void)
 761 {
 762         if (!x86_pmu_initialized())
 763                 return;
 764         return x86_pmu.disable_all();
 765 }
 766
 767 static void intel_pmu_enable_all(void)
 768 {
 769         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 770 }
 771
 772 static void amd_pmu_enable_all(void)
 773 {
 774         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 775         int idx;
 776
 777         if (cpuc->enabled)
 778                 return;
 779
 780         cpuc->enabled = 1;
 781         barrier();
 782
 783         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 784                 u64 val;
 785
 786                 if (!test_bit(idx, cpuc->active_mask))
 787                         continue;
 788                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 789                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 790                         continue;
 791                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 792                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 793         }
 794 }
 795
 796 void hw_perf_enable(void)
 797 {
 798         if (!x86_pmu_initialized())
 799                 return;
 800         x86_pmu.enable_all();
 801 }
 802
 803 static inline u64 intel_pmu_get_status(void)
 804 {
 805         u64 status;
 806
 807         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 808
 809         return status;
 810 }
 811
 812 static inline void intel_pmu_ack_status(u64 ack)
 813 {
 814         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 815 }
 816
 817 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 818 {
 819         int err;
 820         err = checking_wrmsrl(hwc->config_base + idx,
 821                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 822 }
 823
 824 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 825 {
 826         int err;
 827         err = checking_wrmsrl(hwc->config_base + idx,
 828                               hwc->config);
 829 }
 830
 831 static inline void
 832 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 833 {
 834         int idx = __idx - X86_PMC_IDX_FIXED;
 835         u64 ctrl_val, mask;
 836         int err;
 837
 838         mask = 0xfULL << (idx * 4);
 839
 840         rdmsrl(hwc->config_base, ctrl_val);
 841         ctrl_val &= ~mask;
 842         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 843 }
 844
 845 static inline void
 846 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 847 {
 848         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 849                 intel_pmu_disable_fixed(hwc, idx);
 850                 return;
 851         }
 852
 853         x86_pmu_disable_counter(hwc, idx);
 854 }
 855
 856 static inline void
 857 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 858 {
 859         x86_pmu_disable_counter(hwc, idx);
 860 }
 861
 862 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 863
 864 /*
 865  * Set the next IRQ period, based on the hwc->period_left value.
 866  * To be called with the counter disabled in hw:
 867  */
 868 static int
 869 x86_perf_counter_set_period(struct perf_counter *counter,
 870                              struct hw_perf_counter *hwc, int idx)
 871 {
 872         s64 left = atomic64_read(&hwc->period_left);
 873         s64 period = hwc->sample_period;
 874         int err, ret = 0;
 875
 876         /*
 877          * If we are way outside a reasoable range then just skip forward:
 878          */
 879         if (unlikely(left <= -period)) {
 880                 left = period;
 881                 atomic64_set(&hwc->period_left, left);
 882                 ret = 1;
 883         }
 884
 885         if (unlikely(left <= 0)) {
 886                 left += period;
 887                 atomic64_set(&hwc->period_left, left);
 888                 ret = 1;
 889         }
 890         /*
 891          * Quirk: certain CPUs dont like it if just 1 event is left:
 892          */
 893         if (unlikely(left < 2))
 894                 left = 2;
 895
 896         if (left > x86_pmu.max_period)
 897                 left = x86_pmu.max_period;
 898
 899         per_cpu(prev_left[idx], smp_processor_id()) = left;
 900
 901         /*
 902          * The hw counter starts counting from this counter offset,
 903          * mark it to be able to extra future deltas:
 904          */
 905         atomic64_set(&hwc->prev_count, (u64)-left);
 906
 907         err = checking_wrmsrl(hwc->counter_base + idx,
 908                              (u64)(-left) & x86_pmu.counter_mask);
 909
 910         return ret;
 911 }
 912
 913 static inline void
 914 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
 915 {
 916         int idx = __idx - X86_PMC_IDX_FIXED;
 917         u64 ctrl_val, bits, mask;
 918         int err;
 919
 920         /*
 921          * Enable IRQ generation (0x8),
 922          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 923          * if requested:
 924          */
 925         bits = 0x8ULL;
 926         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 927                 bits |= 0x2;
 928         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 929                 bits |= 0x1;
 930         bits <<= (idx * 4);
 931         mask = 0xfULL << (idx * 4);
 932
 933         rdmsrl(hwc->config_base, ctrl_val);
 934         ctrl_val &= ~mask;
 935         ctrl_val |= bits;
 936         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 937 }
 938
 939 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 940 {
 941         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 942                 intel_pmu_enable_fixed(hwc, idx);
 943                 return;
 944         }
 945
 946         x86_pmu_enable_counter(hwc, idx);
 947 }
 948
 949 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 950 {
 951         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 952
 953         if (cpuc->enabled)
 954                 x86_pmu_enable_counter(hwc, idx);
 955         else
 956                 x86_pmu_disable_counter(hwc, idx);
 957 }
 958
 959 static int
 960 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 961 {
 962         unsigned int event;
 963
 964         if (!x86_pmu.num_counters_fixed)
 965                 return -1;
 966
 967         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 968
 969         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
 970                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 971         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
 972                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 973         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
 974                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 975
 976         return -1;
 977 }
 978
 979 /*
 980  * Find a PMC slot for the freshly enabled / scheduled in counter:
 981  */
 982 static int x86_pmu_enable(struct perf_counter *counter)
 983 {
 984         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 985         struct hw_perf_counter *hwc = &counter->hw;
 986         int idx;
 987
 988         idx = fixed_mode_idx(counter, hwc);
 989         if (idx >= 0) {
 990                 /*
 991                  * Try to get the fixed counter, if that is already taken
 992                  * then try to get a generic counter:
 993                  */
 994                 if (test_and_set_bit(idx, cpuc->used_mask))
 995                         goto try_generic;
 996
 997                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 998                 /*
 999                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
1000                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1001                  */
1002                 hwc->counter_base =
1003                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1004                 hwc->idx = idx;
1005         } else {
1006                 idx = hwc->idx;
1007                 /* Try to get the previous generic counter again */
1008                 if (test_and_set_bit(idx, cpuc->used_mask)) {
1009 try_generic:
1010                         idx = find_first_zero_bit(cpuc->used_mask,
1011                                                   x86_pmu.num_counters);
1012                         if (idx == x86_pmu.num_counters)
1013                                 return -EAGAIN;
1014
1015                         set_bit(idx, cpuc->used_mask);
1016                         hwc->idx = idx;
1017                 }
1018                 hwc->config_base  = x86_pmu.eventsel;
1019                 hwc->counter_base = x86_pmu.perfctr;
1020         }
1021
1022         perf_counters_lapic_init();
1023
1024         x86_pmu.disable(hwc, idx);
1025
1026         cpuc->counters[idx] = counter;
1027         set_bit(idx, cpuc->active_mask);
1028
1029         x86_perf_counter_set_period(counter, hwc, idx);
1030         x86_pmu.enable(hwc, idx);
1031
1032         return 0;
1033 }
1034
1035 static void x86_pmu_unthrottle(struct perf_counter *counter)
1036 {
1037         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1038         struct hw_perf_counter *hwc = &counter->hw;
1039
1040         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1041                                 cpuc->counters[hwc->idx] != counter))
1042                 return;
1043
1044         x86_pmu.enable(hwc, hwc->idx);
1045 }
1046
1047 void perf_counter_print_debug(void)
1048 {
1049         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1050         struct cpu_hw_counters *cpuc;
1051         unsigned long flags;
1052         int cpu, idx;
1053
1054         if (!x86_pmu.num_counters)
1055                 return;
1056
1057         local_irq_save(flags);
1058
1059         cpu = smp_processor_id();
1060         cpuc = &per_cpu(cpu_hw_counters, cpu);
1061
1062         if (x86_pmu.version >= 2) {
1063                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1064                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1065                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1066                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1067
1068                 pr_info("\n");
1069                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1070                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1071                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1072                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1073         }
1074         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1075
1076         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1077                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1078                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1079
1080                 prev_left = per_cpu(prev_left[idx], cpu);
1081
1082                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1083                         cpu, idx, pmc_ctrl);
1084                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1085                         cpu, idx, pmc_count);
1086                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1087                         cpu, idx, prev_left);
1088         }
1089         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1090                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1091
1092                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1093                         cpu, idx, pmc_count);
1094         }
1095         local_irq_restore(flags);
1096 }
1097
1098 static void x86_pmu_disable(struct perf_counter *counter)
1099 {
1100         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1101         struct hw_perf_counter *hwc = &counter->hw;
1102         int idx = hwc->idx;
1103
1104         /*
1105          * Must be done before we disable, otherwise the nmi handler
1106          * could reenable again:
1107          */
1108         clear_bit(idx, cpuc->active_mask);
1109         x86_pmu.disable(hwc, idx);
1110
1111         /*
1112          * Make sure the cleared pointer becomes visible before we
1113          * (potentially) free the counter:
1114          */
1115         barrier();
1116
1117         /*
1118          * Drain the remaining delta count out of a counter
1119          * that we are disabling:
1120          */
1121         x86_perf_counter_update(counter, hwc, idx);
1122         cpuc->counters[idx] = NULL;
1123         clear_bit(idx, cpuc->used_mask);
1124 }
1125
1126 /*
1127  * Save and restart an expired counter. Called by NMI contexts,
1128  * so it has to be careful about preempting normal counter ops:
1129  */
1130 static int intel_pmu_save_and_restart(struct perf_counter *counter)
1131 {
1132         struct hw_perf_counter *hwc = &counter->hw;
1133         int idx = hwc->idx;
1134         int ret;
1135
1136         x86_perf_counter_update(counter, hwc, idx);
1137         ret = x86_perf_counter_set_period(counter, hwc, idx);
1138
1139         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1140                 intel_pmu_enable_counter(hwc, idx);
1141
1142         return ret;
1143 }
1144
1145 static void intel_pmu_reset(void)
1146 {
1147         unsigned long flags;
1148         int idx;
1149
1150         if (!x86_pmu.num_counters)
1151                 return;
1152
1153         local_irq_save(flags);
1154
1155         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1156
1157         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1158                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1159                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1160         }
1161         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1162                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1163         }
1164
1165         local_irq_restore(flags);
1166 }
1167
1168
1169 /*
1170  * This handler is triggered by the local APIC, so the APIC IRQ handling
1171  * rules apply:
1172  */
1173 static int intel_pmu_handle_irq(struct pt_regs *regs)
1174 {
1175         struct cpu_hw_counters *cpuc;
1176         struct cpu_hw_counters;
1177         int bit, cpu, loops;
1178         u64 ack, status;
1179
1180         cpu = smp_processor_id();
1181         cpuc = &per_cpu(cpu_hw_counters, cpu);
1182
1183         perf_disable();
1184         status = intel_pmu_get_status();
1185         if (!status) {
1186                 perf_enable();
1187                 return 0;
1188         }
1189
1190         loops = 0;
1191 again:
1192         if (++loops > 100) {
1193                 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
1194                 perf_counter_print_debug();
1195                 intel_pmu_reset();
1196                 perf_enable();
1197                 return 1;
1198         }
1199
1200         inc_irq_stat(apic_perf_irqs);
1201         ack = status;
1202         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1203                 struct perf_counter *counter = cpuc->counters[bit];
1204
1205                 clear_bit(bit, (unsigned long *) &status);
1206                 if (!test_bit(bit, cpuc->active_mask))
1207                         continue;
1208
1209                 if (!intel_pmu_save_and_restart(counter))
1210                         continue;
1211
1212                 if (perf_counter_overflow(counter, 1, regs, 0))
1213                         intel_pmu_disable_counter(&counter->hw, bit);
1214         }
1215
1216         intel_pmu_ack_status(ack);
1217
1218         /*
1219          * Repeat if there is more work to be done:
1220          */
1221         status = intel_pmu_get_status();
1222         if (status)
1223                 goto again;
1224
1225         perf_enable();
1226
1227         return 1;
1228 }
1229
1230 static int amd_pmu_handle_irq(struct pt_regs *regs)
1231 {
1232         int cpu, idx, handled = 0;
1233         struct cpu_hw_counters *cpuc;
1234         struct perf_counter *counter;
1235         struct hw_perf_counter *hwc;
1236         u64 val;
1237
1238         cpu = smp_processor_id();
1239         cpuc = &per_cpu(cpu_hw_counters, cpu);
1240
1241         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1242                 if (!test_bit(idx, cpuc->active_mask))
1243                         continue;
1244
1245                 counter = cpuc->counters[idx];
1246                 hwc = &counter->hw;
1247
1248                 val = x86_perf_counter_update(counter, hwc, idx);
1249                 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
1250                         continue;
1251
1252                 /* counter overflow */
1253                 handled = 1;
1254                 inc_irq_stat(apic_perf_irqs);
1255                 if (!x86_perf_counter_set_period(counter, hwc, idx))
1256                         continue;
1257
1258                 if (perf_counter_overflow(counter, 1, regs, 0))
1259                         amd_pmu_disable_counter(hwc, idx);
1260         }
1261
1262         return handled;
1263 }
1264
1265 void smp_perf_pending_interrupt(struct pt_regs *regs)
1266 {
1267         irq_enter();
1268         ack_APIC_irq();
1269         inc_irq_stat(apic_pending_irqs);
1270         perf_counter_do_pending();
1271         irq_exit();
1272 }
1273
1274 void set_perf_counter_pending(void)
1275 {
1276         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1277 }
1278
1279 void perf_counters_lapic_init(void)
1280 {
1281         if (!x86_pmu_initialized())
1282                 return;
1283
1284         /*
1285          * Always use NMI for PMU
1286          */
1287         apic_write(APIC_LVTPC, APIC_DM_NMI);
1288 }
1289
1290 static int __kprobes
1291 perf_counter_nmi_handler(struct notifier_block *self,
1292                          unsigned long cmd, void *__args)
1293 {
1294         struct die_args *args = __args;
1295         struct pt_regs *regs;
1296
1297         if (!atomic_read(&active_counters))
1298                 return NOTIFY_DONE;
1299
1300         switch (cmd) {
1301         case DIE_NMI:
1302         case DIE_NMI_IPI:
1303                 break;
1304
1305         default:
1306                 return NOTIFY_DONE;
1307         }
1308
1309         regs = args->regs;
1310
1311         apic_write(APIC_LVTPC, APIC_DM_NMI);
1312         /*
1313          * Can't rely on the handled return value to say it was our NMI, two
1314          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
1315          *
1316          * If the first NMI handles both, the latter will be empty and daze
1317          * the CPU.
1318          */
1319         x86_pmu.handle_irq(regs);
1320
1321         return NOTIFY_STOP;
1322 }
1323
1324 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
1325         .notifier_call          = perf_counter_nmi_handler,
1326         .next                   = NULL,
1327         .priority               = 1
1328 };
1329
1330 static struct x86_pmu intel_pmu = {
1331         .name                   = "Intel",
1332         .handle_irq             = intel_pmu_handle_irq,
1333         .disable_all            = intel_pmu_disable_all,
1334         .enable_all             = intel_pmu_enable_all,
1335         .enable                 = intel_pmu_enable_counter,
1336         .disable                = intel_pmu_disable_counter,
1337         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1338         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1339         .event_map              = intel_pmu_event_map,
1340         .raw_event              = intel_pmu_raw_event,
1341         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1342         /*
1343          * Intel PMCs cannot be accessed sanely above 32 bit width,
1344          * so we install an artificial 1<<31 period regardless of
1345          * the generic counter period:
1346          */
1347         .max_period             = (1ULL << 31) - 1,
1348 };
1349
1350 static struct x86_pmu amd_pmu = {
1351         .name                   = "AMD",
1352         .handle_irq             = amd_pmu_handle_irq,
1353         .disable_all            = amd_pmu_disable_all,
1354         .enable_all             = amd_pmu_enable_all,
1355         .enable                 = amd_pmu_enable_counter,
1356         .disable                = amd_pmu_disable_counter,
1357         .eventsel               = MSR_K7_EVNTSEL0,
1358         .perfctr                = MSR_K7_PERFCTR0,
1359         .event_map              = amd_pmu_event_map,
1360         .raw_event              = amd_pmu_raw_event,
1361         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1362         .num_counters           = 4,
1363         .counter_bits           = 48,
1364         .counter_mask           = (1ULL << 48) - 1,
1365         /* use highest bit to detect overflow */
1366         .max_period             = (1ULL << 47) - 1,
1367 };
1368
1369 static int intel_pmu_init(void)
1370 {
1371         union cpuid10_edx edx;
1372         union cpuid10_eax eax;
1373         unsigned int unused;
1374         unsigned int ebx;
1375         int version;
1376
1377         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
1378                 return -ENODEV;
1379
1380         /*
1381          * Check whether the Architectural PerfMon supports
1382          * Branch Misses Retired Event or not.
1383          */
1384         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
1385         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
1386                 return -ENODEV;
1387
1388         version = eax.split.version_id;
1389         if (version < 2)
1390                 return -ENODEV;
1391
1392         x86_pmu                         = intel_pmu;
1393         x86_pmu.version                 = version;
1394         x86_pmu.num_counters            = eax.split.num_counters;
1395         x86_pmu.counter_bits            = eax.split.bit_width;
1396         x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
1397
1398         /*
1399          * Quirk: v2 perfmon does not report fixed-purpose counters, so
1400          * assume at least 3 counters:
1401          */
1402         x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
1403
1404         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1405
1406         /*
1407          * Install the hw-cache-events table:
1408          */
1409         switch (boot_cpu_data.x86_model) {
1410         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1411         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1412         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1413         case 29: /* six-core 45 nm xeon "Dunnington" */
1414                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
1415                        sizeof(hw_cache_event_ids));
1416
1417                 pr_cont("Core2 events, ");
1418                 break;
1419         default:
1420         case 26:
1421                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
1422                        sizeof(hw_cache_event_ids));
1423
1424                 pr_cont("Nehalem/Corei7 events, ");
1425                 break;
1426         case 28:
1427                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
1428                        sizeof(hw_cache_event_ids));
1429
1430                 pr_cont("Atom events, ");
1431                 break;
1432         }
1433         return 0;
1434 }
1435
1436 static int amd_pmu_init(void)
1437 {
1438         x86_pmu = amd_pmu;
1439
1440         switch (boot_cpu_data.x86) {
1441         case 0x0f:
1442         case 0x10:
1443         case 0x11:
1444                 memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
1445                        sizeof(hw_cache_event_ids));
1446
1447                 pr_cont("AMD Family 0f/10/11 events, ");
1448                 break;
1449         }
1450         return 0;
1451 }
1452
1453 void __init init_hw_perf_counters(void)
1454 {
1455         int err;
1456
1457         pr_info("Performance Counters: ");
1458
1459         switch (boot_cpu_data.x86_vendor) {
1460         case X86_VENDOR_INTEL:
1461                 err = intel_pmu_init();
1462                 break;
1463         case X86_VENDOR_AMD:
1464                 err = amd_pmu_init();
1465                 break;
1466         default:
1467                 return;
1468         }
1469         if (err != 0) {
1470                 pr_cont("no PMU driver, software counters only.\n");
1471                 return;
1472         }
1473
1474         pr_cont("%s PMU driver.\n", x86_pmu.name);
1475
1476         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1477                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1478                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1479                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1480         }
1481         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1482         perf_max_counters = x86_pmu.num_counters;
1483
1484         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1485                 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1486                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1487                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1488         }
1489
1490         perf_counter_mask |=
1491                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1492
1493         perf_counters_lapic_init();
1494         register_die_notifier(&perf_counter_nmi_notifier);
1495
1496         pr_info("... version:                 %d\n",     x86_pmu.version);
1497         pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
1498         pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
1499         pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
1500         pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
1501         pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
1502         pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
1503 }
1504
1505 static inline void x86_pmu_read(struct perf_counter *counter)
1506 {
1507         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1508 }
1509
1510 static const struct pmu pmu = {
1511         .enable         = x86_pmu_enable,
1512         .disable        = x86_pmu_disable,
1513         .read           = x86_pmu_read,
1514         .unthrottle     = x86_pmu_unthrottle,
1515 };
1516
1517 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1518 {
1519         int err;
1520
1521         err = __hw_perf_counter_init(counter);
1522         if (err)
1523                 return ERR_PTR(err);
1524
1525         return &pmu;
1526 }
1527
1528 /*
1529  * callchain support
1530  */
1531
1532 static inline
1533 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1534 {
1535         if (entry->nr < MAX_STACK_DEPTH)
1536                 entry->ip[entry->nr++] = ip;
1537 }
1538
1539 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1540 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1541
1542
1543 static void
1544 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1545 {
1546         /* Ignore warnings */
1547 }
1548
1549 static void backtrace_warning(void *data, char *msg)
1550 {
1551         /* Ignore warnings */
1552 }
1553
1554 static int backtrace_stack(void *data, char *name)
1555 {
1556         /* Don't bother with IRQ stacks for now */
1557         return -1;
1558 }
1559
1560 static void backtrace_address(void *data, unsigned long addr, int reliable)
1561 {
1562         struct perf_callchain_entry *entry = data;
1563
1564         if (reliable)
1565                 callchain_store(entry, addr);
1566 }
1567
1568 static const struct stacktrace_ops backtrace_ops = {
1569         .warning                = backtrace_warning,
1570         .warning_symbol         = backtrace_warning_symbol,
1571         .stack                  = backtrace_stack,
1572         .address                = backtrace_address,
1573 };
1574
1575 static void
1576 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1577 {
1578         unsigned long bp;
1579         char *stack;
1580         int nr = entry->nr;
1581
1582         callchain_store(entry, instruction_pointer(regs));
1583
1584         stack = ((char *)regs + sizeof(struct pt_regs));
1585 #ifdef CONFIG_FRAME_POINTER
1586         bp = frame_pointer(regs);
1587 #else
1588         bp = 0;
1589 #endif
1590
1591         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1592
1593         entry->kernel = entry->nr - nr;
1594 }
1595
1596
1597 struct stack_frame {
1598         const void __user       *next_fp;
1599         unsigned long           return_address;
1600 };
1601
1602 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1603 {
1604         int ret;
1605
1606         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1607                 return 0;
1608
1609         ret = 1;
1610         pagefault_disable();
1611         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1612                 ret = 0;
1613         pagefault_enable();
1614
1615         return ret;
1616 }
1617
1618 static void
1619 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1620 {
1621         struct stack_frame frame;
1622         const void __user *fp;
1623         int nr = entry->nr;
1624
1625         regs = (struct pt_regs *)current->thread.sp0 - 1;
1626         fp   = (void __user *)regs->bp;
1627
1628         callchain_store(entry, regs->ip);
1629
1630         while (entry->nr < MAX_STACK_DEPTH) {
1631                 frame.next_fp        = NULL;
1632                 frame.return_address = 0;
1633
1634                 if (!copy_stack_frame(fp, &frame))
1635                         break;
1636
1637                 if ((unsigned long)fp < user_stack_pointer(regs))
1638                         break;
1639
1640                 callchain_store(entry, frame.return_address);
1641                 fp = frame.next_fp;
1642         }
1643
1644         entry->user = entry->nr - nr;
1645 }
1646
1647 static void
1648 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1649 {
1650         int is_user;
1651
1652         if (!regs)
1653                 return;
1654
1655         is_user = user_mode(regs);
1656
1657         if (!current || current->pid == 0)
1658                 return;
1659
1660         if (is_user && current->state != TASK_RUNNING)
1661                 return;
1662
1663         if (!is_user)
1664                 perf_callchain_kernel(regs, entry);
1665
1666         if (current->mm)
1667                 perf_callchain_user(regs, entry);
1668 }
1669
1670 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1671 {
1672         struct perf_callchain_entry *entry;
1673
1674         if (in_nmi())
1675                 entry = &__get_cpu_var(nmi_entry);
1676         else
1677                 entry = &__get_cpu_var(irq_entry);
1678
1679         entry->nr = 0;
1680         entry->hv = 0;
1681         entry->kernel = 0;
1682         entry->user = 0;
1683
1684         perf_do_callchain(regs, entry);
1685
1686         return entry;
1687 }