git.oblomov.eu Git - linux-2.6/blob - kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *
   7  *  For licencing details see kernel-base/COPYING
   8  */
   9
  10 #include <linux/fs.h>
  11 #include <linux/cpu.h>
  12 #include <linux/smp.h>
  13 #include <linux/file.h>
  14 #include <linux/poll.h>
  15 #include <linux/sysfs.h>
  16 #include <linux/ptrace.h>
  17 #include <linux/percpu.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/syscalls.h>
  20 #include <linux/anon_inodes.h>
  21 #include <linux/perf_counter.h>
  22
  23 /*
  24  * Each CPU has a list of per CPU counters:
  25  */
  26 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  27
  28 int perf_max_counters __read_mostly;
  29 static int perf_reserved_percpu __read_mostly;
  30 static int perf_overcommit __read_mostly = 1;
  31
  32 /*
  33  * Mutex for (sysadmin-configurable) counter reservations:
  34  */
  35 static DEFINE_MUTEX(perf_resource_mutex);
  36
  37 /*
  38  * Architecture provided APIs - weak aliases:
  39  */
  40 extern __weak const struct hw_perf_counter_ops *
  41 hw_perf_counter_init(struct perf_counter *counter)
  42 {
  43         return ERR_PTR(-EINVAL);
  44 }
  45
  46 u64 __weak hw_perf_save_disable(void)           { return 0; }
  47 void __weak hw_perf_restore(u64 ctrl)           { }
  48 void __weak hw_perf_counter_setup(void)         { }
  49
  50 static void
  51 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  52 {
  53         struct perf_counter *group_leader = counter->group_leader;
  54
  55         /*
  56          * Depending on whether it is a standalone or sibling counter,
  57          * add it straight to the context's counter list, or to the group
  58          * leader's sibling list:
  59          */
  60         if (counter->group_leader == counter)
  61                 list_add_tail(&counter->list_entry, &ctx->counter_list);
  62         else
  63                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
  64 }
  65
  66 static void
  67 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
  68 {
  69         struct perf_counter *sibling, *tmp;
  70
  71         list_del_init(&counter->list_entry);
  72
  73         /*
  74          * If this was a group counter with sibling counters then
  75          * upgrade the siblings to singleton counters by adding them
  76          * to the context list directly:
  77          */
  78         list_for_each_entry_safe(sibling, tmp,
  79                                  &counter->sibling_list, list_entry) {
  80
  81                 list_del_init(&sibling->list_entry);
  82                 list_add_tail(&sibling->list_entry, &ctx->counter_list);
  83                 WARN_ON_ONCE(!sibling->group_leader);
  84                 WARN_ON_ONCE(sibling->group_leader == sibling);
  85                 sibling->group_leader = sibling;
  86         }
  87 }
  88
  89 /*
  90  * Cross CPU call to remove a performance counter
  91  *
  92  * We disable the counter on the hardware level first. After that we
  93  * remove it from the context list.
  94  */
  95 static void __perf_counter_remove_from_context(void *info)
  96 {
  97         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
  98         struct perf_counter *counter = info;
  99         struct perf_counter_context *ctx = counter->ctx;
 100         u64 perf_flags;
 101
 102         /*
 103          * If this is a task context, we need to check whether it is
 104          * the current task context of this cpu. If not it has been
 105          * scheduled out before the smp call arrived.
 106          */
 107         if (ctx->task && cpuctx->task_ctx != ctx)
 108                 return;
 109
 110         spin_lock(&ctx->lock);
 111
 112         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 113                 counter->hw_ops->hw_perf_counter_disable(counter);
 114                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 115                 ctx->nr_active--;
 116                 cpuctx->active_oncpu--;
 117                 counter->task = NULL;
 118         }
 119         ctx->nr_counters--;
 120
 121         /*
 122          * Protect the list operation against NMI by disabling the
 123          * counters on a global level. NOP for non NMI based counters.
 124          */
 125         perf_flags = hw_perf_save_disable();
 126         list_del_counter(counter, ctx);
 127         hw_perf_restore(perf_flags);
 128
 129         if (!ctx->task) {
 130                 /*
 131                  * Allow more per task counters with respect to the
 132                  * reservation:
 133                  */
 134                 cpuctx->max_pertask =
 135                         min(perf_max_counters - ctx->nr_counters,
 136                             perf_max_counters - perf_reserved_percpu);
 137         }
 138
 139         spin_unlock(&ctx->lock);
 140 }
 141
 142
 143 /*
 144  * Remove the counter from a task's (or a CPU's) list of counters.
 145  *
 146  * Must be called with counter->mutex held.
 147  *
 148  * CPU counters are removed with a smp call. For task counters we only
 149  * call when the task is on a CPU.
 150  */
 151 static void perf_counter_remove_from_context(struct perf_counter *counter)
 152 {
 153         struct perf_counter_context *ctx = counter->ctx;
 154         struct task_struct *task = ctx->task;
 155
 156         if (!task) {
 157                 /*
 158                  * Per cpu counters are removed via an smp call and
 159                  * the removal is always sucessful.
 160                  */
 161                 smp_call_function_single(counter->cpu,
 162                                          __perf_counter_remove_from_context,
 163                                          counter, 1);
 164                 return;
 165         }
 166
 167 retry:
 168         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 169                                  counter);
 170
 171         spin_lock_irq(&ctx->lock);
 172         /*
 173          * If the context is active we need to retry the smp call.
 174          */
 175         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 176                 spin_unlock_irq(&ctx->lock);
 177                 goto retry;
 178         }
 179
 180         /*
 181          * The lock prevents that this context is scheduled in so we
 182          * can remove the counter safely, if the call above did not
 183          * succeed.
 184          */
 185         if (!list_empty(&counter->list_entry)) {
 186                 ctx->nr_counters--;
 187                 list_del_counter(counter, ctx);
 188                 counter->task = NULL;
 189         }
 190         spin_unlock_irq(&ctx->lock);
 191 }
 192
 193 /*
 194  * Cross CPU call to install and enable a preformance counter
 195  */
 196 static void __perf_install_in_context(void *info)
 197 {
 198         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 199         struct perf_counter *counter = info;
 200         struct perf_counter_context *ctx = counter->ctx;
 201         int cpu = smp_processor_id();
 202         u64 perf_flags;
 203
 204         /*
 205          * If this is a task context, we need to check whether it is
 206          * the current task context of this cpu. If not it has been
 207          * scheduled out before the smp call arrived.
 208          */
 209         if (ctx->task && cpuctx->task_ctx != ctx)
 210                 return;
 211
 212         spin_lock(&ctx->lock);
 213
 214         /*
 215          * Protect the list operation against NMI by disabling the
 216          * counters on a global level. NOP for non NMI based counters.
 217          */
 218         perf_flags = hw_perf_save_disable();
 219         list_add_counter(counter, ctx);
 220         hw_perf_restore(perf_flags);
 221
 222         ctx->nr_counters++;
 223
 224         if (cpuctx->active_oncpu < perf_max_counters) {
 225                 counter->state = PERF_COUNTER_STATE_ACTIVE;
 226                 counter->oncpu = cpu;
 227                 ctx->nr_active++;
 228                 cpuctx->active_oncpu++;
 229                 counter->hw_ops->hw_perf_counter_enable(counter);
 230         }
 231
 232         if (!ctx->task && cpuctx->max_pertask)
 233                 cpuctx->max_pertask--;
 234
 235         spin_unlock(&ctx->lock);
 236 }
 237
 238 /*
 239  * Attach a performance counter to a context
 240  *
 241  * First we add the counter to the list with the hardware enable bit
 242  * in counter->hw_config cleared.
 243  *
 244  * If the counter is attached to a task which is on a CPU we use a smp
 245  * call to enable it in the task context. The task might have been
 246  * scheduled away, but we check this in the smp call again.
 247  */
 248 static void
 249 perf_install_in_context(struct perf_counter_context *ctx,
 250                         struct perf_counter *counter,
 251                         int cpu)
 252 {
 253         struct task_struct *task = ctx->task;
 254
 255         counter->ctx = ctx;
 256         if (!task) {
 257                 /*
 258                  * Per cpu counters are installed via an smp call and
 259                  * the install is always sucessful.
 260                  */
 261                 smp_call_function_single(cpu, __perf_install_in_context,
 262                                          counter, 1);
 263                 return;
 264         }
 265
 266         counter->task = task;
 267 retry:
 268         task_oncpu_function_call(task, __perf_install_in_context,
 269                                  counter);
 270
 271         spin_lock_irq(&ctx->lock);
 272         /*
 273          * we need to retry the smp call.
 274          */
 275         if (ctx->nr_active && list_empty(&counter->list_entry)) {
 276                 spin_unlock_irq(&ctx->lock);
 277                 goto retry;
 278         }
 279
 280         /*
 281          * The lock prevents that this context is scheduled in so we
 282          * can add the counter safely, if it the call above did not
 283          * succeed.
 284          */
 285         if (list_empty(&counter->list_entry)) {
 286                 list_add_counter(counter, ctx);
 287                 ctx->nr_counters++;
 288         }
 289         spin_unlock_irq(&ctx->lock);
 290 }
 291
 292 static void
 293 counter_sched_out(struct perf_counter *counter,
 294                   struct perf_cpu_context *cpuctx,
 295                   struct perf_counter_context *ctx)
 296 {
 297         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 298                 return;
 299
 300         counter->hw_ops->hw_perf_counter_disable(counter);
 301         counter->state = PERF_COUNTER_STATE_INACTIVE;
 302         counter->oncpu = -1;
 303
 304         cpuctx->active_oncpu--;
 305         ctx->nr_active--;
 306 }
 307
 308 static void
 309 group_sched_out(struct perf_counter *group_counter,
 310                 struct perf_cpu_context *cpuctx,
 311                 struct perf_counter_context *ctx)
 312 {
 313         struct perf_counter *counter;
 314
 315         counter_sched_out(group_counter, cpuctx, ctx);
 316
 317         /*
 318          * Schedule out siblings (if any):
 319          */
 320         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 321                 counter_sched_out(counter, cpuctx, ctx);
 322 }
 323
 324 /*
 325  * Called from scheduler to remove the counters of the current task,
 326  * with interrupts disabled.
 327  *
 328  * We stop each counter and update the counter value in counter->count.
 329  *
 330  * This does not protect us against NMI, but hw_perf_counter_disable()
 331  * sets the disabled bit in the control field of counter _before_
 332  * accessing the counter control register. If a NMI hits, then it will
 333  * not restart the counter.
 334  */
 335 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 336 {
 337         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 338         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 339         struct perf_counter *counter;
 340
 341         if (likely(!cpuctx->task_ctx))
 342                 return;
 343
 344         spin_lock(&ctx->lock);
 345         if (ctx->nr_active) {
 346                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 347                         group_sched_out(counter, cpuctx, ctx);
 348         }
 349         spin_unlock(&ctx->lock);
 350         cpuctx->task_ctx = NULL;
 351 }
 352
 353 static void
 354 counter_sched_in(struct perf_counter *counter,
 355                  struct perf_cpu_context *cpuctx,
 356                  struct perf_counter_context *ctx,
 357                  int cpu)
 358 {
 359         if (counter->state == PERF_COUNTER_STATE_OFF)
 360                 return;
 361
 362         counter->hw_ops->hw_perf_counter_enable(counter);
 363         counter->state = PERF_COUNTER_STATE_ACTIVE;
 364         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 365
 366         cpuctx->active_oncpu++;
 367         ctx->nr_active++;
 368 }
 369
 370 static void
 371 group_sched_in(struct perf_counter *group_counter,
 372                struct perf_cpu_context *cpuctx,
 373                struct perf_counter_context *ctx,
 374                int cpu)
 375 {
 376         struct perf_counter *counter;
 377
 378         counter_sched_in(group_counter, cpuctx, ctx, cpu);
 379
 380         /*
 381          * Schedule in siblings as one group (if any):
 382          */
 383         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 384                 counter_sched_in(counter, cpuctx, ctx, cpu);
 385 }
 386
 387 /*
 388  * Called from scheduler to add the counters of the current task
 389  * with interrupts disabled.
 390  *
 391  * We restore the counter value and then enable it.
 392  *
 393  * This does not protect us against NMI, but hw_perf_counter_enable()
 394  * sets the enabled bit in the control field of counter _before_
 395  * accessing the counter control register. If a NMI hits, then it will
 396  * keep the counter running.
 397  */
 398 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 399 {
 400         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 401         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 402         struct perf_counter *counter;
 403
 404         if (likely(!ctx->nr_counters))
 405                 return;
 406
 407         spin_lock(&ctx->lock);
 408         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 409                 if (ctx->nr_active == cpuctx->max_pertask)
 410                         break;
 411
 412                 /*
 413                  * Listen to the 'cpu' scheduling filter constraint
 414                  * of counters:
 415                  */
 416                 if (counter->cpu != -1 && counter->cpu != cpu)
 417                         continue;
 418
 419                 group_sched_in(counter, cpuctx, ctx, cpu);
 420         }
 421         spin_unlock(&ctx->lock);
 422
 423         cpuctx->task_ctx = ctx;
 424 }
 425
 426 int perf_counter_task_disable(void)
 427 {
 428         struct task_struct *curr = current;
 429         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 430         struct perf_counter *counter;
 431         u64 perf_flags;
 432         int cpu;
 433
 434         if (likely(!ctx->nr_counters))
 435                 return 0;
 436
 437         local_irq_disable();
 438         cpu = smp_processor_id();
 439
 440         perf_counter_task_sched_out(curr, cpu);
 441
 442         spin_lock(&ctx->lock);
 443
 444         /*
 445          * Disable all the counters:
 446          */
 447         perf_flags = hw_perf_save_disable();
 448
 449         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 450                 WARN_ON_ONCE(counter->state == PERF_COUNTER_STATE_ACTIVE);
 451                 counter->state = PERF_COUNTER_STATE_OFF;
 452         }
 453         hw_perf_restore(perf_flags);
 454
 455         spin_unlock(&ctx->lock);
 456
 457         local_irq_enable();
 458
 459         return 0;
 460 }
 461
 462 int perf_counter_task_enable(void)
 463 {
 464         struct task_struct *curr = current;
 465         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 466         struct perf_counter *counter;
 467         u64 perf_flags;
 468         int cpu;
 469
 470         if (likely(!ctx->nr_counters))
 471                 return 0;
 472
 473         local_irq_disable();
 474         cpu = smp_processor_id();
 475
 476         spin_lock(&ctx->lock);
 477
 478         /*
 479          * Disable all the counters:
 480          */
 481         perf_flags = hw_perf_save_disable();
 482
 483         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 484                 if (counter->state != PERF_COUNTER_STATE_OFF)
 485                         continue;
 486                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 487         }
 488         hw_perf_restore(perf_flags);
 489
 490         spin_unlock(&ctx->lock);
 491
 492         perf_counter_task_sched_in(curr, cpu);
 493
 494         local_irq_enable();
 495
 496         return 0;
 497 }
 498
 499 void perf_counter_task_tick(struct task_struct *curr, int cpu)
 500 {
 501         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 502         struct perf_counter *counter;
 503         u64 perf_flags;
 504
 505         if (likely(!ctx->nr_counters))
 506                 return;
 507
 508         perf_counter_task_sched_out(curr, cpu);
 509
 510         spin_lock(&ctx->lock);
 511
 512         /*
 513          * Rotate the first entry last (works just fine for group counters too):
 514          */
 515         perf_flags = hw_perf_save_disable();
 516         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 517                 list_del(&counter->list_entry);
 518                 list_add_tail(&counter->list_entry, &ctx->counter_list);
 519                 break;
 520         }
 521         hw_perf_restore(perf_flags);
 522
 523         spin_unlock(&ctx->lock);
 524
 525         perf_counter_task_sched_in(curr, cpu);
 526 }
 527
 528 /*
 529  * Initialize the perf_counter context in a task_struct:
 530  */
 531 static void
 532 __perf_counter_init_context(struct perf_counter_context *ctx,
 533                             struct task_struct *task)
 534 {
 535         spin_lock_init(&ctx->lock);
 536         INIT_LIST_HEAD(&ctx->counter_list);
 537         ctx->nr_counters        = 0;
 538         ctx->task               = task;
 539 }
 540 /*
 541  * Initialize the perf_counter context in task_struct
 542  */
 543 void perf_counter_init_task(struct task_struct *task)
 544 {
 545         __perf_counter_init_context(&task->perf_counter_ctx, task);
 546 }
 547
 548 /*
 549  * Cross CPU call to read the hardware counter
 550  */
 551 static void __hw_perf_counter_read(void *info)
 552 {
 553         struct perf_counter *counter = info;
 554
 555         counter->hw_ops->hw_perf_counter_read(counter);
 556 }
 557
 558 static u64 perf_counter_read(struct perf_counter *counter)
 559 {
 560         /*
 561          * If counter is enabled and currently active on a CPU, update the
 562          * value in the counter structure:
 563          */
 564         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 565                 smp_call_function_single(counter->oncpu,
 566                                          __hw_perf_counter_read, counter, 1);
 567         }
 568
 569         return atomic64_read(&counter->count);
 570 }
 571
 572 /*
 573  * Cross CPU call to switch performance data pointers
 574  */
 575 static void __perf_switch_irq_data(void *info)
 576 {
 577         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 578         struct perf_counter *counter = info;
 579         struct perf_counter_context *ctx = counter->ctx;
 580         struct perf_data *oldirqdata = counter->irqdata;
 581
 582         /*
 583          * If this is a task context, we need to check whether it is
 584          * the current task context of this cpu. If not it has been
 585          * scheduled out before the smp call arrived.
 586          */
 587         if (ctx->task) {
 588                 if (cpuctx->task_ctx != ctx)
 589                         return;
 590                 spin_lock(&ctx->lock);
 591         }
 592
 593         /* Change the pointer NMI safe */
 594         atomic_long_set((atomic_long_t *)&counter->irqdata,
 595                         (unsigned long) counter->usrdata);
 596         counter->usrdata = oldirqdata;
 597
 598         if (ctx->task)
 599                 spin_unlock(&ctx->lock);
 600 }
 601
 602 static struct perf_data *perf_switch_irq_data(struct perf_counter *counter)
 603 {
 604         struct perf_counter_context *ctx = counter->ctx;
 605         struct perf_data *oldirqdata = counter->irqdata;
 606         struct task_struct *task = ctx->task;
 607
 608         if (!task) {
 609                 smp_call_function_single(counter->cpu,
 610                                          __perf_switch_irq_data,
 611                                          counter, 1);
 612                 return counter->usrdata;
 613         }
 614
 615 retry:
 616         spin_lock_irq(&ctx->lock);
 617         if (counter->state != PERF_COUNTER_STATE_ACTIVE) {
 618                 counter->irqdata = counter->usrdata;
 619                 counter->usrdata = oldirqdata;
 620                 spin_unlock_irq(&ctx->lock);
 621                 return oldirqdata;
 622         }
 623         spin_unlock_irq(&ctx->lock);
 624         task_oncpu_function_call(task, __perf_switch_irq_data, counter);
 625         /* Might have failed, because task was scheduled out */
 626         if (counter->irqdata == oldirqdata)
 627                 goto retry;
 628
 629         return counter->usrdata;
 630 }
 631
 632 static void put_context(struct perf_counter_context *ctx)
 633 {
 634         if (ctx->task)
 635                 put_task_struct(ctx->task);
 636 }
 637
 638 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
 639 {
 640         struct perf_cpu_context *cpuctx;
 641         struct perf_counter_context *ctx;
 642         struct task_struct *task;
 643
 644         /*
 645          * If cpu is not a wildcard then this is a percpu counter:
 646          */
 647         if (cpu != -1) {
 648                 /* Must be root to operate on a CPU counter: */
 649                 if (!capable(CAP_SYS_ADMIN))
 650                         return ERR_PTR(-EACCES);
 651
 652                 if (cpu < 0 || cpu > num_possible_cpus())
 653                         return ERR_PTR(-EINVAL);
 654
 655                 /*
 656                  * We could be clever and allow to attach a counter to an
 657                  * offline CPU and activate it when the CPU comes up, but
 658                  * that's for later.
 659                  */
 660                 if (!cpu_isset(cpu, cpu_online_map))
 661                         return ERR_PTR(-ENODEV);
 662
 663                 cpuctx = &per_cpu(perf_cpu_context, cpu);
 664                 ctx = &cpuctx->ctx;
 665
 666                 WARN_ON_ONCE(ctx->task);
 667                 return ctx;
 668         }
 669
 670         rcu_read_lock();
 671         if (!pid)
 672                 task = current;
 673         else
 674                 task = find_task_by_vpid(pid);
 675         if (task)
 676                 get_task_struct(task);
 677         rcu_read_unlock();
 678
 679         if (!task)
 680                 return ERR_PTR(-ESRCH);
 681
 682         ctx = &task->perf_counter_ctx;
 683         ctx->task = task;
 684
 685         /* Reuse ptrace permission checks for now. */
 686         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
 687                 put_context(ctx);
 688                 return ERR_PTR(-EACCES);
 689         }
 690
 691         return ctx;
 692 }
 693
 694 /*
 695  * Called when the last reference to the file is gone.
 696  */
 697 static int perf_release(struct inode *inode, struct file *file)
 698 {
 699         struct perf_counter *counter = file->private_data;
 700         struct perf_counter_context *ctx = counter->ctx;
 701
 702         file->private_data = NULL;
 703
 704         mutex_lock(&counter->mutex);
 705
 706         perf_counter_remove_from_context(counter);
 707         put_context(ctx);
 708
 709         mutex_unlock(&counter->mutex);
 710
 711         kfree(counter);
 712
 713         return 0;
 714 }
 715
 716 /*
 717  * Read the performance counter - simple non blocking version for now
 718  */
 719 static ssize_t
 720 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
 721 {
 722         u64 cntval;
 723
 724         if (count != sizeof(cntval))
 725                 return -EINVAL;
 726
 727         mutex_lock(&counter->mutex);
 728         cntval = perf_counter_read(counter);
 729         mutex_unlock(&counter->mutex);
 730
 731         return put_user(cntval, (u64 __user *) buf) ? -EFAULT : sizeof(cntval);
 732 }
 733
 734 static ssize_t
 735 perf_copy_usrdata(struct perf_data *usrdata, char __user *buf, size_t count)
 736 {
 737         if (!usrdata->len)
 738                 return 0;
 739
 740         count = min(count, (size_t)usrdata->len);
 741         if (copy_to_user(buf, usrdata->data + usrdata->rd_idx, count))
 742                 return -EFAULT;
 743
 744         /* Adjust the counters */
 745         usrdata->len -= count;
 746         if (!usrdata->len)
 747                 usrdata->rd_idx = 0;
 748         else
 749                 usrdata->rd_idx += count;
 750
 751         return count;
 752 }
 753
 754 static ssize_t
 755 perf_read_irq_data(struct perf_counter  *counter,
 756                    char __user          *buf,
 757                    size_t               count,
 758                    int                  nonblocking)
 759 {
 760         struct perf_data *irqdata, *usrdata;
 761         DECLARE_WAITQUEUE(wait, current);
 762         ssize_t res;
 763
 764         irqdata = counter->irqdata;
 765         usrdata = counter->usrdata;
 766
 767         if (usrdata->len + irqdata->len >= count)
 768                 goto read_pending;
 769
 770         if (nonblocking)
 771                 return -EAGAIN;
 772
 773         spin_lock_irq(&counter->waitq.lock);
 774         __add_wait_queue(&counter->waitq, &wait);
 775         for (;;) {
 776                 set_current_state(TASK_INTERRUPTIBLE);
 777                 if (usrdata->len + irqdata->len >= count)
 778                         break;
 779
 780                 if (signal_pending(current))
 781                         break;
 782
 783                 spin_unlock_irq(&counter->waitq.lock);
 784                 schedule();
 785                 spin_lock_irq(&counter->waitq.lock);
 786         }
 787         __remove_wait_queue(&counter->waitq, &wait);
 788         __set_current_state(TASK_RUNNING);
 789         spin_unlock_irq(&counter->waitq.lock);
 790
 791         if (usrdata->len + irqdata->len < count)
 792                 return -ERESTARTSYS;
 793 read_pending:
 794         mutex_lock(&counter->mutex);
 795
 796         /* Drain pending data first: */
 797         res = perf_copy_usrdata(usrdata, buf, count);
 798         if (res < 0 || res == count)
 799                 goto out;
 800
 801         /* Switch irq buffer: */
 802         usrdata = perf_switch_irq_data(counter);
 803         if (perf_copy_usrdata(usrdata, buf + res, count - res) < 0) {
 804                 if (!res)
 805                         res = -EFAULT;
 806         } else {
 807                 res = count;
 808         }
 809 out:
 810         mutex_unlock(&counter->mutex);
 811
 812         return res;
 813 }
 814
 815 static ssize_t
 816 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 817 {
 818         struct perf_counter *counter = file->private_data;
 819
 820         switch (counter->hw_event.record_type) {
 821         case PERF_RECORD_SIMPLE:
 822                 return perf_read_hw(counter, buf, count);
 823
 824         case PERF_RECORD_IRQ:
 825         case PERF_RECORD_GROUP:
 826                 return perf_read_irq_data(counter, buf, count,
 827                                           file->f_flags & O_NONBLOCK);
 828         }
 829         return -EINVAL;
 830 }
 831
 832 static unsigned int perf_poll(struct file *file, poll_table *wait)
 833 {
 834         struct perf_counter *counter = file->private_data;
 835         unsigned int events = 0;
 836         unsigned long flags;
 837
 838         poll_wait(file, &counter->waitq, wait);
 839
 840         spin_lock_irqsave(&counter->waitq.lock, flags);
 841         if (counter->usrdata->len || counter->irqdata->len)
 842                 events |= POLLIN;
 843         spin_unlock_irqrestore(&counter->waitq.lock, flags);
 844
 845         return events;
 846 }
 847
 848 static const struct file_operations perf_fops = {
 849         .release                = perf_release,
 850         .read                   = perf_read,
 851         .poll                   = perf_poll,
 852 };
 853
 854 static void cpu_clock_perf_counter_enable(struct perf_counter *counter)
 855 {
 856 }
 857
 858 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
 859 {
 860 }
 861
 862 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
 863 {
 864         int cpu = raw_smp_processor_id();
 865
 866         atomic64_set(&counter->count, cpu_clock(cpu));
 867 }
 868
 869 static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
 870         .hw_perf_counter_enable         = cpu_clock_perf_counter_enable,
 871         .hw_perf_counter_disable        = cpu_clock_perf_counter_disable,
 872         .hw_perf_counter_read           = cpu_clock_perf_counter_read,
 873 };
 874
 875 static void task_clock_perf_counter_enable(struct perf_counter *counter)
 876 {
 877 }
 878
 879 static void task_clock_perf_counter_disable(struct perf_counter *counter)
 880 {
 881 }
 882
 883 static void task_clock_perf_counter_read(struct perf_counter *counter)
 884 {
 885         atomic64_set(&counter->count, current->se.sum_exec_runtime);
 886 }
 887
 888 static const struct hw_perf_counter_ops perf_ops_task_clock = {
 889         .hw_perf_counter_enable         = task_clock_perf_counter_enable,
 890         .hw_perf_counter_disable        = task_clock_perf_counter_disable,
 891         .hw_perf_counter_read           = task_clock_perf_counter_read,
 892 };
 893
 894 static const struct hw_perf_counter_ops *
 895 sw_perf_counter_init(struct perf_counter *counter)
 896 {
 897         const struct hw_perf_counter_ops *hw_ops = NULL;
 898
 899         switch (counter->hw_event.type) {
 900         case PERF_COUNT_CPU_CLOCK:
 901                 hw_ops = &perf_ops_cpu_clock;
 902                 break;
 903         case PERF_COUNT_TASK_CLOCK:
 904                 hw_ops = &perf_ops_task_clock;
 905                 break;
 906         default:
 907                 break;
 908         }
 909         return hw_ops;
 910 }
 911
 912 /*
 913  * Allocate and initialize a counter structure
 914  */
 915 static struct perf_counter *
 916 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
 917                    int cpu,
 918                    struct perf_counter *group_leader)
 919 {
 920         const struct hw_perf_counter_ops *hw_ops;
 921         struct perf_counter *counter;
 922
 923         counter = kzalloc(sizeof(*counter), GFP_KERNEL);
 924         if (!counter)
 925                 return NULL;
 926
 927         /*
 928          * Single counters are their own group leaders, with an
 929          * empty sibling list:
 930          */
 931         if (!group_leader)
 932                 group_leader = counter;
 933
 934         mutex_init(&counter->mutex);
 935         INIT_LIST_HEAD(&counter->list_entry);
 936         INIT_LIST_HEAD(&counter->sibling_list);
 937         init_waitqueue_head(&counter->waitq);
 938
 939         counter->irqdata                = &counter->data[0];
 940         counter->usrdata                = &counter->data[1];
 941         counter->cpu                    = cpu;
 942         counter->hw_event               = *hw_event;
 943         counter->wakeup_pending         = 0;
 944         counter->group_leader           = group_leader;
 945         counter->hw_ops                 = NULL;
 946
 947         hw_ops = NULL;
 948         if (!hw_event->raw && hw_event->type < 0)
 949                 hw_ops = sw_perf_counter_init(counter);
 950         if (!hw_ops) {
 951                 hw_ops = hw_perf_counter_init(counter);
 952         }
 953
 954         if (!hw_ops) {
 955                 kfree(counter);
 956                 return NULL;
 957         }
 958         counter->hw_ops = hw_ops;
 959
 960         return counter;
 961 }
 962
 963 /**
 964  * sys_perf_task_open - open a performance counter, associate it to a task/cpu
 965  *
 966  * @hw_event_uptr:      event type attributes for monitoring/sampling
 967  * @pid:                target pid
 968  * @cpu:                target cpu
 969  * @group_fd:           group leader counter fd
 970  */
 971 asmlinkage int
 972 sys_perf_counter_open(struct perf_counter_hw_event *hw_event_uptr __user,
 973                       pid_t pid, int cpu, int group_fd)
 974 {
 975         struct perf_counter *counter, *group_leader;
 976         struct perf_counter_hw_event hw_event;
 977         struct perf_counter_context *ctx;
 978         struct file *group_file = NULL;
 979         int fput_needed = 0;
 980         int ret;
 981
 982         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
 983                 return -EFAULT;
 984
 985         /*
 986          * Get the target context (task or percpu):
 987          */
 988         ctx = find_get_context(pid, cpu);
 989         if (IS_ERR(ctx))
 990                 return PTR_ERR(ctx);
 991
 992         /*
 993          * Look up the group leader (we will attach this counter to it):
 994          */
 995         group_leader = NULL;
 996         if (group_fd != -1) {
 997                 ret = -EINVAL;
 998                 group_file = fget_light(group_fd, &fput_needed);
 999                 if (!group_file)
1000                         goto err_put_context;
1001                 if (group_file->f_op != &perf_fops)
1002                         goto err_put_context;
1003
1004                 group_leader = group_file->private_data;
1005                 /*
1006                  * Do not allow a recursive hierarchy (this new sibling
1007                  * becoming part of another group-sibling):
1008                  */
1009                 if (group_leader->group_leader != group_leader)
1010                         goto err_put_context;
1011                 /*
1012                  * Do not allow to attach to a group in a different
1013                  * task or CPU context:
1014                  */
1015                 if (group_leader->ctx != ctx)
1016                         goto err_put_context;
1017         }
1018
1019         ret = -EINVAL;
1020         counter = perf_counter_alloc(&hw_event, cpu, group_leader);
1021         if (!counter)
1022                 goto err_put_context;
1023
1024         perf_install_in_context(ctx, counter, cpu);
1025
1026         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
1027         if (ret < 0)
1028                 goto err_remove_free_put_context;
1029
1030 out_fput:
1031         fput_light(group_file, fput_needed);
1032
1033         return ret;
1034
1035 err_remove_free_put_context:
1036         mutex_lock(&counter->mutex);
1037         perf_counter_remove_from_context(counter);
1038         mutex_unlock(&counter->mutex);
1039         kfree(counter);
1040
1041 err_put_context:
1042         put_context(ctx);
1043
1044         goto out_fput;
1045 }
1046
1047 static void __cpuinit perf_counter_init_cpu(int cpu)
1048 {
1049         struct perf_cpu_context *cpuctx;
1050
1051         cpuctx = &per_cpu(perf_cpu_context, cpu);
1052         __perf_counter_init_context(&cpuctx->ctx, NULL);
1053
1054         mutex_lock(&perf_resource_mutex);
1055         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
1056         mutex_unlock(&perf_resource_mutex);
1057
1058         hw_perf_counter_setup();
1059 }
1060
1061 #ifdef CONFIG_HOTPLUG_CPU
1062 static void __perf_counter_exit_cpu(void *info)
1063 {
1064         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1065         struct perf_counter_context *ctx = &cpuctx->ctx;
1066         struct perf_counter *counter, *tmp;
1067
1068         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
1069                 __perf_counter_remove_from_context(counter);
1070
1071 }
1072 static void perf_counter_exit_cpu(int cpu)
1073 {
1074         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
1075 }
1076 #else
1077 static inline void perf_counter_exit_cpu(int cpu) { }
1078 #endif
1079
1080 static int __cpuinit
1081 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
1082 {
1083         unsigned int cpu = (long)hcpu;
1084
1085         switch (action) {
1086
1087         case CPU_UP_PREPARE:
1088         case CPU_UP_PREPARE_FROZEN:
1089                 perf_counter_init_cpu(cpu);
1090                 break;
1091
1092         case CPU_DOWN_PREPARE:
1093         case CPU_DOWN_PREPARE_FROZEN:
1094                 perf_counter_exit_cpu(cpu);
1095                 break;
1096
1097         default:
1098                 break;
1099         }
1100
1101         return NOTIFY_OK;
1102 }
1103
1104 static struct notifier_block __cpuinitdata perf_cpu_nb = {
1105         .notifier_call          = perf_cpu_notify,
1106 };
1107
1108 static int __init perf_counter_init(void)
1109 {
1110         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
1111                         (void *)(long)smp_processor_id());
1112         register_cpu_notifier(&perf_cpu_nb);
1113
1114         return 0;
1115 }
1116 early_initcall(perf_counter_init);
1117
1118 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
1119 {
1120         return sprintf(buf, "%d\n", perf_reserved_percpu);
1121 }
1122
1123 static ssize_t
1124 perf_set_reserve_percpu(struct sysdev_class *class,
1125                         const char *buf,
1126                         size_t count)
1127 {
1128         struct perf_cpu_context *cpuctx;
1129         unsigned long val;
1130         int err, cpu, mpt;
1131
1132         err = strict_strtoul(buf, 10, &val);
1133         if (err)
1134                 return err;
1135         if (val > perf_max_counters)
1136                 return -EINVAL;
1137
1138         mutex_lock(&perf_resource_mutex);
1139         perf_reserved_percpu = val;
1140         for_each_online_cpu(cpu) {
1141                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1142                 spin_lock_irq(&cpuctx->ctx.lock);
1143                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
1144                           perf_max_counters - perf_reserved_percpu);
1145                 cpuctx->max_pertask = mpt;
1146                 spin_unlock_irq(&cpuctx->ctx.lock);
1147         }
1148         mutex_unlock(&perf_resource_mutex);
1149
1150         return count;
1151 }
1152
1153 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
1154 {
1155         return sprintf(buf, "%d\n", perf_overcommit);
1156 }
1157
1158 static ssize_t
1159 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
1160 {
1161         unsigned long val;
1162         int err;
1163
1164         err = strict_strtoul(buf, 10, &val);
1165         if (err)
1166                 return err;
1167         if (val > 1)
1168                 return -EINVAL;
1169
1170         mutex_lock(&perf_resource_mutex);
1171         perf_overcommit = val;
1172         mutex_unlock(&perf_resource_mutex);
1173
1174         return count;
1175 }
1176
1177 static SYSDEV_CLASS_ATTR(
1178                                 reserve_percpu,
1179                                 0644,
1180                                 perf_show_reserve_percpu,
1181                                 perf_set_reserve_percpu
1182                         );
1183
1184 static SYSDEV_CLASS_ATTR(
1185                                 overcommit,
1186                                 0644,
1187                                 perf_show_overcommit,
1188                                 perf_set_overcommit
1189                         );
1190
1191 static struct attribute *perfclass_attrs[] = {
1192         &attr_reserve_percpu.attr,
1193         &attr_overcommit.attr,
1194         NULL
1195 };
1196
1197 static struct attribute_group perfclass_attr_group = {
1198         .attrs                  = perfclass_attrs,
1199         .name                   = "perf_counters",
1200 };
1201
1202 static int __init perf_counter_sysfs_init(void)
1203 {
1204         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
1205                                   &perfclass_attr_group);
1206 }
1207 device_initcall(perf_counter_sysfs_init);
1208