git.oblomov.eu Git - linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30
  31 #include <asm/ftrace.h>
  32
  33 #include "trace.h"
  34
  35 #define FTRACE_WARN_ON(cond)                    \
  36         do {                                    \
  37                 if (WARN_ON(cond))              \
  38                         ftrace_kill();          \
  39         } while (0)
  40
  41 #define FTRACE_WARN_ON_ONCE(cond)               \
  42         do {                                    \
  43                 if (WARN_ON_ONCE(cond))         \
  44                         ftrace_kill();          \
  45         } while (0)
  46
  47 /* ftrace_enabled is a method to turn ftrace on or off */
  48 int ftrace_enabled __read_mostly;
  49 static int last_ftrace_enabled;
  50
  51 /* set when tracing only a pid */
  52 struct pid *ftrace_pid_trace;
  53 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
  54
  55 /* Quick disabling of function tracer. */
  56 int function_trace_stop;
  57
  58 /*
  59  * ftrace_disabled is set when an anomaly is discovered.
  60  * ftrace_disabled is much stronger than ftrace_enabled.
  61  */
  62 static int ftrace_disabled __read_mostly;
  63
  64 static DEFINE_SPINLOCK(ftrace_lock);
  65 static DEFINE_MUTEX(ftrace_sysctl_lock);
  66 static DEFINE_MUTEX(ftrace_start_lock);
  67
  68 static struct ftrace_ops ftrace_list_end __read_mostly =
  69 {
  70         .func = ftrace_stub,
  71 };
  72
  73 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  74 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  75 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  76 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  77
  78 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  79 {
  80         struct ftrace_ops *op = ftrace_list;
  81
  82         /* in case someone actually ports this to alpha! */
  83         read_barrier_depends();
  84
  85         while (op != &ftrace_list_end) {
  86                 /* silly alpha */
  87                 read_barrier_depends();
  88                 op->func(ip, parent_ip);
  89                 op = op->next;
  90         };
  91 }
  92
  93 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  94 {
  95         if (!test_tsk_trace_trace(current))
  96                 return;
  97
  98         ftrace_pid_function(ip, parent_ip);
  99 }
 100
 101 static void set_ftrace_pid_function(ftrace_func_t func)
 102 {
 103         /* do not set ftrace_pid_function to itself! */
 104         if (func != ftrace_pid_func)
 105                 ftrace_pid_function = func;
 106 }
 107
 108 /**
 109  * clear_ftrace_function - reset the ftrace function
 110  *
 111  * This NULLs the ftrace function and in essence stops
 112  * tracing.  There may be lag
 113  */
 114 void clear_ftrace_function(void)
 115 {
 116         ftrace_trace_function = ftrace_stub;
 117         __ftrace_trace_function = ftrace_stub;
 118         ftrace_pid_function = ftrace_stub;
 119 }
 120
 121 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 122 /*
 123  * For those archs that do not test ftrace_trace_stop in their
 124  * mcount call site, we need to do it from C.
 125  */
 126 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 127 {
 128         if (function_trace_stop)
 129                 return;
 130
 131         __ftrace_trace_function(ip, parent_ip);
 132 }
 133 #endif
 134
 135 static int __register_ftrace_function(struct ftrace_ops *ops)
 136 {
 137         /* should not be called from interrupt context */
 138         spin_lock(&ftrace_lock);
 139
 140         ops->next = ftrace_list;
 141         /*
 142          * We are entering ops into the ftrace_list but another
 143          * CPU might be walking that list. We need to make sure
 144          * the ops->next pointer is valid before another CPU sees
 145          * the ops pointer included into the ftrace_list.
 146          */
 147         smp_wmb();
 148         ftrace_list = ops;
 149
 150         if (ftrace_enabled) {
 151                 ftrace_func_t func;
 152
 153                 if (ops->next == &ftrace_list_end)
 154                         func = ops->func;
 155                 else
 156                         func = ftrace_list_func;
 157
 158                 if (ftrace_pid_trace) {
 159                         set_ftrace_pid_function(func);
 160                         func = ftrace_pid_func;
 161                 }
 162
 163                 /*
 164                  * For one func, simply call it directly.
 165                  * For more than one func, call the chain.
 166                  */
 167 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 168                 ftrace_trace_function = func;
 169 #else
 170                 __ftrace_trace_function = func;
 171                 ftrace_trace_function = ftrace_test_stop_func;
 172 #endif
 173         }
 174
 175         spin_unlock(&ftrace_lock);
 176
 177         return 0;
 178 }
 179
 180 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 181 {
 182         struct ftrace_ops **p;
 183         int ret = 0;
 184
 185         /* should not be called from interrupt context */
 186         spin_lock(&ftrace_lock);
 187
 188         /*
 189          * If we are removing the last function, then simply point
 190          * to the ftrace_stub.
 191          */
 192         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 193                 ftrace_trace_function = ftrace_stub;
 194                 ftrace_list = &ftrace_list_end;
 195                 goto out;
 196         }
 197
 198         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 199                 if (*p == ops)
 200                         break;
 201
 202         if (*p != ops) {
 203                 ret = -1;
 204                 goto out;
 205         }
 206
 207         *p = (*p)->next;
 208
 209         if (ftrace_enabled) {
 210                 /* If we only have one func left, then call that directly */
 211                 if (ftrace_list->next == &ftrace_list_end) {
 212                         ftrace_func_t func = ftrace_list->func;
 213
 214                         if (ftrace_pid_trace) {
 215                                 set_ftrace_pid_function(func);
 216                                 func = ftrace_pid_func;
 217                         }
 218 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 219                         ftrace_trace_function = func;
 220 #else
 221                         __ftrace_trace_function = func;
 222 #endif
 223                 }
 224         }
 225
 226  out:
 227         spin_unlock(&ftrace_lock);
 228
 229         return ret;
 230 }
 231
 232 static void ftrace_update_pid_func(void)
 233 {
 234         ftrace_func_t func;
 235
 236         /* should not be called from interrupt context */
 237         spin_lock(&ftrace_lock);
 238
 239         if (ftrace_trace_function == ftrace_stub)
 240                 goto out;
 241
 242         func = ftrace_trace_function;
 243
 244         if (ftrace_pid_trace) {
 245                 set_ftrace_pid_function(func);
 246                 func = ftrace_pid_func;
 247         } else {
 248                 if (func == ftrace_pid_func)
 249                         func = ftrace_pid_function;
 250         }
 251
 252 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 253         ftrace_trace_function = func;
 254 #else
 255         __ftrace_trace_function = func;
 256 #endif
 257
 258  out:
 259         spin_unlock(&ftrace_lock);
 260 }
 261
 262 #ifdef CONFIG_DYNAMIC_FTRACE
 263 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 264 # error Dynamic ftrace depends on MCOUNT_RECORD
 265 #endif
 266
 267 enum {
 268         FTRACE_ENABLE_CALLS             = (1 << 0),
 269         FTRACE_DISABLE_CALLS            = (1 << 1),
 270         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 271         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 272         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 273         FTRACE_START_FUNC_RET           = (1 << 5),
 274         FTRACE_STOP_FUNC_RET            = (1 << 6),
 275 };
 276
 277 static int ftrace_filtered;
 278
 279 static LIST_HEAD(ftrace_new_addrs);
 280
 281 static DEFINE_MUTEX(ftrace_regex_lock);
 282
 283 struct ftrace_page {
 284         struct ftrace_page      *next;
 285         int                     index;
 286         struct dyn_ftrace       records[];
 287 };
 288
 289 #define ENTRIES_PER_PAGE \
 290   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 291
 292 /* estimate from running different kernels */
 293 #define NR_TO_INIT              10000
 294
 295 static struct ftrace_page       *ftrace_pages_start;
 296 static struct ftrace_page       *ftrace_pages;
 297
 298 static struct dyn_ftrace *ftrace_free_records;
 299
 300
 301 #ifdef CONFIG_KPROBES
 302
 303 static int frozen_record_count;
 304
 305 static inline void freeze_record(struct dyn_ftrace *rec)
 306 {
 307         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 308                 rec->flags |= FTRACE_FL_FROZEN;
 309                 frozen_record_count++;
 310         }
 311 }
 312
 313 static inline void unfreeze_record(struct dyn_ftrace *rec)
 314 {
 315         if (rec->flags & FTRACE_FL_FROZEN) {
 316                 rec->flags &= ~FTRACE_FL_FROZEN;
 317                 frozen_record_count--;
 318         }
 319 }
 320
 321 static inline int record_frozen(struct dyn_ftrace *rec)
 322 {
 323         return rec->flags & FTRACE_FL_FROZEN;
 324 }
 325 #else
 326 # define freeze_record(rec)                     ({ 0; })
 327 # define unfreeze_record(rec)                   ({ 0; })
 328 # define record_frozen(rec)                     ({ 0; })
 329 #endif /* CONFIG_KPROBES */
 330
 331 static void ftrace_free_rec(struct dyn_ftrace *rec)
 332 {
 333         rec->ip = (unsigned long)ftrace_free_records;
 334         ftrace_free_records = rec;
 335         rec->flags |= FTRACE_FL_FREE;
 336 }
 337
 338 void ftrace_release(void *start, unsigned long size)
 339 {
 340         struct dyn_ftrace *rec;
 341         struct ftrace_page *pg;
 342         unsigned long s = (unsigned long)start;
 343         unsigned long e = s + size;
 344         int i;
 345
 346         if (ftrace_disabled || !start)
 347                 return;
 348
 349         /* should not be called from interrupt context */
 350         spin_lock(&ftrace_lock);
 351
 352         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 353                 for (i = 0; i < pg->index; i++) {
 354                         rec = &pg->records[i];
 355
 356                         if ((rec->ip >= s) && (rec->ip < e))
 357                                 ftrace_free_rec(rec);
 358                 }
 359         }
 360         spin_unlock(&ftrace_lock);
 361 }
 362
 363 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 364 {
 365         struct dyn_ftrace *rec;
 366
 367         /* First check for freed records */
 368         if (ftrace_free_records) {
 369                 rec = ftrace_free_records;
 370
 371                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 372                         FTRACE_WARN_ON_ONCE(1);
 373                         ftrace_free_records = NULL;
 374                         return NULL;
 375                 }
 376
 377                 ftrace_free_records = (void *)rec->ip;
 378                 memset(rec, 0, sizeof(*rec));
 379                 return rec;
 380         }
 381
 382         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 383                 if (!ftrace_pages->next) {
 384                         /* allocate another page */
 385                         ftrace_pages->next =
 386                                 (void *)get_zeroed_page(GFP_KERNEL);
 387                         if (!ftrace_pages->next)
 388                                 return NULL;
 389                 }
 390                 ftrace_pages = ftrace_pages->next;
 391         }
 392
 393         return &ftrace_pages->records[ftrace_pages->index++];
 394 }
 395
 396 static struct dyn_ftrace *
 397 ftrace_record_ip(unsigned long ip)
 398 {
 399         struct dyn_ftrace *rec;
 400
 401         if (ftrace_disabled)
 402                 return NULL;
 403
 404         rec = ftrace_alloc_dyn_node(ip);
 405         if (!rec)
 406                 return NULL;
 407
 408         rec->ip = ip;
 409
 410         list_add(&rec->list, &ftrace_new_addrs);
 411
 412         return rec;
 413 }
 414
 415 static void print_ip_ins(const char *fmt, unsigned char *p)
 416 {
 417         int i;
 418
 419         printk(KERN_CONT "%s", fmt);
 420
 421         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 422                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 423 }
 424
 425 static void ftrace_bug(int failed, unsigned long ip)
 426 {
 427         switch (failed) {
 428         case -EFAULT:
 429                 FTRACE_WARN_ON_ONCE(1);
 430                 pr_info("ftrace faulted on modifying ");
 431                 print_ip_sym(ip);
 432                 break;
 433         case -EINVAL:
 434                 FTRACE_WARN_ON_ONCE(1);
 435                 pr_info("ftrace failed to modify ");
 436                 print_ip_sym(ip);
 437                 print_ip_ins(" actual: ", (unsigned char *)ip);
 438                 printk(KERN_CONT "\n");
 439                 break;
 440         case -EPERM:
 441                 FTRACE_WARN_ON_ONCE(1);
 442                 pr_info("ftrace faulted on writing ");
 443                 print_ip_sym(ip);
 444                 break;
 445         default:
 446                 FTRACE_WARN_ON_ONCE(1);
 447                 pr_info("ftrace faulted on unknown error ");
 448                 print_ip_sym(ip);
 449         }
 450 }
 451
 452
 453 static int
 454 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 455 {
 456         unsigned long ip, fl;
 457         unsigned long ftrace_addr;
 458
 459         ftrace_addr = (unsigned long)FTRACE_ADDR;
 460
 461         ip = rec->ip;
 462
 463         /*
 464          * If this record is not to be traced and
 465          * it is not enabled then do nothing.
 466          *
 467          * If this record is not to be traced and
 468          * it is enabled then disable it.
 469          *
 470          */
 471         if (rec->flags & FTRACE_FL_NOTRACE) {
 472                 if (rec->flags & FTRACE_FL_ENABLED)
 473                         rec->flags &= ~FTRACE_FL_ENABLED;
 474                 else
 475                         return 0;
 476
 477         } else if (ftrace_filtered && enable) {
 478                 /*
 479                  * Filtering is on:
 480                  */
 481
 482                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
 483
 484                 /* Record is filtered and enabled, do nothing */
 485                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
 486                         return 0;
 487
 488                 /* Record is not filtered or enabled, do nothing */
 489                 if (!fl)
 490                         return 0;
 491
 492                 /* Record is not filtered but enabled, disable it */
 493                 if (fl == FTRACE_FL_ENABLED)
 494                         rec->flags &= ~FTRACE_FL_ENABLED;
 495                 else
 496                 /* Otherwise record is filtered but not enabled, enable it */
 497                         rec->flags |= FTRACE_FL_ENABLED;
 498         } else {
 499                 /* Disable or not filtered */
 500
 501                 if (enable) {
 502                         /* if record is enabled, do nothing */
 503                         if (rec->flags & FTRACE_FL_ENABLED)
 504                                 return 0;
 505
 506                         rec->flags |= FTRACE_FL_ENABLED;
 507
 508                 } else {
 509
 510                         /* if record is not enabled, do nothing */
 511                         if (!(rec->flags & FTRACE_FL_ENABLED))
 512                                 return 0;
 513
 514                         rec->flags &= ~FTRACE_FL_ENABLED;
 515                 }
 516         }
 517
 518         if (rec->flags & FTRACE_FL_ENABLED)
 519                 return ftrace_make_call(rec, ftrace_addr);
 520         else
 521                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 522 }
 523
 524 static void ftrace_replace_code(int enable)
 525 {
 526         int i, failed;
 527         struct dyn_ftrace *rec;
 528         struct ftrace_page *pg;
 529
 530         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 531                 for (i = 0; i < pg->index; i++) {
 532                         rec = &pg->records[i];
 533
 534                         /*
 535                          * Skip over free records and records that have
 536                          * failed.
 537                          */
 538                         if (rec->flags & FTRACE_FL_FREE ||
 539                             rec->flags & FTRACE_FL_FAILED)
 540                                 continue;
 541
 542                         /* ignore updates to this record's mcount site */
 543                         if (get_kprobe((void *)rec->ip)) {
 544                                 freeze_record(rec);
 545                                 continue;
 546                         } else {
 547                                 unfreeze_record(rec);
 548                         }
 549
 550                         failed = __ftrace_replace_code(rec, enable);
 551                         if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 552                                 rec->flags |= FTRACE_FL_FAILED;
 553                                 if ((system_state == SYSTEM_BOOTING) ||
 554                                     !core_kernel_text(rec->ip)) {
 555                                         ftrace_free_rec(rec);
 556                                 } else
 557                                         ftrace_bug(failed, rec->ip);
 558                         }
 559                 }
 560         }
 561 }
 562
 563 static int
 564 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 565 {
 566         unsigned long ip;
 567         int ret;
 568
 569         ip = rec->ip;
 570
 571         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
 572         if (ret) {
 573                 ftrace_bug(ret, ip);
 574                 rec->flags |= FTRACE_FL_FAILED;
 575                 return 0;
 576         }
 577         return 1;
 578 }
 579
 580 static int __ftrace_modify_code(void *data)
 581 {
 582         int *command = data;
 583
 584         if (*command & FTRACE_ENABLE_CALLS)
 585                 ftrace_replace_code(1);
 586         else if (*command & FTRACE_DISABLE_CALLS)
 587                 ftrace_replace_code(0);
 588
 589         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 590                 ftrace_update_ftrace_func(ftrace_trace_function);
 591
 592         if (*command & FTRACE_START_FUNC_RET)
 593                 ftrace_enable_ftrace_graph_caller();
 594         else if (*command & FTRACE_STOP_FUNC_RET)
 595                 ftrace_disable_ftrace_graph_caller();
 596
 597         return 0;
 598 }
 599
 600 static void ftrace_run_update_code(int command)
 601 {
 602         stop_machine(__ftrace_modify_code, &command, NULL);
 603 }
 604
 605 static ftrace_func_t saved_ftrace_func;
 606 static int ftrace_start_up;
 607
 608 static void ftrace_startup_enable(int command)
 609 {
 610         if (saved_ftrace_func != ftrace_trace_function) {
 611                 saved_ftrace_func = ftrace_trace_function;
 612                 command |= FTRACE_UPDATE_TRACE_FUNC;
 613         }
 614
 615         if (!command || !ftrace_enabled)
 616                 return;
 617
 618         ftrace_run_update_code(command);
 619 }
 620
 621 static void ftrace_startup(int command)
 622 {
 623         if (unlikely(ftrace_disabled))
 624                 return;
 625
 626         mutex_lock(&ftrace_start_lock);
 627         ftrace_start_up++;
 628         command |= FTRACE_ENABLE_CALLS;
 629
 630         ftrace_startup_enable(command);
 631
 632         mutex_unlock(&ftrace_start_lock);
 633 }
 634
 635 static void ftrace_shutdown(int command)
 636 {
 637         if (unlikely(ftrace_disabled))
 638                 return;
 639
 640         mutex_lock(&ftrace_start_lock);
 641         ftrace_start_up--;
 642         if (!ftrace_start_up)
 643                 command |= FTRACE_DISABLE_CALLS;
 644
 645         if (saved_ftrace_func != ftrace_trace_function) {
 646                 saved_ftrace_func = ftrace_trace_function;
 647                 command |= FTRACE_UPDATE_TRACE_FUNC;
 648         }
 649
 650         if (!command || !ftrace_enabled)
 651                 goto out;
 652
 653         ftrace_run_update_code(command);
 654  out:
 655         mutex_unlock(&ftrace_start_lock);
 656 }
 657
 658 static void ftrace_startup_sysctl(void)
 659 {
 660         int command = FTRACE_ENABLE_MCOUNT;
 661
 662         if (unlikely(ftrace_disabled))
 663                 return;
 664
 665         mutex_lock(&ftrace_start_lock);
 666         /* Force update next time */
 667         saved_ftrace_func = NULL;
 668         /* ftrace_start_up is true if we want ftrace running */
 669         if (ftrace_start_up)
 670                 command |= FTRACE_ENABLE_CALLS;
 671
 672         ftrace_run_update_code(command);
 673         mutex_unlock(&ftrace_start_lock);
 674 }
 675
 676 static void ftrace_shutdown_sysctl(void)
 677 {
 678         int command = FTRACE_DISABLE_MCOUNT;
 679
 680         if (unlikely(ftrace_disabled))
 681                 return;
 682
 683         mutex_lock(&ftrace_start_lock);
 684         /* ftrace_start_up is true if ftrace is running */
 685         if (ftrace_start_up)
 686                 command |= FTRACE_DISABLE_CALLS;
 687
 688         ftrace_run_update_code(command);
 689         mutex_unlock(&ftrace_start_lock);
 690 }
 691
 692 static cycle_t          ftrace_update_time;
 693 static unsigned long    ftrace_update_cnt;
 694 unsigned long           ftrace_update_tot_cnt;
 695
 696 static int ftrace_update_code(struct module *mod)
 697 {
 698         struct dyn_ftrace *p, *t;
 699         cycle_t start, stop;
 700
 701         start = ftrace_now(raw_smp_processor_id());
 702         ftrace_update_cnt = 0;
 703
 704         list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
 705
 706                 /* If something went wrong, bail without enabling anything */
 707                 if (unlikely(ftrace_disabled))
 708                         return -1;
 709
 710                 list_del_init(&p->list);
 711
 712                 /* convert record (i.e, patch mcount-call with NOP) */
 713                 if (ftrace_code_disable(mod, p)) {
 714                         p->flags |= FTRACE_FL_CONVERTED;
 715                         ftrace_update_cnt++;
 716                 } else
 717                         ftrace_free_rec(p);
 718         }
 719
 720         stop = ftrace_now(raw_smp_processor_id());
 721         ftrace_update_time = stop - start;
 722         ftrace_update_tot_cnt += ftrace_update_cnt;
 723
 724         return 0;
 725 }
 726
 727 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 728 {
 729         struct ftrace_page *pg;
 730         int cnt;
 731         int i;
 732
 733         /* allocate a few pages */
 734         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 735         if (!ftrace_pages_start)
 736                 return -1;
 737
 738         /*
 739          * Allocate a few more pages.
 740          *
 741          * TODO: have some parser search vmlinux before
 742          *   final linking to find all calls to ftrace.
 743          *   Then we can:
 744          *    a) know how many pages to allocate.
 745          *     and/or
 746          *    b) set up the table then.
 747          *
 748          *  The dynamic code is still necessary for
 749          *  modules.
 750          */
 751
 752         pg = ftrace_pages = ftrace_pages_start;
 753
 754         cnt = num_to_init / ENTRIES_PER_PAGE;
 755         pr_info("ftrace: allocating %ld entries in %d pages\n",
 756                 num_to_init, cnt + 1);
 757
 758         for (i = 0; i < cnt; i++) {
 759                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 760
 761                 /* If we fail, we'll try later anyway */
 762                 if (!pg->next)
 763                         break;
 764
 765                 pg = pg->next;
 766         }
 767
 768         return 0;
 769 }
 770
 771 enum {
 772         FTRACE_ITER_FILTER      = (1 << 0),
 773         FTRACE_ITER_CONT        = (1 << 1),
 774         FTRACE_ITER_NOTRACE     = (1 << 2),
 775         FTRACE_ITER_FAILURES    = (1 << 3),
 776 };
 777
 778 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 779
 780 struct ftrace_iterator {
 781         struct ftrace_page      *pg;
 782         int                     idx;
 783         unsigned                flags;
 784         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 785         unsigned                buffer_idx;
 786         unsigned                filtered;
 787 };
 788
 789 static void *
 790 t_next(struct seq_file *m, void *v, loff_t *pos)
 791 {
 792         struct ftrace_iterator *iter = m->private;
 793         struct dyn_ftrace *rec = NULL;
 794
 795         (*pos)++;
 796
 797         /* should not be called from interrupt context */
 798         spin_lock(&ftrace_lock);
 799  retry:
 800         if (iter->idx >= iter->pg->index) {
 801                 if (iter->pg->next) {
 802                         iter->pg = iter->pg->next;
 803                         iter->idx = 0;
 804                         goto retry;
 805                 } else {
 806                         iter->idx = -1;
 807                 }
 808         } else {
 809                 rec = &iter->pg->records[iter->idx++];
 810                 if ((rec->flags & FTRACE_FL_FREE) ||
 811
 812                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 813                      (rec->flags & FTRACE_FL_FAILED)) ||
 814
 815                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 816                      !(rec->flags & FTRACE_FL_FAILED)) ||
 817
 818                     ((iter->flags & FTRACE_ITER_FILTER) &&
 819                      !(rec->flags & FTRACE_FL_FILTER)) ||
 820
 821                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 822                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 823                         rec = NULL;
 824                         goto retry;
 825                 }
 826         }
 827         spin_unlock(&ftrace_lock);
 828
 829         return rec;
 830 }
 831
 832 static void *t_start(struct seq_file *m, loff_t *pos)
 833 {
 834         struct ftrace_iterator *iter = m->private;
 835         void *p = NULL;
 836
 837         if (*pos > 0) {
 838                 if (iter->idx < 0)
 839                         return p;
 840                 (*pos)--;
 841                 iter->idx--;
 842         }
 843
 844         p = t_next(m, p, pos);
 845
 846         return p;
 847 }
 848
 849 static void t_stop(struct seq_file *m, void *p)
 850 {
 851 }
 852
 853 static int t_show(struct seq_file *m, void *v)
 854 {
 855         struct dyn_ftrace *rec = v;
 856         char str[KSYM_SYMBOL_LEN];
 857
 858         if (!rec)
 859                 return 0;
 860
 861         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 862
 863         seq_printf(m, "%s\n", str);
 864
 865         return 0;
 866 }
 867
 868 static struct seq_operations show_ftrace_seq_ops = {
 869         .start = t_start,
 870         .next = t_next,
 871         .stop = t_stop,
 872         .show = t_show,
 873 };
 874
 875 static int
 876 ftrace_avail_open(struct inode *inode, struct file *file)
 877 {
 878         struct ftrace_iterator *iter;
 879         int ret;
 880
 881         if (unlikely(ftrace_disabled))
 882                 return -ENODEV;
 883
 884         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 885         if (!iter)
 886                 return -ENOMEM;
 887
 888         iter->pg = ftrace_pages_start;
 889
 890         ret = seq_open(file, &show_ftrace_seq_ops);
 891         if (!ret) {
 892                 struct seq_file *m = file->private_data;
 893
 894                 m->private = iter;
 895         } else {
 896                 kfree(iter);
 897         }
 898
 899         return ret;
 900 }
 901
 902 int ftrace_avail_release(struct inode *inode, struct file *file)
 903 {
 904         struct seq_file *m = (struct seq_file *)file->private_data;
 905         struct ftrace_iterator *iter = m->private;
 906
 907         seq_release(inode, file);
 908         kfree(iter);
 909
 910         return 0;
 911 }
 912
 913 static int
 914 ftrace_failures_open(struct inode *inode, struct file *file)
 915 {
 916         int ret;
 917         struct seq_file *m;
 918         struct ftrace_iterator *iter;
 919
 920         ret = ftrace_avail_open(inode, file);
 921         if (!ret) {
 922                 m = (struct seq_file *)file->private_data;
 923                 iter = (struct ftrace_iterator *)m->private;
 924                 iter->flags = FTRACE_ITER_FAILURES;
 925         }
 926
 927         return ret;
 928 }
 929
 930
 931 static void ftrace_filter_reset(int enable)
 932 {
 933         struct ftrace_page *pg;
 934         struct dyn_ftrace *rec;
 935         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
 936         unsigned i;
 937
 938         /* should not be called from interrupt context */
 939         spin_lock(&ftrace_lock);
 940         if (enable)
 941                 ftrace_filtered = 0;
 942         pg = ftrace_pages_start;
 943         while (pg) {
 944                 for (i = 0; i < pg->index; i++) {
 945                         rec = &pg->records[i];
 946                         if (rec->flags & FTRACE_FL_FAILED)
 947                                 continue;
 948                         rec->flags &= ~type;
 949                 }
 950                 pg = pg->next;
 951         }
 952         spin_unlock(&ftrace_lock);
 953 }
 954
 955 static int
 956 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
 957 {
 958         struct ftrace_iterator *iter;
 959         int ret = 0;
 960
 961         if (unlikely(ftrace_disabled))
 962                 return -ENODEV;
 963
 964         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 965         if (!iter)
 966                 return -ENOMEM;
 967
 968         mutex_lock(&ftrace_regex_lock);
 969         if ((file->f_mode & FMODE_WRITE) &&
 970             !(file->f_flags & O_APPEND))
 971                 ftrace_filter_reset(enable);
 972
 973         if (file->f_mode & FMODE_READ) {
 974                 iter->pg = ftrace_pages_start;
 975                 iter->flags = enable ? FTRACE_ITER_FILTER :
 976                         FTRACE_ITER_NOTRACE;
 977
 978                 ret = seq_open(file, &show_ftrace_seq_ops);
 979                 if (!ret) {
 980                         struct seq_file *m = file->private_data;
 981                         m->private = iter;
 982                 } else
 983                         kfree(iter);
 984         } else
 985                 file->private_data = iter;
 986         mutex_unlock(&ftrace_regex_lock);
 987
 988         return ret;
 989 }
 990
 991 static int
 992 ftrace_filter_open(struct inode *inode, struct file *file)
 993 {
 994         return ftrace_regex_open(inode, file, 1);
 995 }
 996
 997 static int
 998 ftrace_notrace_open(struct inode *inode, struct file *file)
 999 {
1000         return ftrace_regex_open(inode, file, 0);
1001 }
1002
1003 static ssize_t
1004 ftrace_regex_read(struct file *file, char __user *ubuf,
1005                        size_t cnt, loff_t *ppos)
1006 {
1007         if (file->f_mode & FMODE_READ)
1008                 return seq_read(file, ubuf, cnt, ppos);
1009         else
1010                 return -EPERM;
1011 }
1012
1013 static loff_t
1014 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1015 {
1016         loff_t ret;
1017
1018         if (file->f_mode & FMODE_READ)
1019                 ret = seq_lseek(file, offset, origin);
1020         else
1021                 file->f_pos = ret = 1;
1022
1023         return ret;
1024 }
1025
1026 enum {
1027         MATCH_FULL,
1028         MATCH_FRONT_ONLY,
1029         MATCH_MIDDLE_ONLY,
1030         MATCH_END_ONLY,
1031 };
1032
1033 static void
1034 ftrace_match(unsigned char *buff, int len, int enable)
1035 {
1036         char str[KSYM_SYMBOL_LEN];
1037         char *search = NULL;
1038         struct ftrace_page *pg;
1039         struct dyn_ftrace *rec;
1040         int type = MATCH_FULL;
1041         unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1042         unsigned i, match = 0, search_len = 0;
1043         int not = 0;
1044
1045         if (buff[0] == '!') {
1046                 not = 1;
1047                 buff++;
1048                 len--;
1049         }
1050
1051         for (i = 0; i < len; i++) {
1052                 if (buff[i] == '*') {
1053                         if (!i) {
1054                                 search = buff + i + 1;
1055                                 type = MATCH_END_ONLY;
1056                                 search_len = len - (i + 1);
1057                         } else {
1058                                 if (type == MATCH_END_ONLY) {
1059                                         type = MATCH_MIDDLE_ONLY;
1060                                 } else {
1061                                         match = i;
1062                                         type = MATCH_FRONT_ONLY;
1063                                 }
1064                                 buff[i] = 0;
1065                                 break;
1066                         }
1067                 }
1068         }
1069
1070         /* should not be called from interrupt context */
1071         spin_lock(&ftrace_lock);
1072         if (enable)
1073                 ftrace_filtered = 1;
1074         pg = ftrace_pages_start;
1075         while (pg) {
1076                 for (i = 0; i < pg->index; i++) {
1077                         int matched = 0;
1078                         char *ptr;
1079
1080                         rec = &pg->records[i];
1081                         if (rec->flags & FTRACE_FL_FAILED)
1082                                 continue;
1083                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1084                         switch (type) {
1085                         case MATCH_FULL:
1086                                 if (strcmp(str, buff) == 0)
1087                                         matched = 1;
1088                                 break;
1089                         case MATCH_FRONT_ONLY:
1090                                 if (memcmp(str, buff, match) == 0)
1091                                         matched = 1;
1092                                 break;
1093                         case MATCH_MIDDLE_ONLY:
1094                                 if (strstr(str, search))
1095                                         matched = 1;
1096                                 break;
1097                         case MATCH_END_ONLY:
1098                                 ptr = strstr(str, search);
1099                                 if (ptr && (ptr[search_len] == 0))
1100                                         matched = 1;
1101                                 break;
1102                         }
1103                         if (matched) {
1104                                 if (not)
1105                                         rec->flags &= ~flag;
1106                                 else
1107                                         rec->flags |= flag;
1108                         }
1109                 }
1110                 pg = pg->next;
1111         }
1112         spin_unlock(&ftrace_lock);
1113 }
1114
1115 static ssize_t
1116 ftrace_regex_write(struct file *file, const char __user *ubuf,
1117                    size_t cnt, loff_t *ppos, int enable)
1118 {
1119         struct ftrace_iterator *iter;
1120         char ch;
1121         size_t read = 0;
1122         ssize_t ret;
1123
1124         if (!cnt || cnt < 0)
1125                 return 0;
1126
1127         mutex_lock(&ftrace_regex_lock);
1128
1129         if (file->f_mode & FMODE_READ) {
1130                 struct seq_file *m = file->private_data;
1131                 iter = m->private;
1132         } else
1133                 iter = file->private_data;
1134
1135         if (!*ppos) {
1136                 iter->flags &= ~FTRACE_ITER_CONT;
1137                 iter->buffer_idx = 0;
1138         }
1139
1140         ret = get_user(ch, ubuf++);
1141         if (ret)
1142                 goto out;
1143         read++;
1144         cnt--;
1145
1146         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1147                 /* skip white space */
1148                 while (cnt && isspace(ch)) {
1149                         ret = get_user(ch, ubuf++);
1150                         if (ret)
1151                                 goto out;
1152                         read++;
1153                         cnt--;
1154                 }
1155
1156                 if (isspace(ch)) {
1157                         file->f_pos += read;
1158                         ret = read;
1159                         goto out;
1160                 }
1161
1162                 iter->buffer_idx = 0;
1163         }
1164
1165         while (cnt && !isspace(ch)) {
1166                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1167                         iter->buffer[iter->buffer_idx++] = ch;
1168                 else {
1169                         ret = -EINVAL;
1170                         goto out;
1171                 }
1172                 ret = get_user(ch, ubuf++);
1173                 if (ret)
1174                         goto out;
1175                 read++;
1176                 cnt--;
1177         }
1178
1179         if (isspace(ch)) {
1180                 iter->filtered++;
1181                 iter->buffer[iter->buffer_idx] = 0;
1182                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1183                 iter->buffer_idx = 0;
1184         } else
1185                 iter->flags |= FTRACE_ITER_CONT;
1186
1187
1188         file->f_pos += read;
1189
1190         ret = read;
1191  out:
1192         mutex_unlock(&ftrace_regex_lock);
1193
1194         return ret;
1195 }
1196
1197 static ssize_t
1198 ftrace_filter_write(struct file *file, const char __user *ubuf,
1199                     size_t cnt, loff_t *ppos)
1200 {
1201         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1202 }
1203
1204 static ssize_t
1205 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1206                      size_t cnt, loff_t *ppos)
1207 {
1208         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1209 }
1210
1211 static void
1212 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1213 {
1214         if (unlikely(ftrace_disabled))
1215                 return;
1216
1217         mutex_lock(&ftrace_regex_lock);
1218         if (reset)
1219                 ftrace_filter_reset(enable);
1220         if (buf)
1221                 ftrace_match(buf, len, enable);
1222         mutex_unlock(&ftrace_regex_lock);
1223 }
1224
1225 /**
1226  * ftrace_set_filter - set a function to filter on in ftrace
1227  * @buf - the string that holds the function filter text.
1228  * @len - the length of the string.
1229  * @reset - non zero to reset all filters before applying this filter.
1230  *
1231  * Filters denote which functions should be enabled when tracing is enabled.
1232  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1233  */
1234 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1235 {
1236         ftrace_set_regex(buf, len, reset, 1);
1237 }
1238
1239 /**
1240  * ftrace_set_notrace - set a function to not trace in ftrace
1241  * @buf - the string that holds the function notrace text.
1242  * @len - the length of the string.
1243  * @reset - non zero to reset all filters before applying this filter.
1244  *
1245  * Notrace Filters denote which functions should not be enabled when tracing
1246  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1247  * for tracing.
1248  */
1249 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1250 {
1251         ftrace_set_regex(buf, len, reset, 0);
1252 }
1253
1254 static int
1255 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1256 {
1257         struct seq_file *m = (struct seq_file *)file->private_data;
1258         struct ftrace_iterator *iter;
1259
1260         mutex_lock(&ftrace_regex_lock);
1261         if (file->f_mode & FMODE_READ) {
1262                 iter = m->private;
1263
1264                 seq_release(inode, file);
1265         } else
1266                 iter = file->private_data;
1267
1268         if (iter->buffer_idx) {
1269                 iter->filtered++;
1270                 iter->buffer[iter->buffer_idx] = 0;
1271                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1272         }
1273
1274         mutex_lock(&ftrace_sysctl_lock);
1275         mutex_lock(&ftrace_start_lock);
1276         if (ftrace_start_up && ftrace_enabled)
1277                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1278         mutex_unlock(&ftrace_start_lock);
1279         mutex_unlock(&ftrace_sysctl_lock);
1280
1281         kfree(iter);
1282         mutex_unlock(&ftrace_regex_lock);
1283         return 0;
1284 }
1285
1286 static int
1287 ftrace_filter_release(struct inode *inode, struct file *file)
1288 {
1289         return ftrace_regex_release(inode, file, 1);
1290 }
1291
1292 static int
1293 ftrace_notrace_release(struct inode *inode, struct file *file)
1294 {
1295         return ftrace_regex_release(inode, file, 0);
1296 }
1297
1298 static struct file_operations ftrace_avail_fops = {
1299         .open = ftrace_avail_open,
1300         .read = seq_read,
1301         .llseek = seq_lseek,
1302         .release = ftrace_avail_release,
1303 };
1304
1305 static struct file_operations ftrace_failures_fops = {
1306         .open = ftrace_failures_open,
1307         .read = seq_read,
1308         .llseek = seq_lseek,
1309         .release = ftrace_avail_release,
1310 };
1311
1312 static struct file_operations ftrace_filter_fops = {
1313         .open = ftrace_filter_open,
1314         .read = ftrace_regex_read,
1315         .write = ftrace_filter_write,
1316         .llseek = ftrace_regex_lseek,
1317         .release = ftrace_filter_release,
1318 };
1319
1320 static struct file_operations ftrace_notrace_fops = {
1321         .open = ftrace_notrace_open,
1322         .read = ftrace_regex_read,
1323         .write = ftrace_notrace_write,
1324         .llseek = ftrace_regex_lseek,
1325         .release = ftrace_notrace_release,
1326 };
1327
1328 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1329
1330 static DEFINE_MUTEX(graph_lock);
1331
1332 int ftrace_graph_count;
1333 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1334
1335 static void *
1336 g_next(struct seq_file *m, void *v, loff_t *pos)
1337 {
1338         unsigned long *array = m->private;
1339         int index = *pos;
1340
1341         (*pos)++;
1342
1343         if (index >= ftrace_graph_count)
1344                 return NULL;
1345
1346         return &array[index];
1347 }
1348
1349 static void *g_start(struct seq_file *m, loff_t *pos)
1350 {
1351         void *p = NULL;
1352
1353         mutex_lock(&graph_lock);
1354
1355         p = g_next(m, p, pos);
1356
1357         return p;
1358 }
1359
1360 static void g_stop(struct seq_file *m, void *p)
1361 {
1362         mutex_unlock(&graph_lock);
1363 }
1364
1365 static int g_show(struct seq_file *m, void *v)
1366 {
1367         unsigned long *ptr = v;
1368         char str[KSYM_SYMBOL_LEN];
1369
1370         if (!ptr)
1371                 return 0;
1372
1373         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1374
1375         seq_printf(m, "%s\n", str);
1376
1377         return 0;
1378 }
1379
1380 static struct seq_operations ftrace_graph_seq_ops = {
1381         .start = g_start,
1382         .next = g_next,
1383         .stop = g_stop,
1384         .show = g_show,
1385 };
1386
1387 static int
1388 ftrace_graph_open(struct inode *inode, struct file *file)
1389 {
1390         int ret = 0;
1391
1392         if (unlikely(ftrace_disabled))
1393                 return -ENODEV;
1394
1395         mutex_lock(&graph_lock);
1396         if ((file->f_mode & FMODE_WRITE) &&
1397             !(file->f_flags & O_APPEND)) {
1398                 ftrace_graph_count = 0;
1399                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1400         }
1401
1402         if (file->f_mode & FMODE_READ) {
1403                 ret = seq_open(file, &ftrace_graph_seq_ops);
1404                 if (!ret) {
1405                         struct seq_file *m = file->private_data;
1406                         m->private = ftrace_graph_funcs;
1407                 }
1408         } else
1409                 file->private_data = ftrace_graph_funcs;
1410         mutex_unlock(&graph_lock);
1411
1412         return ret;
1413 }
1414
1415 static ssize_t
1416 ftrace_graph_read(struct file *file, char __user *ubuf,
1417                        size_t cnt, loff_t *ppos)
1418 {
1419         if (file->f_mode & FMODE_READ)
1420                 return seq_read(file, ubuf, cnt, ppos);
1421         else
1422                 return -EPERM;
1423 }
1424
1425 static int
1426 ftrace_set_func(unsigned long *array, int idx, char *buffer)
1427 {
1428         char str[KSYM_SYMBOL_LEN];
1429         struct dyn_ftrace *rec;
1430         struct ftrace_page *pg;
1431         int found = 0;
1432         int i, j;
1433
1434         if (ftrace_disabled)
1435                 return -ENODEV;
1436
1437         /* should not be called from interrupt context */
1438         spin_lock(&ftrace_lock);
1439
1440         for (pg = ftrace_pages_start; pg; pg = pg->next) {
1441                 for (i = 0; i < pg->index; i++) {
1442                         rec = &pg->records[i];
1443
1444                         if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
1445                                 continue;
1446
1447                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1448                         if (strcmp(str, buffer) == 0) {
1449                                 found = 1;
1450                                 for (j = 0; j < idx; j++)
1451                                         if (array[j] == rec->ip) {
1452                                                 found = 0;
1453                                                 break;
1454                                         }
1455                                 if (found)
1456                                         array[idx] = rec->ip;
1457                                 break;
1458                         }
1459                 }
1460         }
1461         spin_unlock(&ftrace_lock);
1462
1463         return found ? 0 : -EINVAL;
1464 }
1465
1466 static ssize_t
1467 ftrace_graph_write(struct file *file, const char __user *ubuf,
1468                    size_t cnt, loff_t *ppos)
1469 {
1470         unsigned char buffer[FTRACE_BUFF_MAX+1];
1471         unsigned long *array;
1472         size_t read = 0;
1473         ssize_t ret;
1474         int index = 0;
1475         char ch;
1476
1477         if (!cnt || cnt < 0)
1478                 return 0;
1479
1480         mutex_lock(&graph_lock);
1481
1482         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
1483                 ret = -EBUSY;
1484                 goto out;
1485         }
1486
1487         if (file->f_mode & FMODE_READ) {
1488                 struct seq_file *m = file->private_data;
1489                 array = m->private;
1490         } else
1491                 array = file->private_data;
1492
1493         ret = get_user(ch, ubuf++);
1494         if (ret)
1495                 goto out;
1496         read++;
1497         cnt--;
1498
1499         /* skip white space */
1500         while (cnt && isspace(ch)) {
1501                 ret = get_user(ch, ubuf++);
1502                 if (ret)
1503                         goto out;
1504                 read++;
1505                 cnt--;
1506         }
1507
1508         if (isspace(ch)) {
1509                 *ppos += read;
1510                 ret = read;
1511                 goto out;
1512         }
1513
1514         while (cnt && !isspace(ch)) {
1515                 if (index < FTRACE_BUFF_MAX)
1516                         buffer[index++] = ch;
1517                 else {
1518                         ret = -EINVAL;
1519                         goto out;
1520                 }
1521                 ret = get_user(ch, ubuf++);
1522                 if (ret)
1523                         goto out;
1524                 read++;
1525                 cnt--;
1526         }
1527         buffer[index] = 0;
1528
1529         /* we allow only one at a time */
1530         ret = ftrace_set_func(array, ftrace_graph_count, buffer);
1531         if (ret)
1532                 goto out;
1533
1534         ftrace_graph_count++;
1535
1536         file->f_pos += read;
1537
1538         ret = read;
1539  out:
1540         mutex_unlock(&graph_lock);
1541
1542         return ret;
1543 }
1544
1545 static const struct file_operations ftrace_graph_fops = {
1546         .open = ftrace_graph_open,
1547         .read = ftrace_graph_read,
1548         .write = ftrace_graph_write,
1549 };
1550 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1551
1552 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
1553 {
1554         struct dentry *entry;
1555
1556         entry = debugfs_create_file("available_filter_functions", 0444,
1557                                     d_tracer, NULL, &ftrace_avail_fops);
1558         if (!entry)
1559                 pr_warning("Could not create debugfs "
1560                            "'available_filter_functions' entry\n");
1561
1562         entry = debugfs_create_file("failures", 0444,
1563                                     d_tracer, NULL, &ftrace_failures_fops);
1564         if (!entry)
1565                 pr_warning("Could not create debugfs 'failures' entry\n");
1566
1567         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1568                                     NULL, &ftrace_filter_fops);
1569         if (!entry)
1570                 pr_warning("Could not create debugfs "
1571                            "'set_ftrace_filter' entry\n");
1572
1573         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1574                                     NULL, &ftrace_notrace_fops);
1575         if (!entry)
1576                 pr_warning("Could not create debugfs "
1577                            "'set_ftrace_notrace' entry\n");
1578
1579 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1580         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
1581                                     NULL,
1582                                     &ftrace_graph_fops);
1583         if (!entry)
1584                 pr_warning("Could not create debugfs "
1585                            "'set_graph_function' entry\n");
1586 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1587
1588         return 0;
1589 }
1590
1591 static int ftrace_convert_nops(struct module *mod,
1592                                unsigned long *start,
1593                                unsigned long *end)
1594 {
1595         unsigned long *p;
1596         unsigned long addr;
1597         unsigned long flags;
1598
1599         mutex_lock(&ftrace_start_lock);
1600         p = start;
1601         while (p < end) {
1602                 addr = ftrace_call_adjust(*p++);
1603                 /*
1604                  * Some architecture linkers will pad between
1605                  * the different mcount_loc sections of different
1606                  * object files to satisfy alignments.
1607                  * Skip any NULL pointers.
1608                  */
1609                 if (!addr)
1610                         continue;
1611                 ftrace_record_ip(addr);
1612         }
1613
1614         /* disable interrupts to prevent kstop machine */
1615         local_irq_save(flags);
1616         ftrace_update_code(mod);
1617         local_irq_restore(flags);
1618         mutex_unlock(&ftrace_start_lock);
1619
1620         return 0;
1621 }
1622
1623 void ftrace_init_module(struct module *mod,
1624                         unsigned long *start, unsigned long *end)
1625 {
1626         if (ftrace_disabled || start == end)
1627                 return;
1628         ftrace_convert_nops(mod, start, end);
1629 }
1630
1631 extern unsigned long __start_mcount_loc[];
1632 extern unsigned long __stop_mcount_loc[];
1633
1634 void __init ftrace_init(void)
1635 {
1636         unsigned long count, addr, flags;
1637         int ret;
1638
1639         /* Keep the ftrace pointer to the stub */
1640         addr = (unsigned long)ftrace_stub;
1641
1642         local_irq_save(flags);
1643         ftrace_dyn_arch_init(&addr);
1644         local_irq_restore(flags);
1645
1646         /* ftrace_dyn_arch_init places the return code in addr */
1647         if (addr)
1648                 goto failed;
1649
1650         count = __stop_mcount_loc - __start_mcount_loc;
1651
1652         ret = ftrace_dyn_table_alloc(count);
1653         if (ret)
1654                 goto failed;
1655
1656         last_ftrace_enabled = ftrace_enabled = 1;
1657
1658         ret = ftrace_convert_nops(NULL,
1659                                   __start_mcount_loc,
1660                                   __stop_mcount_loc);
1661
1662         return;
1663  failed:
1664         ftrace_disabled = 1;
1665 }
1666
1667 #else
1668
1669 static int __init ftrace_nodyn_init(void)
1670 {
1671         ftrace_enabled = 1;
1672         return 0;
1673 }
1674 device_initcall(ftrace_nodyn_init);
1675
1676 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
1677 static inline void ftrace_startup_enable(int command) { }
1678 /* Keep as macros so we do not need to define the commands */
1679 # define ftrace_startup(command)        do { } while (0)
1680 # define ftrace_shutdown(command)       do { } while (0)
1681 # define ftrace_startup_sysctl()        do { } while (0)
1682 # define ftrace_shutdown_sysctl()       do { } while (0)
1683 #endif /* CONFIG_DYNAMIC_FTRACE */
1684
1685 static ssize_t
1686 ftrace_pid_read(struct file *file, char __user *ubuf,
1687                        size_t cnt, loff_t *ppos)
1688 {
1689         char buf[64];
1690         int r;
1691
1692         if (ftrace_pid_trace == ftrace_swapper_pid)
1693                 r = sprintf(buf, "swapper tasks\n");
1694         else if (ftrace_pid_trace)
1695                 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
1696         else
1697                 r = sprintf(buf, "no pid\n");
1698
1699         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1700 }
1701
1702 static void clear_ftrace_swapper(void)
1703 {
1704         struct task_struct *p;
1705         int cpu;
1706
1707         get_online_cpus();
1708         for_each_online_cpu(cpu) {
1709                 p = idle_task(cpu);
1710                 clear_tsk_trace_trace(p);
1711         }
1712         put_online_cpus();
1713 }
1714
1715 static void set_ftrace_swapper(void)
1716 {
1717         struct task_struct *p;
1718         int cpu;
1719
1720         get_online_cpus();
1721         for_each_online_cpu(cpu) {
1722                 p = idle_task(cpu);
1723                 set_tsk_trace_trace(p);
1724         }
1725         put_online_cpus();
1726 }
1727
1728 static void clear_ftrace_pid(struct pid *pid)
1729 {
1730         struct task_struct *p;
1731
1732         rcu_read_lock();
1733         do_each_pid_task(pid, PIDTYPE_PID, p) {
1734                 clear_tsk_trace_trace(p);
1735         } while_each_pid_task(pid, PIDTYPE_PID, p);
1736         rcu_read_unlock();
1737
1738         put_pid(pid);
1739 }
1740
1741 static void set_ftrace_pid(struct pid *pid)
1742 {
1743         struct task_struct *p;
1744
1745         rcu_read_lock();
1746         do_each_pid_task(pid, PIDTYPE_PID, p) {
1747                 set_tsk_trace_trace(p);
1748         } while_each_pid_task(pid, PIDTYPE_PID, p);
1749         rcu_read_unlock();
1750 }
1751
1752 static void clear_ftrace_pid_task(struct pid **pid)
1753 {
1754         if (*pid == ftrace_swapper_pid)
1755                 clear_ftrace_swapper();
1756         else
1757                 clear_ftrace_pid(*pid);
1758
1759         *pid = NULL;
1760 }
1761
1762 static void set_ftrace_pid_task(struct pid *pid)
1763 {
1764         if (pid == ftrace_swapper_pid)
1765                 set_ftrace_swapper();
1766         else
1767                 set_ftrace_pid(pid);
1768 }
1769
1770 static ssize_t
1771 ftrace_pid_write(struct file *filp, const char __user *ubuf,
1772                    size_t cnt, loff_t *ppos)
1773 {
1774         struct pid *pid;
1775         char buf[64];
1776         long val;
1777         int ret;
1778
1779         if (cnt >= sizeof(buf))
1780                 return -EINVAL;
1781
1782         if (copy_from_user(&buf, ubuf, cnt))
1783                 return -EFAULT;
1784
1785         buf[cnt] = 0;
1786
1787         ret = strict_strtol(buf, 10, &val);
1788         if (ret < 0)
1789                 return ret;
1790
1791         mutex_lock(&ftrace_start_lock);
1792         if (val < 0) {
1793                 /* disable pid tracing */
1794                 if (!ftrace_pid_trace)
1795                         goto out;
1796
1797                 clear_ftrace_pid_task(&ftrace_pid_trace);
1798
1799         } else {
1800                 /* swapper task is special */
1801                 if (!val) {
1802                         pid = ftrace_swapper_pid;
1803                         if (pid == ftrace_pid_trace)
1804                                 goto out;
1805                 } else {
1806                         pid = find_get_pid(val);
1807
1808                         if (pid == ftrace_pid_trace) {
1809                                 put_pid(pid);
1810                                 goto out;
1811                         }
1812                 }
1813
1814                 if (ftrace_pid_trace)
1815                         clear_ftrace_pid_task(&ftrace_pid_trace);
1816
1817                 if (!pid)
1818                         goto out;
1819
1820                 ftrace_pid_trace = pid;
1821
1822                 set_ftrace_pid_task(ftrace_pid_trace);
1823         }
1824
1825         /* update the function call */
1826         ftrace_update_pid_func();
1827         ftrace_startup_enable(0);
1828
1829  out:
1830         mutex_unlock(&ftrace_start_lock);
1831
1832         return cnt;
1833 }
1834
1835 static struct file_operations ftrace_pid_fops = {
1836         .read = ftrace_pid_read,
1837         .write = ftrace_pid_write,
1838 };
1839
1840 static __init int ftrace_init_debugfs(void)
1841 {
1842         struct dentry *d_tracer;
1843         struct dentry *entry;
1844
1845         d_tracer = tracing_init_dentry();
1846         if (!d_tracer)
1847                 return 0;
1848
1849         ftrace_init_dyn_debugfs(d_tracer);
1850
1851         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
1852                                     NULL, &ftrace_pid_fops);
1853         if (!entry)
1854                 pr_warning("Could not create debugfs "
1855                            "'set_ftrace_pid' entry\n");
1856         return 0;
1857 }
1858
1859 fs_initcall(ftrace_init_debugfs);
1860
1861 /**
1862  * ftrace_kill - kill ftrace
1863  *
1864  * This function should be used by panic code. It stops ftrace
1865  * but in a not so nice way. If you need to simply kill ftrace
1866  * from a non-atomic section, use ftrace_kill.
1867  */
1868 void ftrace_kill(void)
1869 {
1870         ftrace_disabled = 1;
1871         ftrace_enabled = 0;
1872         clear_ftrace_function();
1873 }
1874
1875 /**
1876  * register_ftrace_function - register a function for profiling
1877  * @ops - ops structure that holds the function for profiling.
1878  *
1879  * Register a function to be called by all functions in the
1880  * kernel.
1881  *
1882  * Note: @ops->func and all the functions it calls must be labeled
1883  *       with "notrace", otherwise it will go into a
1884  *       recursive loop.
1885  */
1886 int register_ftrace_function(struct ftrace_ops *ops)
1887 {
1888         int ret;
1889
1890         if (unlikely(ftrace_disabled))
1891                 return -1;
1892
1893         mutex_lock(&ftrace_sysctl_lock);
1894
1895         ret = __register_ftrace_function(ops);
1896         ftrace_startup(0);
1897
1898         mutex_unlock(&ftrace_sysctl_lock);
1899         return ret;
1900 }
1901
1902 /**
1903  * unregister_ftrace_function - unregister a function for profiling.
1904  * @ops - ops structure that holds the function to unregister
1905  *
1906  * Unregister a function that was added to be called by ftrace profiling.
1907  */
1908 int unregister_ftrace_function(struct ftrace_ops *ops)
1909 {
1910         int ret;
1911
1912         mutex_lock(&ftrace_sysctl_lock);
1913         ret = __unregister_ftrace_function(ops);
1914         ftrace_shutdown(0);
1915         mutex_unlock(&ftrace_sysctl_lock);
1916
1917         return ret;
1918 }
1919
1920 int
1921 ftrace_enable_sysctl(struct ctl_table *table, int write,
1922                      struct file *file, void __user *buffer, size_t *lenp,
1923                      loff_t *ppos)
1924 {
1925         int ret;
1926
1927         if (unlikely(ftrace_disabled))
1928                 return -ENODEV;
1929
1930         mutex_lock(&ftrace_sysctl_lock);
1931
1932         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
1933
1934         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1935                 goto out;
1936
1937         last_ftrace_enabled = ftrace_enabled;
1938
1939         if (ftrace_enabled) {
1940
1941                 ftrace_startup_sysctl();
1942
1943                 /* we are starting ftrace again */
1944                 if (ftrace_list != &ftrace_list_end) {
1945                         if (ftrace_list->next == &ftrace_list_end)
1946                                 ftrace_trace_function = ftrace_list->func;
1947                         else
1948                                 ftrace_trace_function = ftrace_list_func;
1949                 }
1950
1951         } else {
1952                 /* stopping ftrace calls (just send to ftrace_stub) */
1953                 ftrace_trace_function = ftrace_stub;
1954
1955                 ftrace_shutdown_sysctl();
1956         }
1957
1958  out:
1959         mutex_unlock(&ftrace_sysctl_lock);
1960         return ret;
1961 }
1962
1963 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1964
1965 static atomic_t ftrace_graph_active;
1966 static struct notifier_block ftrace_suspend_notifier;
1967
1968 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
1969 {
1970         return 0;
1971 }
1972
1973 /* The callbacks that hook a function */
1974 trace_func_graph_ret_t ftrace_graph_return =
1975                         (trace_func_graph_ret_t)ftrace_stub;
1976 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1977
1978 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1979 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
1980 {
1981         int i;
1982         int ret = 0;
1983         unsigned long flags;
1984         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1985         struct task_struct *g, *t;
1986
1987         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1988                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
1989                                         * sizeof(struct ftrace_ret_stack),
1990                                         GFP_KERNEL);
1991                 if (!ret_stack_list[i]) {
1992                         start = 0;
1993                         end = i;
1994                         ret = -ENOMEM;
1995                         goto free;
1996                 }
1997         }
1998
1999         read_lock_irqsave(&tasklist_lock, flags);
2000         do_each_thread(g, t) {
2001                 if (start == end) {
2002                         ret = -EAGAIN;
2003                         goto unlock;
2004                 }
2005
2006                 if (t->ret_stack == NULL) {
2007                         t->curr_ret_stack = -1;
2008                         /* Make sure IRQs see the -1 first: */
2009                         barrier();
2010                         t->ret_stack = ret_stack_list[start++];
2011                         atomic_set(&t->tracing_graph_pause, 0);
2012                         atomic_set(&t->trace_overrun, 0);
2013                 }
2014         } while_each_thread(g, t);
2015
2016 unlock:
2017         read_unlock_irqrestore(&tasklist_lock, flags);
2018 free:
2019         for (i = start; i < end; i++)
2020                 kfree(ret_stack_list[i]);
2021         return ret;
2022 }
2023
2024 /* Allocate a return stack for each task */
2025 static int start_graph_tracing(void)
2026 {
2027         struct ftrace_ret_stack **ret_stack_list;
2028         int ret;
2029
2030         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2031                                 sizeof(struct ftrace_ret_stack *),
2032                                 GFP_KERNEL);
2033
2034         if (!ret_stack_list)
2035                 return -ENOMEM;
2036
2037         do {
2038                 ret = alloc_retstack_tasklist(ret_stack_list);
2039         } while (ret == -EAGAIN);
2040
2041         kfree(ret_stack_list);
2042         return ret;
2043 }
2044
2045 /*
2046  * Hibernation protection.
2047  * The state of the current task is too much unstable during
2048  * suspend/restore to disk. We want to protect against that.
2049  */
2050 static int
2051 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2052                                                         void *unused)
2053 {
2054         switch (state) {
2055         case PM_HIBERNATION_PREPARE:
2056                 pause_graph_tracing();
2057                 break;
2058
2059         case PM_POST_HIBERNATION:
2060                 unpause_graph_tracing();
2061                 break;
2062         }
2063         return NOTIFY_DONE;
2064 }
2065
2066 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2067                         trace_func_graph_ent_t entryfunc)
2068 {
2069         int ret = 0;
2070
2071         mutex_lock(&ftrace_sysctl_lock);
2072
2073         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2074         register_pm_notifier(&ftrace_suspend_notifier);
2075
2076         atomic_inc(&ftrace_graph_active);
2077         ret = start_graph_tracing();
2078         if (ret) {
2079                 atomic_dec(&ftrace_graph_active);
2080                 goto out;
2081         }
2082
2083         ftrace_graph_return = retfunc;
2084         ftrace_graph_entry = entryfunc;
2085
2086         ftrace_startup(FTRACE_START_FUNC_RET);
2087
2088 out:
2089         mutex_unlock(&ftrace_sysctl_lock);
2090         return ret;
2091 }
2092
2093 void unregister_ftrace_graph(void)
2094 {
2095         mutex_lock(&ftrace_sysctl_lock);
2096
2097         atomic_dec(&ftrace_graph_active);
2098         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2099         ftrace_graph_entry = ftrace_graph_entry_stub;
2100         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2101         unregister_pm_notifier(&ftrace_suspend_notifier);
2102
2103         mutex_unlock(&ftrace_sysctl_lock);
2104 }
2105
2106 /* Allocate a return stack for newly created task */
2107 void ftrace_graph_init_task(struct task_struct *t)
2108 {
2109         if (atomic_read(&ftrace_graph_active)) {
2110                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2111                                 * sizeof(struct ftrace_ret_stack),
2112                                 GFP_KERNEL);
2113                 if (!t->ret_stack)
2114                         return;
2115                 t->curr_ret_stack = -1;
2116                 atomic_set(&t->tracing_graph_pause, 0);
2117                 atomic_set(&t->trace_overrun, 0);
2118         } else
2119                 t->ret_stack = NULL;
2120 }
2121
2122 void ftrace_graph_exit_task(struct task_struct *t)
2123 {
2124         struct ftrace_ret_stack *ret_stack = t->ret_stack;
2125
2126         t->ret_stack = NULL;
2127         /* NULL must become visible to IRQs before we free it: */
2128         barrier();
2129
2130         kfree(ret_stack);
2131 }
2132
2133 void ftrace_graph_stop(void)
2134 {
2135         ftrace_stop();
2136 }
2137 #endif
2138