git.oblomov.eu Git - linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <trace/sched.h>
  33
  34 #include <asm/ftrace.h>
  35
  36 #include "trace.h"
  37
  38 #define FTRACE_WARN_ON(cond)                    \
  39         do {                                    \
  40                 if (WARN_ON(cond))              \
  41                         ftrace_kill();          \
  42         } while (0)
  43
  44 #define FTRACE_WARN_ON_ONCE(cond)               \
  45         do {                                    \
  46                 if (WARN_ON_ONCE(cond))         \
  47                         ftrace_kill();          \
  48         } while (0)
  49
  50 /* hash bits for specific function selection */
  51 #define FTRACE_HASH_BITS 7
  52 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  53
  54 /* ftrace_enabled is a method to turn ftrace on or off */
  55 int ftrace_enabled __read_mostly;
  56 static int last_ftrace_enabled;
  57
  58 /* Quick disabling of function tracer. */
  59 int function_trace_stop;
  60
  61 /*
  62  * ftrace_disabled is set when an anomaly is discovered.
  63  * ftrace_disabled is much stronger than ftrace_enabled.
  64  */
  65 static int ftrace_disabled __read_mostly;
  66
  67 static DEFINE_MUTEX(ftrace_lock);
  68
  69 static struct ftrace_ops ftrace_list_end __read_mostly =
  70 {
  71         .func = ftrace_stub,
  72 };
  73
  74 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  75 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  76 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  77 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  78
  79 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  80 {
  81         struct ftrace_ops *op = ftrace_list;
  82
  83         /* in case someone actually ports this to alpha! */
  84         read_barrier_depends();
  85
  86         while (op != &ftrace_list_end) {
  87                 /* silly alpha */
  88                 read_barrier_depends();
  89                 op->func(ip, parent_ip);
  90                 op = op->next;
  91         };
  92 }
  93
  94 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  95 {
  96         if (!test_tsk_trace_trace(current))
  97                 return;
  98
  99         ftrace_pid_function(ip, parent_ip);
 100 }
 101
 102 static void set_ftrace_pid_function(ftrace_func_t func)
 103 {
 104         /* do not set ftrace_pid_function to itself! */
 105         if (func != ftrace_pid_func)
 106                 ftrace_pid_function = func;
 107 }
 108
 109 /**
 110  * clear_ftrace_function - reset the ftrace function
 111  *
 112  * This NULLs the ftrace function and in essence stops
 113  * tracing.  There may be lag
 114  */
 115 void clear_ftrace_function(void)
 116 {
 117         ftrace_trace_function = ftrace_stub;
 118         __ftrace_trace_function = ftrace_stub;
 119         ftrace_pid_function = ftrace_stub;
 120 }
 121
 122 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 123 /*
 124  * For those archs that do not test ftrace_trace_stop in their
 125  * mcount call site, we need to do it from C.
 126  */
 127 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 128 {
 129         if (function_trace_stop)
 130                 return;
 131
 132         __ftrace_trace_function(ip, parent_ip);
 133 }
 134 #endif
 135
 136 static int __register_ftrace_function(struct ftrace_ops *ops)
 137 {
 138         ops->next = ftrace_list;
 139         /*
 140          * We are entering ops into the ftrace_list but another
 141          * CPU might be walking that list. We need to make sure
 142          * the ops->next pointer is valid before another CPU sees
 143          * the ops pointer included into the ftrace_list.
 144          */
 145         smp_wmb();
 146         ftrace_list = ops;
 147
 148         if (ftrace_enabled) {
 149                 ftrace_func_t func;
 150
 151                 if (ops->next == &ftrace_list_end)
 152                         func = ops->func;
 153                 else
 154                         func = ftrace_list_func;
 155
 156                 if (ftrace_pid_trace) {
 157                         set_ftrace_pid_function(func);
 158                         func = ftrace_pid_func;
 159                 }
 160
 161                 /*
 162                  * For one func, simply call it directly.
 163                  * For more than one func, call the chain.
 164                  */
 165 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 166                 ftrace_trace_function = func;
 167 #else
 168                 __ftrace_trace_function = func;
 169                 ftrace_trace_function = ftrace_test_stop_func;
 170 #endif
 171         }
 172
 173         return 0;
 174 }
 175
 176 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 177 {
 178         struct ftrace_ops **p;
 179
 180         /*
 181          * If we are removing the last function, then simply point
 182          * to the ftrace_stub.
 183          */
 184         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 185                 ftrace_trace_function = ftrace_stub;
 186                 ftrace_list = &ftrace_list_end;
 187                 return 0;
 188         }
 189
 190         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 191                 if (*p == ops)
 192                         break;
 193
 194         if (*p != ops)
 195                 return -1;
 196
 197         *p = (*p)->next;
 198
 199         if (ftrace_enabled) {
 200                 /* If we only have one func left, then call that directly */
 201                 if (ftrace_list->next == &ftrace_list_end) {
 202                         ftrace_func_t func = ftrace_list->func;
 203
 204                         if (ftrace_pid_trace) {
 205                                 set_ftrace_pid_function(func);
 206                                 func = ftrace_pid_func;
 207                         }
 208 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 209                         ftrace_trace_function = func;
 210 #else
 211                         __ftrace_trace_function = func;
 212 #endif
 213                 }
 214         }
 215
 216         return 0;
 217 }
 218
 219 static void ftrace_update_pid_func(void)
 220 {
 221         ftrace_func_t func;
 222
 223         if (ftrace_trace_function == ftrace_stub)
 224                 return;
 225
 226         func = ftrace_trace_function;
 227
 228         if (ftrace_pid_trace) {
 229                 set_ftrace_pid_function(func);
 230                 func = ftrace_pid_func;
 231         } else {
 232                 if (func == ftrace_pid_func)
 233                         func = ftrace_pid_function;
 234         }
 235
 236 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 237         ftrace_trace_function = func;
 238 #else
 239         __ftrace_trace_function = func;
 240 #endif
 241 }
 242
 243 /* set when tracing only a pid */
 244 struct pid *ftrace_pid_trace;
 245 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 246
 247 #ifdef CONFIG_DYNAMIC_FTRACE
 248
 249 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 250 # error Dynamic ftrace depends on MCOUNT_RECORD
 251 #endif
 252
 253 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 254
 255 struct ftrace_func_probe {
 256         struct hlist_node       node;
 257         struct ftrace_probe_ops *ops;
 258         unsigned long           flags;
 259         unsigned long           ip;
 260         void                    *data;
 261         struct rcu_head         rcu;
 262 };
 263
 264
 265 enum {
 266         FTRACE_ENABLE_CALLS             = (1 << 0),
 267         FTRACE_DISABLE_CALLS            = (1 << 1),
 268         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 269         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 270         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 271         FTRACE_START_FUNC_RET           = (1 << 5),
 272         FTRACE_STOP_FUNC_RET            = (1 << 6),
 273 };
 274
 275 static int ftrace_filtered;
 276
 277 static struct dyn_ftrace *ftrace_new_addrs;
 278
 279 static DEFINE_MUTEX(ftrace_regex_lock);
 280
 281 struct ftrace_page {
 282         struct ftrace_page      *next;
 283         int                     index;
 284         struct dyn_ftrace       records[];
 285 };
 286
 287 #define ENTRIES_PER_PAGE \
 288   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 289
 290 /* estimate from running different kernels */
 291 #define NR_TO_INIT              10000
 292
 293 static struct ftrace_page       *ftrace_pages_start;
 294 static struct ftrace_page       *ftrace_pages;
 295
 296 static struct dyn_ftrace *ftrace_free_records;
 297
 298 /*
 299  * This is a double for. Do not use 'break' to break out of the loop,
 300  * you must use a goto.
 301  */
 302 #define do_for_each_ftrace_rec(pg, rec)                                 \
 303         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 304                 int _____i;                                             \
 305                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 306                         rec = &pg->records[_____i];
 307
 308 #define while_for_each_ftrace_rec()             \
 309                 }                               \
 310         }
 311
 312 #ifdef CONFIG_KPROBES
 313
 314 static int frozen_record_count;
 315
 316 static inline void freeze_record(struct dyn_ftrace *rec)
 317 {
 318         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 319                 rec->flags |= FTRACE_FL_FROZEN;
 320                 frozen_record_count++;
 321         }
 322 }
 323
 324 static inline void unfreeze_record(struct dyn_ftrace *rec)
 325 {
 326         if (rec->flags & FTRACE_FL_FROZEN) {
 327                 rec->flags &= ~FTRACE_FL_FROZEN;
 328                 frozen_record_count--;
 329         }
 330 }
 331
 332 static inline int record_frozen(struct dyn_ftrace *rec)
 333 {
 334         return rec->flags & FTRACE_FL_FROZEN;
 335 }
 336 #else
 337 # define freeze_record(rec)                     ({ 0; })
 338 # define unfreeze_record(rec)                   ({ 0; })
 339 # define record_frozen(rec)                     ({ 0; })
 340 #endif /* CONFIG_KPROBES */
 341
 342 static void ftrace_free_rec(struct dyn_ftrace *rec)
 343 {
 344         rec->freelist = ftrace_free_records;
 345         ftrace_free_records = rec;
 346         rec->flags |= FTRACE_FL_FREE;
 347 }
 348
 349 void ftrace_release(void *start, unsigned long size)
 350 {
 351         struct dyn_ftrace *rec;
 352         struct ftrace_page *pg;
 353         unsigned long s = (unsigned long)start;
 354         unsigned long e = s + size;
 355
 356         if (ftrace_disabled || !start)
 357                 return;
 358
 359         mutex_lock(&ftrace_lock);
 360         do_for_each_ftrace_rec(pg, rec) {
 361                 if ((rec->ip >= s) && (rec->ip < e)) {
 362                         /*
 363                          * rec->ip is changed in ftrace_free_rec()
 364                          * It should not between s and e if record was freed.
 365                          */
 366                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
 367                         ftrace_free_rec(rec);
 368                 }
 369         } while_for_each_ftrace_rec();
 370         mutex_unlock(&ftrace_lock);
 371 }
 372
 373 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 374 {
 375         struct dyn_ftrace *rec;
 376
 377         /* First check for freed records */
 378         if (ftrace_free_records) {
 379                 rec = ftrace_free_records;
 380
 381                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 382                         FTRACE_WARN_ON_ONCE(1);
 383                         ftrace_free_records = NULL;
 384                         return NULL;
 385                 }
 386
 387                 ftrace_free_records = rec->freelist;
 388                 memset(rec, 0, sizeof(*rec));
 389                 return rec;
 390         }
 391
 392         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 393                 if (!ftrace_pages->next) {
 394                         /* allocate another page */
 395                         ftrace_pages->next =
 396                                 (void *)get_zeroed_page(GFP_KERNEL);
 397                         if (!ftrace_pages->next)
 398                                 return NULL;
 399                 }
 400                 ftrace_pages = ftrace_pages->next;
 401         }
 402
 403         return &ftrace_pages->records[ftrace_pages->index++];
 404 }
 405
 406 static struct dyn_ftrace *
 407 ftrace_record_ip(unsigned long ip)
 408 {
 409         struct dyn_ftrace *rec;
 410
 411         if (ftrace_disabled)
 412                 return NULL;
 413
 414         rec = ftrace_alloc_dyn_node(ip);
 415         if (!rec)
 416                 return NULL;
 417
 418         rec->ip = ip;
 419         rec->newlist = ftrace_new_addrs;
 420         ftrace_new_addrs = rec;
 421
 422         return rec;
 423 }
 424
 425 static void print_ip_ins(const char *fmt, unsigned char *p)
 426 {
 427         int i;
 428
 429         printk(KERN_CONT "%s", fmt);
 430
 431         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 432                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 433 }
 434
 435 static void ftrace_bug(int failed, unsigned long ip)
 436 {
 437         switch (failed) {
 438         case -EFAULT:
 439                 FTRACE_WARN_ON_ONCE(1);
 440                 pr_info("ftrace faulted on modifying ");
 441                 print_ip_sym(ip);
 442                 break;
 443         case -EINVAL:
 444                 FTRACE_WARN_ON_ONCE(1);
 445                 pr_info("ftrace failed to modify ");
 446                 print_ip_sym(ip);
 447                 print_ip_ins(" actual: ", (unsigned char *)ip);
 448                 printk(KERN_CONT "\n");
 449                 break;
 450         case -EPERM:
 451                 FTRACE_WARN_ON_ONCE(1);
 452                 pr_info("ftrace faulted on writing ");
 453                 print_ip_sym(ip);
 454                 break;
 455         default:
 456                 FTRACE_WARN_ON_ONCE(1);
 457                 pr_info("ftrace faulted on unknown error ");
 458                 print_ip_sym(ip);
 459         }
 460 }
 461
 462
 463 static int
 464 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 465 {
 466         unsigned long ftrace_addr;
 467         unsigned long ip, fl;
 468
 469         ftrace_addr = (unsigned long)FTRACE_ADDR;
 470
 471         ip = rec->ip;
 472
 473         /*
 474          * If this record is not to be traced and
 475          * it is not enabled then do nothing.
 476          *
 477          * If this record is not to be traced and
 478          * it is enabled then disable it.
 479          *
 480          */
 481         if (rec->flags & FTRACE_FL_NOTRACE) {
 482                 if (rec->flags & FTRACE_FL_ENABLED)
 483                         rec->flags &= ~FTRACE_FL_ENABLED;
 484                 else
 485                         return 0;
 486
 487         } else if (ftrace_filtered && enable) {
 488                 /*
 489                  * Filtering is on:
 490                  */
 491
 492                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
 493
 494                 /* Record is filtered and enabled, do nothing */
 495                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
 496                         return 0;
 497
 498                 /* Record is not filtered or enabled, do nothing */
 499                 if (!fl)
 500                         return 0;
 501
 502                 /* Record is not filtered but enabled, disable it */
 503                 if (fl == FTRACE_FL_ENABLED)
 504                         rec->flags &= ~FTRACE_FL_ENABLED;
 505                 else
 506                 /* Otherwise record is filtered but not enabled, enable it */
 507                         rec->flags |= FTRACE_FL_ENABLED;
 508         } else {
 509                 /* Disable or not filtered */
 510
 511                 if (enable) {
 512                         /* if record is enabled, do nothing */
 513                         if (rec->flags & FTRACE_FL_ENABLED)
 514                                 return 0;
 515
 516                         rec->flags |= FTRACE_FL_ENABLED;
 517
 518                 } else {
 519
 520                         /* if record is not enabled, do nothing */
 521                         if (!(rec->flags & FTRACE_FL_ENABLED))
 522                                 return 0;
 523
 524                         rec->flags &= ~FTRACE_FL_ENABLED;
 525                 }
 526         }
 527
 528         if (rec->flags & FTRACE_FL_ENABLED)
 529                 return ftrace_make_call(rec, ftrace_addr);
 530         else
 531                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 532 }
 533
 534 static void ftrace_replace_code(int enable)
 535 {
 536         struct dyn_ftrace *rec;
 537         struct ftrace_page *pg;
 538         int failed;
 539
 540         do_for_each_ftrace_rec(pg, rec) {
 541                 /*
 542                  * Skip over free records, records that have
 543                  * failed and not converted.
 544                  */
 545                 if (rec->flags & FTRACE_FL_FREE ||
 546                     rec->flags & FTRACE_FL_FAILED ||
 547                     !(rec->flags & FTRACE_FL_CONVERTED))
 548                         continue;
 549
 550                 /* ignore updates to this record's mcount site */
 551                 if (get_kprobe((void *)rec->ip)) {
 552                         freeze_record(rec);
 553                         continue;
 554                 } else {
 555                         unfreeze_record(rec);
 556                 }
 557
 558                 failed = __ftrace_replace_code(rec, enable);
 559                 if (failed) {
 560                         rec->flags |= FTRACE_FL_FAILED;
 561                         if ((system_state == SYSTEM_BOOTING) ||
 562                             !core_kernel_text(rec->ip)) {
 563                                 ftrace_free_rec(rec);
 564                                 } else {
 565                                 ftrace_bug(failed, rec->ip);
 566                                         /* Stop processing */
 567                                         return;
 568                                 }
 569                 }
 570         } while_for_each_ftrace_rec();
 571 }
 572
 573 static int
 574 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 575 {
 576         unsigned long ip;
 577         int ret;
 578
 579         ip = rec->ip;
 580
 581         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
 582         if (ret) {
 583                 ftrace_bug(ret, ip);
 584                 rec->flags |= FTRACE_FL_FAILED;
 585                 return 0;
 586         }
 587         return 1;
 588 }
 589
 590 /*
 591  * archs can override this function if they must do something
 592  * before the modifying code is performed.
 593  */
 594 int __weak ftrace_arch_code_modify_prepare(void)
 595 {
 596         return 0;
 597 }
 598
 599 /*
 600  * archs can override this function if they must do something
 601  * after the modifying code is performed.
 602  */
 603 int __weak ftrace_arch_code_modify_post_process(void)
 604 {
 605         return 0;
 606 }
 607
 608 static int __ftrace_modify_code(void *data)
 609 {
 610         int *command = data;
 611
 612         if (*command & FTRACE_ENABLE_CALLS)
 613                 ftrace_replace_code(1);
 614         else if (*command & FTRACE_DISABLE_CALLS)
 615                 ftrace_replace_code(0);
 616
 617         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 618                 ftrace_update_ftrace_func(ftrace_trace_function);
 619
 620         if (*command & FTRACE_START_FUNC_RET)
 621                 ftrace_enable_ftrace_graph_caller();
 622         else if (*command & FTRACE_STOP_FUNC_RET)
 623                 ftrace_disable_ftrace_graph_caller();
 624
 625         return 0;
 626 }
 627
 628 static void ftrace_run_update_code(int command)
 629 {
 630         int ret;
 631
 632         ret = ftrace_arch_code_modify_prepare();
 633         FTRACE_WARN_ON(ret);
 634         if (ret)
 635                 return;
 636
 637         stop_machine(__ftrace_modify_code, &command, NULL);
 638
 639         ret = ftrace_arch_code_modify_post_process();
 640         FTRACE_WARN_ON(ret);
 641 }
 642
 643 static ftrace_func_t saved_ftrace_func;
 644 static int ftrace_start_up;
 645
 646 static void ftrace_startup_enable(int command)
 647 {
 648         if (saved_ftrace_func != ftrace_trace_function) {
 649                 saved_ftrace_func = ftrace_trace_function;
 650                 command |= FTRACE_UPDATE_TRACE_FUNC;
 651         }
 652
 653         if (!command || !ftrace_enabled)
 654                 return;
 655
 656         ftrace_run_update_code(command);
 657 }
 658
 659 static void ftrace_startup(int command)
 660 {
 661         if (unlikely(ftrace_disabled))
 662                 return;
 663
 664         ftrace_start_up++;
 665         command |= FTRACE_ENABLE_CALLS;
 666
 667         ftrace_startup_enable(command);
 668 }
 669
 670 static void ftrace_shutdown(int command)
 671 {
 672         if (unlikely(ftrace_disabled))
 673                 return;
 674
 675         ftrace_start_up--;
 676         if (!ftrace_start_up)
 677                 command |= FTRACE_DISABLE_CALLS;
 678
 679         if (saved_ftrace_func != ftrace_trace_function) {
 680                 saved_ftrace_func = ftrace_trace_function;
 681                 command |= FTRACE_UPDATE_TRACE_FUNC;
 682         }
 683
 684         if (!command || !ftrace_enabled)
 685                 return;
 686
 687         ftrace_run_update_code(command);
 688 }
 689
 690 static void ftrace_startup_sysctl(void)
 691 {
 692         int command = FTRACE_ENABLE_MCOUNT;
 693
 694         if (unlikely(ftrace_disabled))
 695                 return;
 696
 697         /* Force update next time */
 698         saved_ftrace_func = NULL;
 699         /* ftrace_start_up is true if we want ftrace running */
 700         if (ftrace_start_up)
 701                 command |= FTRACE_ENABLE_CALLS;
 702
 703         ftrace_run_update_code(command);
 704 }
 705
 706 static void ftrace_shutdown_sysctl(void)
 707 {
 708         int command = FTRACE_DISABLE_MCOUNT;
 709
 710         if (unlikely(ftrace_disabled))
 711                 return;
 712
 713         /* ftrace_start_up is true if ftrace is running */
 714         if (ftrace_start_up)
 715                 command |= FTRACE_DISABLE_CALLS;
 716
 717         ftrace_run_update_code(command);
 718 }
 719
 720 static cycle_t          ftrace_update_time;
 721 static unsigned long    ftrace_update_cnt;
 722 unsigned long           ftrace_update_tot_cnt;
 723
 724 static int ftrace_update_code(struct module *mod)
 725 {
 726         struct dyn_ftrace *p;
 727         cycle_t start, stop;
 728
 729         start = ftrace_now(raw_smp_processor_id());
 730         ftrace_update_cnt = 0;
 731
 732         while (ftrace_new_addrs) {
 733
 734                 /* If something went wrong, bail without enabling anything */
 735                 if (unlikely(ftrace_disabled))
 736                         return -1;
 737
 738                 p = ftrace_new_addrs;
 739                 ftrace_new_addrs = p->newlist;
 740                 p->flags = 0L;
 741
 742                 /* convert record (i.e, patch mcount-call with NOP) */
 743                 if (ftrace_code_disable(mod, p)) {
 744                         p->flags |= FTRACE_FL_CONVERTED;
 745                         ftrace_update_cnt++;
 746                 } else
 747                         ftrace_free_rec(p);
 748         }
 749
 750         stop = ftrace_now(raw_smp_processor_id());
 751         ftrace_update_time = stop - start;
 752         ftrace_update_tot_cnt += ftrace_update_cnt;
 753
 754         return 0;
 755 }
 756
 757 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 758 {
 759         struct ftrace_page *pg;
 760         int cnt;
 761         int i;
 762
 763         /* allocate a few pages */
 764         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 765         if (!ftrace_pages_start)
 766                 return -1;
 767
 768         /*
 769          * Allocate a few more pages.
 770          *
 771          * TODO: have some parser search vmlinux before
 772          *   final linking to find all calls to ftrace.
 773          *   Then we can:
 774          *    a) know how many pages to allocate.
 775          *     and/or
 776          *    b) set up the table then.
 777          *
 778          *  The dynamic code is still necessary for
 779          *  modules.
 780          */
 781
 782         pg = ftrace_pages = ftrace_pages_start;
 783
 784         cnt = num_to_init / ENTRIES_PER_PAGE;
 785         pr_info("ftrace: allocating %ld entries in %d pages\n",
 786                 num_to_init, cnt + 1);
 787
 788         for (i = 0; i < cnt; i++) {
 789                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 790
 791                 /* If we fail, we'll try later anyway */
 792                 if (!pg->next)
 793                         break;
 794
 795                 pg = pg->next;
 796         }
 797
 798         return 0;
 799 }
 800
 801 enum {
 802         FTRACE_ITER_FILTER      = (1 << 0),
 803         FTRACE_ITER_CONT        = (1 << 1),
 804         FTRACE_ITER_NOTRACE     = (1 << 2),
 805         FTRACE_ITER_FAILURES    = (1 << 3),
 806         FTRACE_ITER_PRINTALL    = (1 << 4),
 807         FTRACE_ITER_HASH        = (1 << 5),
 808 };
 809
 810 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 811
 812 struct ftrace_iterator {
 813         struct ftrace_page      *pg;
 814         int                     hidx;
 815         int                     idx;
 816         unsigned                flags;
 817         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 818         unsigned                buffer_idx;
 819         unsigned                filtered;
 820 };
 821
 822 static void *
 823 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
 824 {
 825         struct ftrace_iterator *iter = m->private;
 826         struct hlist_node *hnd = v;
 827         struct hlist_head *hhd;
 828
 829         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
 830
 831         (*pos)++;
 832
 833  retry:
 834         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 835                 return NULL;
 836
 837         hhd = &ftrace_func_hash[iter->hidx];
 838
 839         if (hlist_empty(hhd)) {
 840                 iter->hidx++;
 841                 hnd = NULL;
 842                 goto retry;
 843         }
 844
 845         if (!hnd)
 846                 hnd = hhd->first;
 847         else {
 848                 hnd = hnd->next;
 849                 if (!hnd) {
 850                         iter->hidx++;
 851                         goto retry;
 852                 }
 853         }
 854
 855         return hnd;
 856 }
 857
 858 static void *t_hash_start(struct seq_file *m, loff_t *pos)
 859 {
 860         struct ftrace_iterator *iter = m->private;
 861         void *p = NULL;
 862
 863         iter->flags |= FTRACE_ITER_HASH;
 864
 865         return t_hash_next(m, p, pos);
 866 }
 867
 868 static int t_hash_show(struct seq_file *m, void *v)
 869 {
 870         struct ftrace_func_probe *rec;
 871         struct hlist_node *hnd = v;
 872         char str[KSYM_SYMBOL_LEN];
 873
 874         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
 875
 876         if (rec->ops->print)
 877                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
 878
 879         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 880         seq_printf(m, "%s:", str);
 881
 882         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
 883         seq_printf(m, "%s", str);
 884
 885         if (rec->data)
 886                 seq_printf(m, ":%p", rec->data);
 887         seq_putc(m, '\n');
 888
 889         return 0;
 890 }
 891
 892 static void *
 893 t_next(struct seq_file *m, void *v, loff_t *pos)
 894 {
 895         struct ftrace_iterator *iter = m->private;
 896         struct dyn_ftrace *rec = NULL;
 897
 898         if (iter->flags & FTRACE_ITER_HASH)
 899                 return t_hash_next(m, v, pos);
 900
 901         (*pos)++;
 902
 903         if (iter->flags & FTRACE_ITER_PRINTALL)
 904                 return NULL;
 905
 906  retry:
 907         if (iter->idx >= iter->pg->index) {
 908                 if (iter->pg->next) {
 909                         iter->pg = iter->pg->next;
 910                         iter->idx = 0;
 911                         goto retry;
 912                 } else {
 913                         iter->idx = -1;
 914                 }
 915         } else {
 916                 rec = &iter->pg->records[iter->idx++];
 917                 if ((rec->flags & FTRACE_FL_FREE) ||
 918
 919                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 920                      (rec->flags & FTRACE_FL_FAILED)) ||
 921
 922                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 923                      !(rec->flags & FTRACE_FL_FAILED)) ||
 924
 925                     ((iter->flags & FTRACE_ITER_FILTER) &&
 926                      !(rec->flags & FTRACE_FL_FILTER)) ||
 927
 928                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 929                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 930                         rec = NULL;
 931                         goto retry;
 932                 }
 933         }
 934
 935         return rec;
 936 }
 937
 938 static void *t_start(struct seq_file *m, loff_t *pos)
 939 {
 940         struct ftrace_iterator *iter = m->private;
 941         void *p = NULL;
 942
 943         mutex_lock(&ftrace_lock);
 944         /*
 945          * For set_ftrace_filter reading, if we have the filter
 946          * off, we can short cut and just print out that all
 947          * functions are enabled.
 948          */
 949         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
 950                 if (*pos > 0)
 951                         return t_hash_start(m, pos);
 952                 iter->flags |= FTRACE_ITER_PRINTALL;
 953                 (*pos)++;
 954                 return iter;
 955         }
 956
 957         if (iter->flags & FTRACE_ITER_HASH)
 958                 return t_hash_start(m, pos);
 959
 960         if (*pos > 0) {
 961                 if (iter->idx < 0)
 962                         return p;
 963                 (*pos)--;
 964                 iter->idx--;
 965         }
 966
 967         p = t_next(m, p, pos);
 968
 969         if (!p)
 970                 return t_hash_start(m, pos);
 971
 972         return p;
 973 }
 974
 975 static void t_stop(struct seq_file *m, void *p)
 976 {
 977         mutex_unlock(&ftrace_lock);
 978 }
 979
 980 static int t_show(struct seq_file *m, void *v)
 981 {
 982         struct ftrace_iterator *iter = m->private;
 983         struct dyn_ftrace *rec = v;
 984         char str[KSYM_SYMBOL_LEN];
 985
 986         if (iter->flags & FTRACE_ITER_HASH)
 987                 return t_hash_show(m, v);
 988
 989         if (iter->flags & FTRACE_ITER_PRINTALL) {
 990                 seq_printf(m, "#### all functions enabled ####\n");
 991                 return 0;
 992         }
 993
 994         if (!rec)
 995                 return 0;
 996
 997         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 998
 999         seq_printf(m, "%s\n", str);
1000
1001         return 0;
1002 }
1003
1004 static struct seq_operations show_ftrace_seq_ops = {
1005         .start = t_start,
1006         .next = t_next,
1007         .stop = t_stop,
1008         .show = t_show,
1009 };
1010
1011 static int
1012 ftrace_avail_open(struct inode *inode, struct file *file)
1013 {
1014         struct ftrace_iterator *iter;
1015         int ret;
1016
1017         if (unlikely(ftrace_disabled))
1018                 return -ENODEV;
1019
1020         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1021         if (!iter)
1022                 return -ENOMEM;
1023
1024         iter->pg = ftrace_pages_start;
1025
1026         ret = seq_open(file, &show_ftrace_seq_ops);
1027         if (!ret) {
1028                 struct seq_file *m = file->private_data;
1029
1030                 m->private = iter;
1031         } else {
1032                 kfree(iter);
1033         }
1034
1035         return ret;
1036 }
1037
1038 int ftrace_avail_release(struct inode *inode, struct file *file)
1039 {
1040         struct seq_file *m = (struct seq_file *)file->private_data;
1041         struct ftrace_iterator *iter = m->private;
1042
1043         seq_release(inode, file);
1044         kfree(iter);
1045
1046         return 0;
1047 }
1048
1049 static int
1050 ftrace_failures_open(struct inode *inode, struct file *file)
1051 {
1052         int ret;
1053         struct seq_file *m;
1054         struct ftrace_iterator *iter;
1055
1056         ret = ftrace_avail_open(inode, file);
1057         if (!ret) {
1058                 m = (struct seq_file *)file->private_data;
1059                 iter = (struct ftrace_iterator *)m->private;
1060                 iter->flags = FTRACE_ITER_FAILURES;
1061         }
1062
1063         return ret;
1064 }
1065
1066
1067 static void ftrace_filter_reset(int enable)
1068 {
1069         struct ftrace_page *pg;
1070         struct dyn_ftrace *rec;
1071         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1072
1073         mutex_lock(&ftrace_lock);
1074         if (enable)
1075                 ftrace_filtered = 0;
1076         do_for_each_ftrace_rec(pg, rec) {
1077                 if (rec->flags & FTRACE_FL_FAILED)
1078                         continue;
1079                 rec->flags &= ~type;
1080         } while_for_each_ftrace_rec();
1081         mutex_unlock(&ftrace_lock);
1082 }
1083
1084 static int
1085 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1086 {
1087         struct ftrace_iterator *iter;
1088         int ret = 0;
1089
1090         if (unlikely(ftrace_disabled))
1091                 return -ENODEV;
1092
1093         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1094         if (!iter)
1095                 return -ENOMEM;
1096
1097         mutex_lock(&ftrace_regex_lock);
1098         if ((file->f_mode & FMODE_WRITE) &&
1099             !(file->f_flags & O_APPEND))
1100                 ftrace_filter_reset(enable);
1101
1102         if (file->f_mode & FMODE_READ) {
1103                 iter->pg = ftrace_pages_start;
1104                 iter->flags = enable ? FTRACE_ITER_FILTER :
1105                         FTRACE_ITER_NOTRACE;
1106
1107                 ret = seq_open(file, &show_ftrace_seq_ops);
1108                 if (!ret) {
1109                         struct seq_file *m = file->private_data;
1110                         m->private = iter;
1111                 } else
1112                         kfree(iter);
1113         } else
1114                 file->private_data = iter;
1115         mutex_unlock(&ftrace_regex_lock);
1116
1117         return ret;
1118 }
1119
1120 static int
1121 ftrace_filter_open(struct inode *inode, struct file *file)
1122 {
1123         return ftrace_regex_open(inode, file, 1);
1124 }
1125
1126 static int
1127 ftrace_notrace_open(struct inode *inode, struct file *file)
1128 {
1129         return ftrace_regex_open(inode, file, 0);
1130 }
1131
1132 static loff_t
1133 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1134 {
1135         loff_t ret;
1136
1137         if (file->f_mode & FMODE_READ)
1138                 ret = seq_lseek(file, offset, origin);
1139         else
1140                 file->f_pos = ret = 1;
1141
1142         return ret;
1143 }
1144
1145 enum {
1146         MATCH_FULL,
1147         MATCH_FRONT_ONLY,
1148         MATCH_MIDDLE_ONLY,
1149         MATCH_END_ONLY,
1150 };
1151
1152 /*
1153  * (static function - no need for kernel doc)
1154  *
1155  * Pass in a buffer containing a glob and this function will
1156  * set search to point to the search part of the buffer and
1157  * return the type of search it is (see enum above).
1158  * This does modify buff.
1159  *
1160  * Returns enum type.
1161  *  search returns the pointer to use for comparison.
1162  *  not returns 1 if buff started with a '!'
1163  *     0 otherwise.
1164  */
1165 static int
1166 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1167 {
1168         int type = MATCH_FULL;
1169         int i;
1170
1171         if (buff[0] == '!') {
1172                 *not = 1;
1173                 buff++;
1174                 len--;
1175         } else
1176                 *not = 0;
1177
1178         *search = buff;
1179
1180         for (i = 0; i < len; i++) {
1181                 if (buff[i] == '*') {
1182                         if (!i) {
1183                                 *search = buff + 1;
1184                                 type = MATCH_END_ONLY;
1185                         } else {
1186                                 if (type == MATCH_END_ONLY)
1187                                         type = MATCH_MIDDLE_ONLY;
1188                                 else
1189                                         type = MATCH_FRONT_ONLY;
1190                                 buff[i] = 0;
1191                                 break;
1192                         }
1193                 }
1194         }
1195
1196         return type;
1197 }
1198
1199 static int ftrace_match(char *str, char *regex, int len, int type)
1200 {
1201         int matched = 0;
1202         char *ptr;
1203
1204         switch (type) {
1205         case MATCH_FULL:
1206                 if (strcmp(str, regex) == 0)
1207                         matched = 1;
1208                 break;
1209         case MATCH_FRONT_ONLY:
1210                 if (strncmp(str, regex, len) == 0)
1211                         matched = 1;
1212                 break;
1213         case MATCH_MIDDLE_ONLY:
1214                 if (strstr(str, regex))
1215                         matched = 1;
1216                 break;
1217         case MATCH_END_ONLY:
1218                 ptr = strstr(str, regex);
1219                 if (ptr && (ptr[len] == 0))
1220                         matched = 1;
1221                 break;
1222         }
1223
1224         return matched;
1225 }
1226
1227 static int
1228 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1229 {
1230         char str[KSYM_SYMBOL_LEN];
1231
1232         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1233         return ftrace_match(str, regex, len, type);
1234 }
1235
1236 static void ftrace_match_records(char *buff, int len, int enable)
1237 {
1238         unsigned int search_len;
1239         struct ftrace_page *pg;
1240         struct dyn_ftrace *rec;
1241         unsigned long flag;
1242         char *search;
1243         int type;
1244         int not;
1245
1246         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1247         type = ftrace_setup_glob(buff, len, &search, &not);
1248
1249         search_len = strlen(search);
1250
1251         mutex_lock(&ftrace_lock);
1252         do_for_each_ftrace_rec(pg, rec) {
1253
1254                 if (rec->flags & FTRACE_FL_FAILED)
1255                         continue;
1256
1257                 if (ftrace_match_record(rec, search, search_len, type)) {
1258                         if (not)
1259                                 rec->flags &= ~flag;
1260                         else
1261                                 rec->flags |= flag;
1262                 }
1263                 /*
1264                  * Only enable filtering if we have a function that
1265                  * is filtered on.
1266                  */
1267                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1268                         ftrace_filtered = 1;
1269         } while_for_each_ftrace_rec();
1270         mutex_unlock(&ftrace_lock);
1271 }
1272
1273 static int
1274 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1275                            char *regex, int len, int type)
1276 {
1277         char str[KSYM_SYMBOL_LEN];
1278         char *modname;
1279
1280         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1281
1282         if (!modname || strcmp(modname, mod))
1283                 return 0;
1284
1285         /* blank search means to match all funcs in the mod */
1286         if (len)
1287                 return ftrace_match(str, regex, len, type);
1288         else
1289                 return 1;
1290 }
1291
1292 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1293 {
1294         unsigned search_len = 0;
1295         struct ftrace_page *pg;
1296         struct dyn_ftrace *rec;
1297         int type = MATCH_FULL;
1298         char *search = buff;
1299         unsigned long flag;
1300         int not = 0;
1301
1302         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1303
1304         /* blank or '*' mean the same */
1305         if (strcmp(buff, "*") == 0)
1306                 buff[0] = 0;
1307
1308         /* handle the case of 'dont filter this module' */
1309         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1310                 buff[0] = 0;
1311                 not = 1;
1312         }
1313
1314         if (strlen(buff)) {
1315                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1316                 search_len = strlen(search);
1317         }
1318
1319         mutex_lock(&ftrace_lock);
1320         do_for_each_ftrace_rec(pg, rec) {
1321
1322                 if (rec->flags & FTRACE_FL_FAILED)
1323                         continue;
1324
1325                 if (ftrace_match_module_record(rec, mod,
1326                                                search, search_len, type)) {
1327                         if (not)
1328                                 rec->flags &= ~flag;
1329                         else
1330                                 rec->flags |= flag;
1331                 }
1332                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1333                         ftrace_filtered = 1;
1334
1335         } while_for_each_ftrace_rec();
1336         mutex_unlock(&ftrace_lock);
1337 }
1338
1339 /*
1340  * We register the module command as a template to show others how
1341  * to register the a command as well.
1342  */
1343
1344 static int
1345 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1346 {
1347         char *mod;
1348
1349         /*
1350          * cmd == 'mod' because we only registered this func
1351          * for the 'mod' ftrace_func_command.
1352          * But if you register one func with multiple commands,
1353          * you can tell which command was used by the cmd
1354          * parameter.
1355          */
1356
1357         /* we must have a module name */
1358         if (!param)
1359                 return -EINVAL;
1360
1361         mod = strsep(&param, ":");
1362         if (!strlen(mod))
1363                 return -EINVAL;
1364
1365         ftrace_match_module_records(func, mod, enable);
1366         return 0;
1367 }
1368
1369 static struct ftrace_func_command ftrace_mod_cmd = {
1370         .name                   = "mod",
1371         .func                   = ftrace_mod_callback,
1372 };
1373
1374 static int __init ftrace_mod_cmd_init(void)
1375 {
1376         return register_ftrace_command(&ftrace_mod_cmd);
1377 }
1378 device_initcall(ftrace_mod_cmd_init);
1379
1380 static void
1381 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1382 {
1383         struct ftrace_func_probe *entry;
1384         struct hlist_head *hhd;
1385         struct hlist_node *n;
1386         unsigned long key;
1387         int resched;
1388
1389         key = hash_long(ip, FTRACE_HASH_BITS);
1390
1391         hhd = &ftrace_func_hash[key];
1392
1393         if (hlist_empty(hhd))
1394                 return;
1395
1396         /*
1397          * Disable preemption for these calls to prevent a RCU grace
1398          * period. This syncs the hash iteration and freeing of items
1399          * on the hash. rcu_read_lock is too dangerous here.
1400          */
1401         resched = ftrace_preempt_disable();
1402         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1403                 if (entry->ip == ip)
1404                         entry->ops->func(ip, parent_ip, &entry->data);
1405         }
1406         ftrace_preempt_enable(resched);
1407 }
1408
1409 static struct ftrace_ops trace_probe_ops __read_mostly =
1410 {
1411         .func = function_trace_probe_call,
1412 };
1413
1414 static int ftrace_probe_registered;
1415
1416 static void __enable_ftrace_function_probe(void)
1417 {
1418         int i;
1419
1420         if (ftrace_probe_registered)
1421                 return;
1422
1423         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1424                 struct hlist_head *hhd = &ftrace_func_hash[i];
1425                 if (hhd->first)
1426                         break;
1427         }
1428         /* Nothing registered? */
1429         if (i == FTRACE_FUNC_HASHSIZE)
1430                 return;
1431
1432         __register_ftrace_function(&trace_probe_ops);
1433         ftrace_startup(0);
1434         ftrace_probe_registered = 1;
1435 }
1436
1437 static void __disable_ftrace_function_probe(void)
1438 {
1439         int i;
1440
1441         if (!ftrace_probe_registered)
1442                 return;
1443
1444         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1445                 struct hlist_head *hhd = &ftrace_func_hash[i];
1446                 if (hhd->first)
1447                         return;
1448         }
1449
1450         /* no more funcs left */
1451         __unregister_ftrace_function(&trace_probe_ops);
1452         ftrace_shutdown(0);
1453         ftrace_probe_registered = 0;
1454 }
1455
1456
1457 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1458 {
1459         struct ftrace_func_probe *entry =
1460                 container_of(rhp, struct ftrace_func_probe, rcu);
1461
1462         if (entry->ops->free)
1463                 entry->ops->free(&entry->data);
1464         kfree(entry);
1465 }
1466
1467
1468 int
1469 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1470                               void *data)
1471 {
1472         struct ftrace_func_probe *entry;
1473         struct ftrace_page *pg;
1474         struct dyn_ftrace *rec;
1475         int type, len, not;
1476         unsigned long key;
1477         int count = 0;
1478         char *search;
1479
1480         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1481         len = strlen(search);
1482
1483         /* we do not support '!' for function probes */
1484         if (WARN_ON(not))
1485                 return -EINVAL;
1486
1487         mutex_lock(&ftrace_lock);
1488         do_for_each_ftrace_rec(pg, rec) {
1489
1490                 if (rec->flags & FTRACE_FL_FAILED)
1491                         continue;
1492
1493                 if (!ftrace_match_record(rec, search, len, type))
1494                         continue;
1495
1496                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1497                 if (!entry) {
1498                         /* If we did not process any, then return error */
1499                         if (!count)
1500                                 count = -ENOMEM;
1501                         goto out_unlock;
1502                 }
1503
1504                 count++;
1505
1506                 entry->data = data;
1507
1508                 /*
1509                  * The caller might want to do something special
1510                  * for each function we find. We call the callback
1511                  * to give the caller an opportunity to do so.
1512                  */
1513                 if (ops->callback) {
1514                         if (ops->callback(rec->ip, &entry->data) < 0) {
1515                                 /* caller does not like this func */
1516                                 kfree(entry);
1517                                 continue;
1518                         }
1519                 }
1520
1521                 entry->ops = ops;
1522                 entry->ip = rec->ip;
1523
1524                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1525                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1526
1527         } while_for_each_ftrace_rec();
1528         __enable_ftrace_function_probe();
1529
1530  out_unlock:
1531         mutex_unlock(&ftrace_lock);
1532
1533         return count;
1534 }
1535
1536 enum {
1537         PROBE_TEST_FUNC         = 1,
1538         PROBE_TEST_DATA         = 2
1539 };
1540
1541 static void
1542 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1543                                   void *data, int flags)
1544 {
1545         struct ftrace_func_probe *entry;
1546         struct hlist_node *n, *tmp;
1547         char str[KSYM_SYMBOL_LEN];
1548         int type = MATCH_FULL;
1549         int i, len = 0;
1550         char *search;
1551
1552         if (glob && (strcmp(glob, "*") || !strlen(glob)))
1553                 glob = NULL;
1554         else {
1555                 int not;
1556
1557                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1558                 len = strlen(search);
1559
1560                 /* we do not support '!' for function probes */
1561                 if (WARN_ON(not))
1562                         return;
1563         }
1564
1565         mutex_lock(&ftrace_lock);
1566         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1567                 struct hlist_head *hhd = &ftrace_func_hash[i];
1568
1569                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1570
1571                         /* break up if statements for readability */
1572                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1573                                 continue;
1574
1575                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
1576                                 continue;
1577
1578                         /* do this last, since it is the most expensive */
1579                         if (glob) {
1580                                 kallsyms_lookup(entry->ip, NULL, NULL,
1581                                                 NULL, str);
1582                                 if (!ftrace_match(str, glob, len, type))
1583                                         continue;
1584                         }
1585
1586                         hlist_del(&entry->node);
1587                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1588                 }
1589         }
1590         __disable_ftrace_function_probe();
1591         mutex_unlock(&ftrace_lock);
1592 }
1593
1594 void
1595 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1596                                 void *data)
1597 {
1598         __unregister_ftrace_function_probe(glob, ops, data,
1599                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
1600 }
1601
1602 void
1603 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1604 {
1605         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1606 }
1607
1608 void unregister_ftrace_function_probe_all(char *glob)
1609 {
1610         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1611 }
1612
1613 static LIST_HEAD(ftrace_commands);
1614 static DEFINE_MUTEX(ftrace_cmd_mutex);
1615
1616 int register_ftrace_command(struct ftrace_func_command *cmd)
1617 {
1618         struct ftrace_func_command *p;
1619         int ret = 0;
1620
1621         mutex_lock(&ftrace_cmd_mutex);
1622         list_for_each_entry(p, &ftrace_commands, list) {
1623                 if (strcmp(cmd->name, p->name) == 0) {
1624                         ret = -EBUSY;
1625                         goto out_unlock;
1626                 }
1627         }
1628         list_add(&cmd->list, &ftrace_commands);
1629  out_unlock:
1630         mutex_unlock(&ftrace_cmd_mutex);
1631
1632         return ret;
1633 }
1634
1635 int unregister_ftrace_command(struct ftrace_func_command *cmd)
1636 {
1637         struct ftrace_func_command *p, *n;
1638         int ret = -ENODEV;
1639
1640         mutex_lock(&ftrace_cmd_mutex);
1641         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1642                 if (strcmp(cmd->name, p->name) == 0) {
1643                         ret = 0;
1644                         list_del_init(&p->list);
1645                         goto out_unlock;
1646                 }
1647         }
1648  out_unlock:
1649         mutex_unlock(&ftrace_cmd_mutex);
1650
1651         return ret;
1652 }
1653
1654 static int ftrace_process_regex(char *buff, int len, int enable)
1655 {
1656         char *func, *command, *next = buff;
1657         struct ftrace_func_command *p;
1658         int ret = -EINVAL;
1659
1660         func = strsep(&next, ":");
1661
1662         if (!next) {
1663                 ftrace_match_records(func, len, enable);
1664                 return 0;
1665         }
1666
1667         /* command found */
1668
1669         command = strsep(&next, ":");
1670
1671         mutex_lock(&ftrace_cmd_mutex);
1672         list_for_each_entry(p, &ftrace_commands, list) {
1673                 if (strcmp(p->name, command) == 0) {
1674                         ret = p->func(func, command, next, enable);
1675                         goto out_unlock;
1676                 }
1677         }
1678  out_unlock:
1679         mutex_unlock(&ftrace_cmd_mutex);
1680
1681         return ret;
1682 }
1683
1684 static ssize_t
1685 ftrace_regex_write(struct file *file, const char __user *ubuf,
1686                    size_t cnt, loff_t *ppos, int enable)
1687 {
1688         struct ftrace_iterator *iter;
1689         char ch;
1690         size_t read = 0;
1691         ssize_t ret;
1692
1693         if (!cnt || cnt < 0)
1694                 return 0;
1695
1696         mutex_lock(&ftrace_regex_lock);
1697
1698         if (file->f_mode & FMODE_READ) {
1699                 struct seq_file *m = file->private_data;
1700                 iter = m->private;
1701         } else
1702                 iter = file->private_data;
1703
1704         if (!*ppos) {
1705                 iter->flags &= ~FTRACE_ITER_CONT;
1706                 iter->buffer_idx = 0;
1707         }
1708
1709         ret = get_user(ch, ubuf++);
1710         if (ret)
1711                 goto out;
1712         read++;
1713         cnt--;
1714
1715         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1716                 /* skip white space */
1717                 while (cnt && isspace(ch)) {
1718                         ret = get_user(ch, ubuf++);
1719                         if (ret)
1720                                 goto out;
1721                         read++;
1722                         cnt--;
1723                 }
1724
1725                 if (isspace(ch)) {
1726                         file->f_pos += read;
1727                         ret = read;
1728                         goto out;
1729                 }
1730
1731                 iter->buffer_idx = 0;
1732         }
1733
1734         while (cnt && !isspace(ch)) {
1735                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1736                         iter->buffer[iter->buffer_idx++] = ch;
1737                 else {
1738                         ret = -EINVAL;
1739                         goto out;
1740                 }
1741                 ret = get_user(ch, ubuf++);
1742                 if (ret)
1743                         goto out;
1744                 read++;
1745                 cnt--;
1746         }
1747
1748         if (isspace(ch)) {
1749                 iter->filtered++;
1750                 iter->buffer[iter->buffer_idx] = 0;
1751                 ret = ftrace_process_regex(iter->buffer,
1752                                            iter->buffer_idx, enable);
1753                 if (ret)
1754                         goto out;
1755                 iter->buffer_idx = 0;
1756         } else
1757                 iter->flags |= FTRACE_ITER_CONT;
1758
1759
1760         file->f_pos += read;
1761
1762         ret = read;
1763  out:
1764         mutex_unlock(&ftrace_regex_lock);
1765
1766         return ret;
1767 }
1768
1769 static ssize_t
1770 ftrace_filter_write(struct file *file, const char __user *ubuf,
1771                     size_t cnt, loff_t *ppos)
1772 {
1773         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1774 }
1775
1776 static ssize_t
1777 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1778                      size_t cnt, loff_t *ppos)
1779 {
1780         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1781 }
1782
1783 static void
1784 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1785 {
1786         if (unlikely(ftrace_disabled))
1787                 return;
1788
1789         mutex_lock(&ftrace_regex_lock);
1790         if (reset)
1791                 ftrace_filter_reset(enable);
1792         if (buf)
1793                 ftrace_match_records(buf, len, enable);
1794         mutex_unlock(&ftrace_regex_lock);
1795 }
1796
1797 /**
1798  * ftrace_set_filter - set a function to filter on in ftrace
1799  * @buf - the string that holds the function filter text.
1800  * @len - the length of the string.
1801  * @reset - non zero to reset all filters before applying this filter.
1802  *
1803  * Filters denote which functions should be enabled when tracing is enabled.
1804  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1805  */
1806 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1807 {
1808         ftrace_set_regex(buf, len, reset, 1);
1809 }
1810
1811 /**
1812  * ftrace_set_notrace - set a function to not trace in ftrace
1813  * @buf - the string that holds the function notrace text.
1814  * @len - the length of the string.
1815  * @reset - non zero to reset all filters before applying this filter.
1816  *
1817  * Notrace Filters denote which functions should not be enabled when tracing
1818  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1819  * for tracing.
1820  */
1821 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1822 {
1823         ftrace_set_regex(buf, len, reset, 0);
1824 }
1825
1826 static int
1827 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1828 {
1829         struct seq_file *m = (struct seq_file *)file->private_data;
1830         struct ftrace_iterator *iter;
1831
1832         mutex_lock(&ftrace_regex_lock);
1833         if (file->f_mode & FMODE_READ) {
1834                 iter = m->private;
1835
1836                 seq_release(inode, file);
1837         } else
1838                 iter = file->private_data;
1839
1840         if (iter->buffer_idx) {
1841                 iter->filtered++;
1842                 iter->buffer[iter->buffer_idx] = 0;
1843                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1844         }
1845
1846         mutex_lock(&ftrace_lock);
1847         if (ftrace_start_up && ftrace_enabled)
1848                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1849         mutex_unlock(&ftrace_lock);
1850
1851         kfree(iter);
1852         mutex_unlock(&ftrace_regex_lock);
1853         return 0;
1854 }
1855
1856 static int
1857 ftrace_filter_release(struct inode *inode, struct file *file)
1858 {
1859         return ftrace_regex_release(inode, file, 1);
1860 }
1861
1862 static int
1863 ftrace_notrace_release(struct inode *inode, struct file *file)
1864 {
1865         return ftrace_regex_release(inode, file, 0);
1866 }
1867
1868 static const struct file_operations ftrace_avail_fops = {
1869         .open = ftrace_avail_open,
1870         .read = seq_read,
1871         .llseek = seq_lseek,
1872         .release = ftrace_avail_release,
1873 };
1874
1875 static const struct file_operations ftrace_failures_fops = {
1876         .open = ftrace_failures_open,
1877         .read = seq_read,
1878         .llseek = seq_lseek,
1879         .release = ftrace_avail_release,
1880 };
1881
1882 static const struct file_operations ftrace_filter_fops = {
1883         .open = ftrace_filter_open,
1884         .read = seq_read,
1885         .write = ftrace_filter_write,
1886         .llseek = ftrace_regex_lseek,
1887         .release = ftrace_filter_release,
1888 };
1889
1890 static const struct file_operations ftrace_notrace_fops = {
1891         .open = ftrace_notrace_open,
1892         .read = seq_read,
1893         .write = ftrace_notrace_write,
1894         .llseek = ftrace_regex_lseek,
1895         .release = ftrace_notrace_release,
1896 };
1897
1898 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1899
1900 static DEFINE_MUTEX(graph_lock);
1901
1902 int ftrace_graph_count;
1903 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1904
1905 static void *
1906 g_next(struct seq_file *m, void *v, loff_t *pos)
1907 {
1908         unsigned long *array = m->private;
1909         int index = *pos;
1910
1911         (*pos)++;
1912
1913         if (index >= ftrace_graph_count)
1914                 return NULL;
1915
1916         return &array[index];
1917 }
1918
1919 static void *g_start(struct seq_file *m, loff_t *pos)
1920 {
1921         void *p = NULL;
1922
1923         mutex_lock(&graph_lock);
1924
1925         /* Nothing, tell g_show to print all functions are enabled */
1926         if (!ftrace_graph_count && !*pos)
1927                 return (void *)1;
1928
1929         p = g_next(m, p, pos);
1930
1931         return p;
1932 }
1933
1934 static void g_stop(struct seq_file *m, void *p)
1935 {
1936         mutex_unlock(&graph_lock);
1937 }
1938
1939 static int g_show(struct seq_file *m, void *v)
1940 {
1941         unsigned long *ptr = v;
1942         char str[KSYM_SYMBOL_LEN];
1943
1944         if (!ptr)
1945                 return 0;
1946
1947         if (ptr == (unsigned long *)1) {
1948                 seq_printf(m, "#### all functions enabled ####\n");
1949                 return 0;
1950         }
1951
1952         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1953
1954         seq_printf(m, "%s\n", str);
1955
1956         return 0;
1957 }
1958
1959 static struct seq_operations ftrace_graph_seq_ops = {
1960         .start = g_start,
1961         .next = g_next,
1962         .stop = g_stop,
1963         .show = g_show,
1964 };
1965
1966 static int
1967 ftrace_graph_open(struct inode *inode, struct file *file)
1968 {
1969         int ret = 0;
1970
1971         if (unlikely(ftrace_disabled))
1972                 return -ENODEV;
1973
1974         mutex_lock(&graph_lock);
1975         if ((file->f_mode & FMODE_WRITE) &&
1976             !(file->f_flags & O_APPEND)) {
1977                 ftrace_graph_count = 0;
1978                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1979         }
1980
1981         if (file->f_mode & FMODE_READ) {
1982                 ret = seq_open(file, &ftrace_graph_seq_ops);
1983                 if (!ret) {
1984                         struct seq_file *m = file->private_data;
1985                         m->private = ftrace_graph_funcs;
1986                 }
1987         } else
1988                 file->private_data = ftrace_graph_funcs;
1989         mutex_unlock(&graph_lock);
1990
1991         return ret;
1992 }
1993
1994 static int
1995 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
1996 {
1997         struct dyn_ftrace *rec;
1998         struct ftrace_page *pg;
1999         int search_len;
2000         int found = 0;
2001         int type, not;
2002         char *search;
2003         bool exists;
2004         int i;
2005
2006         if (ftrace_disabled)
2007                 return -ENODEV;
2008
2009         /* decode regex */
2010         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2011         if (not)
2012                 return -EINVAL;
2013
2014         search_len = strlen(search);
2015
2016         mutex_lock(&ftrace_lock);
2017         do_for_each_ftrace_rec(pg, rec) {
2018
2019                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2020                         break;
2021
2022                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2023                         continue;
2024
2025                 if (ftrace_match_record(rec, search, search_len, type)) {
2026                         /* ensure it is not already in the array */
2027                         exists = false;
2028                         for (i = 0; i < *idx; i++)
2029                                 if (array[i] == rec->ip) {
2030                                         exists = true;
2031                                         break;
2032                                 }
2033                         if (!exists) {
2034                                 array[(*idx)++] = rec->ip;
2035                                 found = 1;
2036                         }
2037                 }
2038         } while_for_each_ftrace_rec();
2039
2040         mutex_unlock(&ftrace_lock);
2041
2042         return found ? 0 : -EINVAL;
2043 }
2044
2045 static ssize_t
2046 ftrace_graph_write(struct file *file, const char __user *ubuf,
2047                    size_t cnt, loff_t *ppos)
2048 {
2049         unsigned char buffer[FTRACE_BUFF_MAX+1];
2050         unsigned long *array;
2051         size_t read = 0;
2052         ssize_t ret;
2053         int index = 0;
2054         char ch;
2055
2056         if (!cnt || cnt < 0)
2057                 return 0;
2058
2059         mutex_lock(&graph_lock);
2060
2061         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2062                 ret = -EBUSY;
2063                 goto out;
2064         }
2065
2066         if (file->f_mode & FMODE_READ) {
2067                 struct seq_file *m = file->private_data;
2068                 array = m->private;
2069         } else
2070                 array = file->private_data;
2071
2072         ret = get_user(ch, ubuf++);
2073         if (ret)
2074                 goto out;
2075         read++;
2076         cnt--;
2077
2078         /* skip white space */
2079         while (cnt && isspace(ch)) {
2080                 ret = get_user(ch, ubuf++);
2081                 if (ret)
2082                         goto out;
2083                 read++;
2084                 cnt--;
2085         }
2086
2087         if (isspace(ch)) {
2088                 *ppos += read;
2089                 ret = read;
2090                 goto out;
2091         }
2092
2093         while (cnt && !isspace(ch)) {
2094                 if (index < FTRACE_BUFF_MAX)
2095                         buffer[index++] = ch;
2096                 else {
2097                         ret = -EINVAL;
2098                         goto out;
2099                 }
2100                 ret = get_user(ch, ubuf++);
2101                 if (ret)
2102                         goto out;
2103                 read++;
2104                 cnt--;
2105         }
2106         buffer[index] = 0;
2107
2108         /* we allow only one expression at a time */
2109         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2110         if (ret)
2111                 goto out;
2112
2113         file->f_pos += read;
2114
2115         ret = read;
2116  out:
2117         mutex_unlock(&graph_lock);
2118
2119         return ret;
2120 }
2121
2122 static const struct file_operations ftrace_graph_fops = {
2123         .open = ftrace_graph_open,
2124         .read = seq_read,
2125         .write = ftrace_graph_write,
2126 };
2127 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2128
2129 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2130 {
2131         struct dentry *entry;
2132
2133         entry = debugfs_create_file("available_filter_functions", 0444,
2134                                     d_tracer, NULL, &ftrace_avail_fops);
2135         if (!entry)
2136                 pr_warning("Could not create debugfs "
2137                            "'available_filter_functions' entry\n");
2138
2139         entry = debugfs_create_file("failures", 0444,
2140                                     d_tracer, NULL, &ftrace_failures_fops);
2141         if (!entry)
2142                 pr_warning("Could not create debugfs 'failures' entry\n");
2143
2144         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2145                                     NULL, &ftrace_filter_fops);
2146         if (!entry)
2147                 pr_warning("Could not create debugfs "
2148                            "'set_ftrace_filter' entry\n");
2149
2150         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2151                                     NULL, &ftrace_notrace_fops);
2152         if (!entry)
2153                 pr_warning("Could not create debugfs "
2154                            "'set_ftrace_notrace' entry\n");
2155
2156 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2157         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2158                                     NULL,
2159                                     &ftrace_graph_fops);
2160         if (!entry)
2161                 pr_warning("Could not create debugfs "
2162                            "'set_graph_function' entry\n");
2163 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2164
2165         return 0;
2166 }
2167
2168 static int ftrace_convert_nops(struct module *mod,
2169                                unsigned long *start,
2170                                unsigned long *end)
2171 {
2172         unsigned long *p;
2173         unsigned long addr;
2174         unsigned long flags;
2175
2176         mutex_lock(&ftrace_lock);
2177         p = start;
2178         while (p < end) {
2179                 addr = ftrace_call_adjust(*p++);
2180                 /*
2181                  * Some architecture linkers will pad between
2182                  * the different mcount_loc sections of different
2183                  * object files to satisfy alignments.
2184                  * Skip any NULL pointers.
2185                  */
2186                 if (!addr)
2187                         continue;
2188                 ftrace_record_ip(addr);
2189         }
2190
2191         /* disable interrupts to prevent kstop machine */
2192         local_irq_save(flags);
2193         ftrace_update_code(mod);
2194         local_irq_restore(flags);
2195         mutex_unlock(&ftrace_lock);
2196
2197         return 0;
2198 }
2199
2200 void ftrace_init_module(struct module *mod,
2201                         unsigned long *start, unsigned long *end)
2202 {
2203         if (ftrace_disabled || start == end)
2204                 return;
2205         ftrace_convert_nops(mod, start, end);
2206 }
2207
2208 extern unsigned long __start_mcount_loc[];
2209 extern unsigned long __stop_mcount_loc[];
2210
2211 void __init ftrace_init(void)
2212 {
2213         unsigned long count, addr, flags;
2214         int ret;
2215
2216         /* Keep the ftrace pointer to the stub */
2217         addr = (unsigned long)ftrace_stub;
2218
2219         local_irq_save(flags);
2220         ftrace_dyn_arch_init(&addr);
2221         local_irq_restore(flags);
2222
2223         /* ftrace_dyn_arch_init places the return code in addr */
2224         if (addr)
2225                 goto failed;
2226
2227         count = __stop_mcount_loc - __start_mcount_loc;
2228
2229         ret = ftrace_dyn_table_alloc(count);
2230         if (ret)
2231                 goto failed;
2232
2233         last_ftrace_enabled = ftrace_enabled = 1;
2234
2235         ret = ftrace_convert_nops(NULL,
2236                                   __start_mcount_loc,
2237                                   __stop_mcount_loc);
2238
2239         return;
2240  failed:
2241         ftrace_disabled = 1;
2242 }
2243
2244 #else
2245
2246 static int __init ftrace_nodyn_init(void)
2247 {
2248         ftrace_enabled = 1;
2249         return 0;
2250 }
2251 device_initcall(ftrace_nodyn_init);
2252
2253 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2254 static inline void ftrace_startup_enable(int command) { }
2255 /* Keep as macros so we do not need to define the commands */
2256 # define ftrace_startup(command)        do { } while (0)
2257 # define ftrace_shutdown(command)       do { } while (0)
2258 # define ftrace_startup_sysctl()        do { } while (0)
2259 # define ftrace_shutdown_sysctl()       do { } while (0)
2260 #endif /* CONFIG_DYNAMIC_FTRACE */
2261
2262 static ssize_t
2263 ftrace_pid_read(struct file *file, char __user *ubuf,
2264                        size_t cnt, loff_t *ppos)
2265 {
2266         char buf[64];
2267         int r;
2268
2269         if (ftrace_pid_trace == ftrace_swapper_pid)
2270                 r = sprintf(buf, "swapper tasks\n");
2271         else if (ftrace_pid_trace)
2272                 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2273         else
2274                 r = sprintf(buf, "no pid\n");
2275
2276         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2277 }
2278
2279 static void clear_ftrace_swapper(void)
2280 {
2281         struct task_struct *p;
2282         int cpu;
2283
2284         get_online_cpus();
2285         for_each_online_cpu(cpu) {
2286                 p = idle_task(cpu);
2287                 clear_tsk_trace_trace(p);
2288         }
2289         put_online_cpus();
2290 }
2291
2292 static void set_ftrace_swapper(void)
2293 {
2294         struct task_struct *p;
2295         int cpu;
2296
2297         get_online_cpus();
2298         for_each_online_cpu(cpu) {
2299                 p = idle_task(cpu);
2300                 set_tsk_trace_trace(p);
2301         }
2302         put_online_cpus();
2303 }
2304
2305 static void clear_ftrace_pid(struct pid *pid)
2306 {
2307         struct task_struct *p;
2308
2309         rcu_read_lock();
2310         do_each_pid_task(pid, PIDTYPE_PID, p) {
2311                 clear_tsk_trace_trace(p);
2312         } while_each_pid_task(pid, PIDTYPE_PID, p);
2313         rcu_read_unlock();
2314
2315         put_pid(pid);
2316 }
2317
2318 static void set_ftrace_pid(struct pid *pid)
2319 {
2320         struct task_struct *p;
2321
2322         rcu_read_lock();
2323         do_each_pid_task(pid, PIDTYPE_PID, p) {
2324                 set_tsk_trace_trace(p);
2325         } while_each_pid_task(pid, PIDTYPE_PID, p);
2326         rcu_read_unlock();
2327 }
2328
2329 static void clear_ftrace_pid_task(struct pid **pid)
2330 {
2331         if (*pid == ftrace_swapper_pid)
2332                 clear_ftrace_swapper();
2333         else
2334                 clear_ftrace_pid(*pid);
2335
2336         *pid = NULL;
2337 }
2338
2339 static void set_ftrace_pid_task(struct pid *pid)
2340 {
2341         if (pid == ftrace_swapper_pid)
2342                 set_ftrace_swapper();
2343         else
2344                 set_ftrace_pid(pid);
2345 }
2346
2347 static ssize_t
2348 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2349                    size_t cnt, loff_t *ppos)
2350 {
2351         struct pid *pid;
2352         char buf[64];
2353         long val;
2354         int ret;
2355
2356         if (cnt >= sizeof(buf))
2357                 return -EINVAL;
2358
2359         if (copy_from_user(&buf, ubuf, cnt))
2360                 return -EFAULT;
2361
2362         buf[cnt] = 0;
2363
2364         ret = strict_strtol(buf, 10, &val);
2365         if (ret < 0)
2366                 return ret;
2367
2368         mutex_lock(&ftrace_lock);
2369         if (val < 0) {
2370                 /* disable pid tracing */
2371                 if (!ftrace_pid_trace)
2372                         goto out;
2373
2374                 clear_ftrace_pid_task(&ftrace_pid_trace);
2375
2376         } else {
2377                 /* swapper task is special */
2378                 if (!val) {
2379                         pid = ftrace_swapper_pid;
2380                         if (pid == ftrace_pid_trace)
2381                                 goto out;
2382                 } else {
2383                         pid = find_get_pid(val);
2384
2385                         if (pid == ftrace_pid_trace) {
2386                                 put_pid(pid);
2387                                 goto out;
2388                         }
2389                 }
2390
2391                 if (ftrace_pid_trace)
2392                         clear_ftrace_pid_task(&ftrace_pid_trace);
2393
2394                 if (!pid)
2395                         goto out;
2396
2397                 ftrace_pid_trace = pid;
2398
2399                 set_ftrace_pid_task(ftrace_pid_trace);
2400         }
2401
2402         /* update the function call */
2403         ftrace_update_pid_func();
2404         ftrace_startup_enable(0);
2405
2406  out:
2407         mutex_unlock(&ftrace_lock);
2408
2409         return cnt;
2410 }
2411
2412 static const struct file_operations ftrace_pid_fops = {
2413         .read = ftrace_pid_read,
2414         .write = ftrace_pid_write,
2415 };
2416
2417 static __init int ftrace_init_debugfs(void)
2418 {
2419         struct dentry *d_tracer;
2420         struct dentry *entry;
2421
2422         d_tracer = tracing_init_dentry();
2423         if (!d_tracer)
2424                 return 0;
2425
2426         ftrace_init_dyn_debugfs(d_tracer);
2427
2428         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2429                                     NULL, &ftrace_pid_fops);
2430         if (!entry)
2431                 pr_warning("Could not create debugfs "
2432                            "'set_ftrace_pid' entry\n");
2433         return 0;
2434 }
2435 fs_initcall(ftrace_init_debugfs);
2436
2437 /**
2438  * ftrace_kill - kill ftrace
2439  *
2440  * This function should be used by panic code. It stops ftrace
2441  * but in a not so nice way. If you need to simply kill ftrace
2442  * from a non-atomic section, use ftrace_kill.
2443  */
2444 void ftrace_kill(void)
2445 {
2446         ftrace_disabled = 1;
2447         ftrace_enabled = 0;
2448         clear_ftrace_function();
2449 }
2450
2451 /**
2452  * register_ftrace_function - register a function for profiling
2453  * @ops - ops structure that holds the function for profiling.
2454  *
2455  * Register a function to be called by all functions in the
2456  * kernel.
2457  *
2458  * Note: @ops->func and all the functions it calls must be labeled
2459  *       with "notrace", otherwise it will go into a
2460  *       recursive loop.
2461  */
2462 int register_ftrace_function(struct ftrace_ops *ops)
2463 {
2464         int ret;
2465
2466         if (unlikely(ftrace_disabled))
2467                 return -1;
2468
2469         mutex_lock(&ftrace_lock);
2470
2471         ret = __register_ftrace_function(ops);
2472         ftrace_startup(0);
2473
2474         mutex_unlock(&ftrace_lock);
2475         return ret;
2476 }
2477
2478 /**
2479  * unregister_ftrace_function - unregister a function for profiling.
2480  * @ops - ops structure that holds the function to unregister
2481  *
2482  * Unregister a function that was added to be called by ftrace profiling.
2483  */
2484 int unregister_ftrace_function(struct ftrace_ops *ops)
2485 {
2486         int ret;
2487
2488         mutex_lock(&ftrace_lock);
2489         ret = __unregister_ftrace_function(ops);
2490         ftrace_shutdown(0);
2491         mutex_unlock(&ftrace_lock);
2492
2493         return ret;
2494 }
2495
2496 int
2497 ftrace_enable_sysctl(struct ctl_table *table, int write,
2498                      struct file *file, void __user *buffer, size_t *lenp,
2499                      loff_t *ppos)
2500 {
2501         int ret;
2502
2503         if (unlikely(ftrace_disabled))
2504                 return -ENODEV;
2505
2506         mutex_lock(&ftrace_lock);
2507
2508         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
2509
2510         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
2511                 goto out;
2512
2513         last_ftrace_enabled = ftrace_enabled;
2514
2515         if (ftrace_enabled) {
2516
2517                 ftrace_startup_sysctl();
2518
2519                 /* we are starting ftrace again */
2520                 if (ftrace_list != &ftrace_list_end) {
2521                         if (ftrace_list->next == &ftrace_list_end)
2522                                 ftrace_trace_function = ftrace_list->func;
2523                         else
2524                                 ftrace_trace_function = ftrace_list_func;
2525                 }
2526
2527         } else {
2528                 /* stopping ftrace calls (just send to ftrace_stub) */
2529                 ftrace_trace_function = ftrace_stub;
2530
2531                 ftrace_shutdown_sysctl();
2532         }
2533
2534  out:
2535         mutex_unlock(&ftrace_lock);
2536         return ret;
2537 }
2538
2539 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2540
2541 static atomic_t ftrace_graph_active;
2542 static struct notifier_block ftrace_suspend_notifier;
2543
2544 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
2545 {
2546         return 0;
2547 }
2548
2549 /* The callbacks that hook a function */
2550 trace_func_graph_ret_t ftrace_graph_return =
2551                         (trace_func_graph_ret_t)ftrace_stub;
2552 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
2553
2554 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2555 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
2556 {
2557         int i;
2558         int ret = 0;
2559         unsigned long flags;
2560         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
2561         struct task_struct *g, *t;
2562
2563         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
2564                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
2565                                         * sizeof(struct ftrace_ret_stack),
2566                                         GFP_KERNEL);
2567                 if (!ret_stack_list[i]) {
2568                         start = 0;
2569                         end = i;
2570                         ret = -ENOMEM;
2571                         goto free;
2572                 }
2573         }
2574
2575         read_lock_irqsave(&tasklist_lock, flags);
2576         do_each_thread(g, t) {
2577                 if (start == end) {
2578                         ret = -EAGAIN;
2579                         goto unlock;
2580                 }
2581
2582                 if (t->ret_stack == NULL) {
2583                         t->curr_ret_stack = -1;
2584                         /* Make sure IRQs see the -1 first: */
2585                         barrier();
2586                         t->ret_stack = ret_stack_list[start++];
2587                         atomic_set(&t->tracing_graph_pause, 0);
2588                         atomic_set(&t->trace_overrun, 0);
2589                 }
2590         } while_each_thread(g, t);
2591
2592 unlock:
2593         read_unlock_irqrestore(&tasklist_lock, flags);
2594 free:
2595         for (i = start; i < end; i++)
2596                 kfree(ret_stack_list[i]);
2597         return ret;
2598 }
2599
2600 static void
2601 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
2602                                 struct task_struct *next)
2603 {
2604         unsigned long long timestamp;
2605         int index;
2606
2607         /*
2608          * Does the user want to count the time a function was asleep.
2609          * If so, do not update the time stamps.
2610          */
2611         if (trace_flags & TRACE_ITER_SLEEP_TIME)
2612                 return;
2613
2614         timestamp = trace_clock_local();
2615
2616         prev->ftrace_timestamp = timestamp;
2617
2618         /* only process tasks that we timestamped */
2619         if (!next->ftrace_timestamp)
2620                 return;
2621
2622         /*
2623          * Update all the counters in next to make up for the
2624          * time next was sleeping.
2625          */
2626         timestamp -= next->ftrace_timestamp;
2627
2628         for (index = next->curr_ret_stack; index >= 0; index--)
2629                 next->ret_stack[index].calltime += timestamp;
2630 }
2631
2632 /* Allocate a return stack for each task */
2633 static int start_graph_tracing(void)
2634 {
2635         struct ftrace_ret_stack **ret_stack_list;
2636         int ret, cpu;
2637
2638         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2639                                 sizeof(struct ftrace_ret_stack *),
2640                                 GFP_KERNEL);
2641
2642         if (!ret_stack_list)
2643                 return -ENOMEM;
2644
2645         /* The cpu_boot init_task->ret_stack will never be freed */
2646         for_each_online_cpu(cpu)
2647                 ftrace_graph_init_task(idle_task(cpu));
2648
2649         do {
2650                 ret = alloc_retstack_tasklist(ret_stack_list);
2651         } while (ret == -EAGAIN);
2652
2653         if (!ret) {
2654                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
2655                 if (ret)
2656                         pr_info("ftrace_graph: Couldn't activate tracepoint"
2657                                 " probe to kernel_sched_switch\n");
2658         }
2659
2660         kfree(ret_stack_list);
2661         return ret;
2662 }
2663
2664 /*
2665  * Hibernation protection.
2666  * The state of the current task is too much unstable during
2667  * suspend/restore to disk. We want to protect against that.
2668  */
2669 static int
2670 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2671                                                         void *unused)
2672 {
2673         switch (state) {
2674         case PM_HIBERNATION_PREPARE:
2675                 pause_graph_tracing();
2676                 break;
2677
2678         case PM_POST_HIBERNATION:
2679                 unpause_graph_tracing();
2680                 break;
2681         }
2682         return NOTIFY_DONE;
2683 }
2684
2685 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2686                         trace_func_graph_ent_t entryfunc)
2687 {
2688         int ret = 0;
2689
2690         mutex_lock(&ftrace_lock);
2691
2692         /* we currently allow only one tracer registered at a time */
2693         if (atomic_read(&ftrace_graph_active)) {
2694                 ret = -EBUSY;
2695                 goto out;
2696         }
2697
2698         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2699         register_pm_notifier(&ftrace_suspend_notifier);
2700
2701         atomic_inc(&ftrace_graph_active);
2702         ret = start_graph_tracing();
2703         if (ret) {
2704                 atomic_dec(&ftrace_graph_active);
2705                 goto out;
2706         }
2707
2708         ftrace_graph_return = retfunc;
2709         ftrace_graph_entry = entryfunc;
2710
2711         ftrace_startup(FTRACE_START_FUNC_RET);
2712
2713 out:
2714         mutex_unlock(&ftrace_lock);
2715         return ret;
2716 }
2717
2718 void unregister_ftrace_graph(void)
2719 {
2720         mutex_lock(&ftrace_lock);
2721
2722         if (!unlikely(atomic_read(&ftrace_graph_active)))
2723                 goto out;
2724
2725         atomic_dec(&ftrace_graph_active);
2726         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
2727         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2728         ftrace_graph_entry = ftrace_graph_entry_stub;
2729         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2730         unregister_pm_notifier(&ftrace_suspend_notifier);
2731
2732  out:
2733         mutex_unlock(&ftrace_lock);
2734 }
2735
2736 /* Allocate a return stack for newly created task */
2737 void ftrace_graph_init_task(struct task_struct *t)
2738 {
2739         if (atomic_read(&ftrace_graph_active)) {
2740                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2741                                 * sizeof(struct ftrace_ret_stack),
2742                                 GFP_KERNEL);
2743                 if (!t->ret_stack)
2744                         return;
2745                 t->curr_ret_stack = -1;
2746                 atomic_set(&t->tracing_graph_pause, 0);
2747                 atomic_set(&t->trace_overrun, 0);
2748                 t->ftrace_timestamp = 0;
2749         } else
2750                 t->ret_stack = NULL;
2751 }
2752
2753 void ftrace_graph_exit_task(struct task_struct *t)
2754 {
2755         struct ftrace_ret_stack *ret_stack = t->ret_stack;
2756
2757         t->ret_stack = NULL;
2758         /* NULL must become visible to IRQs before we free it: */
2759         barrier();
2760
2761         kfree(ret_stack);
2762 }
2763
2764 void ftrace_graph_stop(void)
2765 {
2766         ftrace_stop();
2767 }
2768 #endif
2769