git.oblomov.eu Git - linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <asm/ftrace.h>
  33
  34 #include "trace.h"
  35
  36 #define FTRACE_WARN_ON(cond)                    \
  37         do {                                    \
  38                 if (WARN_ON(cond))              \
  39                         ftrace_kill();          \
  40         } while (0)
  41
  42 #define FTRACE_WARN_ON_ONCE(cond)               \
  43         do {                                    \
  44                 if (WARN_ON_ONCE(cond))         \
  45                         ftrace_kill();          \
  46         } while (0)
  47
  48 /* hash bits for specific function selection */
  49 #define FTRACE_HASH_BITS 7
  50 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  51
  52 /* ftrace_enabled is a method to turn ftrace on or off */
  53 int ftrace_enabled __read_mostly;
  54 static int last_ftrace_enabled;
  55
  56 /* Quick disabling of function tracer. */
  57 int function_trace_stop;
  58
  59 /*
  60  * ftrace_disabled is set when an anomaly is discovered.
  61  * ftrace_disabled is much stronger than ftrace_enabled.
  62  */
  63 static int ftrace_disabled __read_mostly;
  64
  65 static DEFINE_MUTEX(ftrace_lock);
  66
  67 static struct ftrace_ops ftrace_list_end __read_mostly =
  68 {
  69         .func = ftrace_stub,
  70 };
  71
  72 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  73 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  74 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  75 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  76
  77 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  78 {
  79         struct ftrace_ops *op = ftrace_list;
  80
  81         /* in case someone actually ports this to alpha! */
  82         read_barrier_depends();
  83
  84         while (op != &ftrace_list_end) {
  85                 /* silly alpha */
  86                 read_barrier_depends();
  87                 op->func(ip, parent_ip);
  88                 op = op->next;
  89         };
  90 }
  91
  92 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  93 {
  94         if (!test_tsk_trace_trace(current))
  95                 return;
  96
  97         ftrace_pid_function(ip, parent_ip);
  98 }
  99
 100 static void set_ftrace_pid_function(ftrace_func_t func)
 101 {
 102         /* do not set ftrace_pid_function to itself! */
 103         if (func != ftrace_pid_func)
 104                 ftrace_pid_function = func;
 105 }
 106
 107 /**
 108  * clear_ftrace_function - reset the ftrace function
 109  *
 110  * This NULLs the ftrace function and in essence stops
 111  * tracing.  There may be lag
 112  */
 113 void clear_ftrace_function(void)
 114 {
 115         ftrace_trace_function = ftrace_stub;
 116         __ftrace_trace_function = ftrace_stub;
 117         ftrace_pid_function = ftrace_stub;
 118 }
 119
 120 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 121 /*
 122  * For those archs that do not test ftrace_trace_stop in their
 123  * mcount call site, we need to do it from C.
 124  */
 125 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 126 {
 127         if (function_trace_stop)
 128                 return;
 129
 130         __ftrace_trace_function(ip, parent_ip);
 131 }
 132 #endif
 133
 134 static int __register_ftrace_function(struct ftrace_ops *ops)
 135 {
 136         ops->next = ftrace_list;
 137         /*
 138          * We are entering ops into the ftrace_list but another
 139          * CPU might be walking that list. We need to make sure
 140          * the ops->next pointer is valid before another CPU sees
 141          * the ops pointer included into the ftrace_list.
 142          */
 143         smp_wmb();
 144         ftrace_list = ops;
 145
 146         if (ftrace_enabled) {
 147                 ftrace_func_t func;
 148
 149                 if (ops->next == &ftrace_list_end)
 150                         func = ops->func;
 151                 else
 152                         func = ftrace_list_func;
 153
 154                 if (ftrace_pid_trace) {
 155                         set_ftrace_pid_function(func);
 156                         func = ftrace_pid_func;
 157                 }
 158
 159                 /*
 160                  * For one func, simply call it directly.
 161                  * For more than one func, call the chain.
 162                  */
 163 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 164                 ftrace_trace_function = func;
 165 #else
 166                 __ftrace_trace_function = func;
 167                 ftrace_trace_function = ftrace_test_stop_func;
 168 #endif
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 175 {
 176         struct ftrace_ops **p;
 177
 178         /*
 179          * If we are removing the last function, then simply point
 180          * to the ftrace_stub.
 181          */
 182         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 183                 ftrace_trace_function = ftrace_stub;
 184                 ftrace_list = &ftrace_list_end;
 185                 return 0;
 186         }
 187
 188         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 189                 if (*p == ops)
 190                         break;
 191
 192         if (*p != ops)
 193                 return -1;
 194
 195         *p = (*p)->next;
 196
 197         if (ftrace_enabled) {
 198                 /* If we only have one func left, then call that directly */
 199                 if (ftrace_list->next == &ftrace_list_end) {
 200                         ftrace_func_t func = ftrace_list->func;
 201
 202                         if (ftrace_pid_trace) {
 203                                 set_ftrace_pid_function(func);
 204                                 func = ftrace_pid_func;
 205                         }
 206 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 207                         ftrace_trace_function = func;
 208 #else
 209                         __ftrace_trace_function = func;
 210 #endif
 211                 }
 212         }
 213
 214         return 0;
 215 }
 216
 217 static void ftrace_update_pid_func(void)
 218 {
 219         ftrace_func_t func;
 220
 221         if (ftrace_trace_function == ftrace_stub)
 222                 return;
 223
 224         func = ftrace_trace_function;
 225
 226         if (ftrace_pid_trace) {
 227                 set_ftrace_pid_function(func);
 228                 func = ftrace_pid_func;
 229         } else {
 230                 if (func == ftrace_pid_func)
 231                         func = ftrace_pid_function;
 232         }
 233
 234 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 235         ftrace_trace_function = func;
 236 #else
 237         __ftrace_trace_function = func;
 238 #endif
 239 }
 240
 241 /* set when tracing only a pid */
 242 struct pid *ftrace_pid_trace;
 243 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 244
 245 #ifdef CONFIG_DYNAMIC_FTRACE
 246
 247 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 248 # error Dynamic ftrace depends on MCOUNT_RECORD
 249 #endif
 250
 251 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 252
 253 struct ftrace_func_probe {
 254         struct hlist_node       node;
 255         struct ftrace_probe_ops *ops;
 256         unsigned long           flags;
 257         unsigned long           ip;
 258         void                    *data;
 259         struct rcu_head         rcu;
 260 };
 261
 262
 263 enum {
 264         FTRACE_ENABLE_CALLS             = (1 << 0),
 265         FTRACE_DISABLE_CALLS            = (1 << 1),
 266         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 267         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 268         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 269         FTRACE_START_FUNC_RET           = (1 << 5),
 270         FTRACE_STOP_FUNC_RET            = (1 << 6),
 271 };
 272
 273 static int ftrace_filtered;
 274
 275 static LIST_HEAD(ftrace_new_addrs);
 276
 277 static DEFINE_MUTEX(ftrace_regex_lock);
 278
 279 struct ftrace_page {
 280         struct ftrace_page      *next;
 281         int                     index;
 282         struct dyn_ftrace       records[];
 283 };
 284
 285 #define ENTRIES_PER_PAGE \
 286   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 287
 288 /* estimate from running different kernels */
 289 #define NR_TO_INIT              10000
 290
 291 static struct ftrace_page       *ftrace_pages_start;
 292 static struct ftrace_page       *ftrace_pages;
 293
 294 static struct dyn_ftrace *ftrace_free_records;
 295
 296 /*
 297  * This is a double for. Do not use 'break' to break out of the loop,
 298  * you must use a goto.
 299  */
 300 #define do_for_each_ftrace_rec(pg, rec)                                 \
 301         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 302                 int _____i;                                             \
 303                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 304                         rec = &pg->records[_____i];
 305
 306 #define while_for_each_ftrace_rec()             \
 307                 }                               \
 308         }
 309
 310 #ifdef CONFIG_KPROBES
 311
 312 static int frozen_record_count;
 313
 314 static inline void freeze_record(struct dyn_ftrace *rec)
 315 {
 316         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 317                 rec->flags |= FTRACE_FL_FROZEN;
 318                 frozen_record_count++;
 319         }
 320 }
 321
 322 static inline void unfreeze_record(struct dyn_ftrace *rec)
 323 {
 324         if (rec->flags & FTRACE_FL_FROZEN) {
 325                 rec->flags &= ~FTRACE_FL_FROZEN;
 326                 frozen_record_count--;
 327         }
 328 }
 329
 330 static inline int record_frozen(struct dyn_ftrace *rec)
 331 {
 332         return rec->flags & FTRACE_FL_FROZEN;
 333 }
 334 #else
 335 # define freeze_record(rec)                     ({ 0; })
 336 # define unfreeze_record(rec)                   ({ 0; })
 337 # define record_frozen(rec)                     ({ 0; })
 338 #endif /* CONFIG_KPROBES */
 339
 340 static void ftrace_free_rec(struct dyn_ftrace *rec)
 341 {
 342         rec->ip = (unsigned long)ftrace_free_records;
 343         ftrace_free_records = rec;
 344         rec->flags |= FTRACE_FL_FREE;
 345 }
 346
 347 void ftrace_release(void *start, unsigned long size)
 348 {
 349         struct dyn_ftrace *rec;
 350         struct ftrace_page *pg;
 351         unsigned long s = (unsigned long)start;
 352         unsigned long e = s + size;
 353
 354         if (ftrace_disabled || !start)
 355                 return;
 356
 357         mutex_lock(&ftrace_lock);
 358         do_for_each_ftrace_rec(pg, rec) {
 359                 if ((rec->ip >= s) && (rec->ip < e))
 360                         ftrace_free_rec(rec);
 361         } while_for_each_ftrace_rec();
 362         mutex_unlock(&ftrace_lock);
 363 }
 364
 365 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 366 {
 367         struct dyn_ftrace *rec;
 368
 369         /* First check for freed records */
 370         if (ftrace_free_records) {
 371                 rec = ftrace_free_records;
 372
 373                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 374                         FTRACE_WARN_ON_ONCE(1);
 375                         ftrace_free_records = NULL;
 376                         return NULL;
 377                 }
 378
 379                 ftrace_free_records = (void *)rec->ip;
 380                 memset(rec, 0, sizeof(*rec));
 381                 return rec;
 382         }
 383
 384         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 385                 if (!ftrace_pages->next) {
 386                         /* allocate another page */
 387                         ftrace_pages->next =
 388                                 (void *)get_zeroed_page(GFP_KERNEL);
 389                         if (!ftrace_pages->next)
 390                                 return NULL;
 391                 }
 392                 ftrace_pages = ftrace_pages->next;
 393         }
 394
 395         return &ftrace_pages->records[ftrace_pages->index++];
 396 }
 397
 398 static struct dyn_ftrace *
 399 ftrace_record_ip(unsigned long ip)
 400 {
 401         struct dyn_ftrace *rec;
 402
 403         if (ftrace_disabled)
 404                 return NULL;
 405
 406         rec = ftrace_alloc_dyn_node(ip);
 407         if (!rec)
 408                 return NULL;
 409
 410         rec->ip = ip;
 411
 412         list_add(&rec->list, &ftrace_new_addrs);
 413
 414         return rec;
 415 }
 416
 417 static void print_ip_ins(const char *fmt, unsigned char *p)
 418 {
 419         int i;
 420
 421         printk(KERN_CONT "%s", fmt);
 422
 423         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 424                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 425 }
 426
 427 static void ftrace_bug(int failed, unsigned long ip)
 428 {
 429         switch (failed) {
 430         case -EFAULT:
 431                 FTRACE_WARN_ON_ONCE(1);
 432                 pr_info("ftrace faulted on modifying ");
 433                 print_ip_sym(ip);
 434                 break;
 435         case -EINVAL:
 436                 FTRACE_WARN_ON_ONCE(1);
 437                 pr_info("ftrace failed to modify ");
 438                 print_ip_sym(ip);
 439                 print_ip_ins(" actual: ", (unsigned char *)ip);
 440                 printk(KERN_CONT "\n");
 441                 break;
 442         case -EPERM:
 443                 FTRACE_WARN_ON_ONCE(1);
 444                 pr_info("ftrace faulted on writing ");
 445                 print_ip_sym(ip);
 446                 break;
 447         default:
 448                 FTRACE_WARN_ON_ONCE(1);
 449                 pr_info("ftrace faulted on unknown error ");
 450                 print_ip_sym(ip);
 451         }
 452 }
 453
 454
 455 static int
 456 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 457 {
 458         unsigned long ftrace_addr;
 459         unsigned long ip, fl;
 460
 461         ftrace_addr = (unsigned long)FTRACE_ADDR;
 462
 463         ip = rec->ip;
 464
 465         /*
 466          * If this record is not to be traced and
 467          * it is not enabled then do nothing.
 468          *
 469          * If this record is not to be traced and
 470          * it is enabled then disable it.
 471          *
 472          */
 473         if (rec->flags & FTRACE_FL_NOTRACE) {
 474                 if (rec->flags & FTRACE_FL_ENABLED)
 475                         rec->flags &= ~FTRACE_FL_ENABLED;
 476                 else
 477                         return 0;
 478
 479         } else if (ftrace_filtered && enable) {
 480                 /*
 481                  * Filtering is on:
 482                  */
 483
 484                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
 485
 486                 /* Record is filtered and enabled, do nothing */
 487                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
 488                         return 0;
 489
 490                 /* Record is not filtered or enabled, do nothing */
 491                 if (!fl)
 492                         return 0;
 493
 494                 /* Record is not filtered but enabled, disable it */
 495                 if (fl == FTRACE_FL_ENABLED)
 496                         rec->flags &= ~FTRACE_FL_ENABLED;
 497                 else
 498                 /* Otherwise record is filtered but not enabled, enable it */
 499                         rec->flags |= FTRACE_FL_ENABLED;
 500         } else {
 501                 /* Disable or not filtered */
 502
 503                 if (enable) {
 504                         /* if record is enabled, do nothing */
 505                         if (rec->flags & FTRACE_FL_ENABLED)
 506                                 return 0;
 507
 508                         rec->flags |= FTRACE_FL_ENABLED;
 509
 510                 } else {
 511
 512                         /* if record is not enabled, do nothing */
 513                         if (!(rec->flags & FTRACE_FL_ENABLED))
 514                                 return 0;
 515
 516                         rec->flags &= ~FTRACE_FL_ENABLED;
 517                 }
 518         }
 519
 520         if (rec->flags & FTRACE_FL_ENABLED)
 521                 return ftrace_make_call(rec, ftrace_addr);
 522         else
 523                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 524 }
 525
 526 static void ftrace_replace_code(int enable)
 527 {
 528         struct dyn_ftrace *rec;
 529         struct ftrace_page *pg;
 530         int failed;
 531
 532         do_for_each_ftrace_rec(pg, rec) {
 533                 /*
 534                  * Skip over free records and records that have
 535                  * failed.
 536                  */
 537                 if (rec->flags & FTRACE_FL_FREE ||
 538                     rec->flags & FTRACE_FL_FAILED)
 539                         continue;
 540
 541                 /* ignore updates to this record's mcount site */
 542                 if (get_kprobe((void *)rec->ip)) {
 543                         freeze_record(rec);
 544                         continue;
 545                 } else {
 546                         unfreeze_record(rec);
 547                 }
 548
 549                 failed = __ftrace_replace_code(rec, enable);
 550                 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 551                         rec->flags |= FTRACE_FL_FAILED;
 552                         if ((system_state == SYSTEM_BOOTING) ||
 553                             !core_kernel_text(rec->ip)) {
 554                                 ftrace_free_rec(rec);
 555                                 } else {
 556                                 ftrace_bug(failed, rec->ip);
 557                                         /* Stop processing */
 558                                         return;
 559                                 }
 560                 }
 561         } while_for_each_ftrace_rec();
 562 }
 563
 564 static int
 565 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 566 {
 567         unsigned long ip;
 568         int ret;
 569
 570         ip = rec->ip;
 571
 572         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
 573         if (ret) {
 574                 ftrace_bug(ret, ip);
 575                 rec->flags |= FTRACE_FL_FAILED;
 576                 return 0;
 577         }
 578         return 1;
 579 }
 580
 581 /*
 582  * archs can override this function if they must do something
 583  * before the modifying code is performed.
 584  */
 585 int __weak ftrace_arch_code_modify_prepare(void)
 586 {
 587         return 0;
 588 }
 589
 590 /*
 591  * archs can override this function if they must do something
 592  * after the modifying code is performed.
 593  */
 594 int __weak ftrace_arch_code_modify_post_process(void)
 595 {
 596         return 0;
 597 }
 598
 599 static int __ftrace_modify_code(void *data)
 600 {
 601         int *command = data;
 602
 603         if (*command & FTRACE_ENABLE_CALLS)
 604                 ftrace_replace_code(1);
 605         else if (*command & FTRACE_DISABLE_CALLS)
 606                 ftrace_replace_code(0);
 607
 608         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 609                 ftrace_update_ftrace_func(ftrace_trace_function);
 610
 611         if (*command & FTRACE_START_FUNC_RET)
 612                 ftrace_enable_ftrace_graph_caller();
 613         else if (*command & FTRACE_STOP_FUNC_RET)
 614                 ftrace_disable_ftrace_graph_caller();
 615
 616         return 0;
 617 }
 618
 619 static void ftrace_run_update_code(int command)
 620 {
 621         int ret;
 622
 623         ret = ftrace_arch_code_modify_prepare();
 624         FTRACE_WARN_ON(ret);
 625         if (ret)
 626                 return;
 627
 628         stop_machine(__ftrace_modify_code, &command, NULL);
 629
 630         ret = ftrace_arch_code_modify_post_process();
 631         FTRACE_WARN_ON(ret);
 632 }
 633
 634 static ftrace_func_t saved_ftrace_func;
 635 static int ftrace_start_up;
 636
 637 static void ftrace_startup_enable(int command)
 638 {
 639         if (saved_ftrace_func != ftrace_trace_function) {
 640                 saved_ftrace_func = ftrace_trace_function;
 641                 command |= FTRACE_UPDATE_TRACE_FUNC;
 642         }
 643
 644         if (!command || !ftrace_enabled)
 645                 return;
 646
 647         ftrace_run_update_code(command);
 648 }
 649
 650 static void ftrace_startup(int command)
 651 {
 652         if (unlikely(ftrace_disabled))
 653                 return;
 654
 655         ftrace_start_up++;
 656         command |= FTRACE_ENABLE_CALLS;
 657
 658         ftrace_startup_enable(command);
 659 }
 660
 661 static void ftrace_shutdown(int command)
 662 {
 663         if (unlikely(ftrace_disabled))
 664                 return;
 665
 666         ftrace_start_up--;
 667         if (!ftrace_start_up)
 668                 command |= FTRACE_DISABLE_CALLS;
 669
 670         if (saved_ftrace_func != ftrace_trace_function) {
 671                 saved_ftrace_func = ftrace_trace_function;
 672                 command |= FTRACE_UPDATE_TRACE_FUNC;
 673         }
 674
 675         if (!command || !ftrace_enabled)
 676                 return;
 677
 678         ftrace_run_update_code(command);
 679 }
 680
 681 static void ftrace_startup_sysctl(void)
 682 {
 683         int command = FTRACE_ENABLE_MCOUNT;
 684
 685         if (unlikely(ftrace_disabled))
 686                 return;
 687
 688         /* Force update next time */
 689         saved_ftrace_func = NULL;
 690         /* ftrace_start_up is true if we want ftrace running */
 691         if (ftrace_start_up)
 692                 command |= FTRACE_ENABLE_CALLS;
 693
 694         ftrace_run_update_code(command);
 695 }
 696
 697 static void ftrace_shutdown_sysctl(void)
 698 {
 699         int command = FTRACE_DISABLE_MCOUNT;
 700
 701         if (unlikely(ftrace_disabled))
 702                 return;
 703
 704         /* ftrace_start_up is true if ftrace is running */
 705         if (ftrace_start_up)
 706                 command |= FTRACE_DISABLE_CALLS;
 707
 708         ftrace_run_update_code(command);
 709 }
 710
 711 static cycle_t          ftrace_update_time;
 712 static unsigned long    ftrace_update_cnt;
 713 unsigned long           ftrace_update_tot_cnt;
 714
 715 static int ftrace_update_code(struct module *mod)
 716 {
 717         struct dyn_ftrace *p, *t;
 718         cycle_t start, stop;
 719
 720         start = ftrace_now(raw_smp_processor_id());
 721         ftrace_update_cnt = 0;
 722
 723         list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
 724
 725                 /* If something went wrong, bail without enabling anything */
 726                 if (unlikely(ftrace_disabled))
 727                         return -1;
 728
 729                 list_del_init(&p->list);
 730
 731                 /* convert record (i.e, patch mcount-call with NOP) */
 732                 if (ftrace_code_disable(mod, p)) {
 733                         p->flags |= FTRACE_FL_CONVERTED;
 734                         ftrace_update_cnt++;
 735                 } else
 736                         ftrace_free_rec(p);
 737         }
 738
 739         stop = ftrace_now(raw_smp_processor_id());
 740         ftrace_update_time = stop - start;
 741         ftrace_update_tot_cnt += ftrace_update_cnt;
 742
 743         return 0;
 744 }
 745
 746 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 747 {
 748         struct ftrace_page *pg;
 749         int cnt;
 750         int i;
 751
 752         /* allocate a few pages */
 753         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 754         if (!ftrace_pages_start)
 755                 return -1;
 756
 757         /*
 758          * Allocate a few more pages.
 759          *
 760          * TODO: have some parser search vmlinux before
 761          *   final linking to find all calls to ftrace.
 762          *   Then we can:
 763          *    a) know how many pages to allocate.
 764          *     and/or
 765          *    b) set up the table then.
 766          *
 767          *  The dynamic code is still necessary for
 768          *  modules.
 769          */
 770
 771         pg = ftrace_pages = ftrace_pages_start;
 772
 773         cnt = num_to_init / ENTRIES_PER_PAGE;
 774         pr_info("ftrace: allocating %ld entries in %d pages\n",
 775                 num_to_init, cnt + 1);
 776
 777         for (i = 0; i < cnt; i++) {
 778                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 779
 780                 /* If we fail, we'll try later anyway */
 781                 if (!pg->next)
 782                         break;
 783
 784                 pg = pg->next;
 785         }
 786
 787         return 0;
 788 }
 789
 790 enum {
 791         FTRACE_ITER_FILTER      = (1 << 0),
 792         FTRACE_ITER_CONT        = (1 << 1),
 793         FTRACE_ITER_NOTRACE     = (1 << 2),
 794         FTRACE_ITER_FAILURES    = (1 << 3),
 795         FTRACE_ITER_PRINTALL    = (1 << 4),
 796         FTRACE_ITER_HASH        = (1 << 5),
 797 };
 798
 799 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 800
 801 struct ftrace_iterator {
 802         struct ftrace_page      *pg;
 803         int                     hidx;
 804         int                     idx;
 805         unsigned                flags;
 806         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 807         unsigned                buffer_idx;
 808         unsigned                filtered;
 809 };
 810
 811 static void *
 812 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
 813 {
 814         struct ftrace_iterator *iter = m->private;
 815         struct hlist_node *hnd = v;
 816         struct hlist_head *hhd;
 817
 818         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
 819
 820         (*pos)++;
 821
 822  retry:
 823         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 824                 return NULL;
 825
 826         hhd = &ftrace_func_hash[iter->hidx];
 827
 828         if (hlist_empty(hhd)) {
 829                 iter->hidx++;
 830                 hnd = NULL;
 831                 goto retry;
 832         }
 833
 834         if (!hnd)
 835                 hnd = hhd->first;
 836         else {
 837                 hnd = hnd->next;
 838                 if (!hnd) {
 839                         iter->hidx++;
 840                         goto retry;
 841                 }
 842         }
 843
 844         return hnd;
 845 }
 846
 847 static void *t_hash_start(struct seq_file *m, loff_t *pos)
 848 {
 849         struct ftrace_iterator *iter = m->private;
 850         void *p = NULL;
 851
 852         iter->flags |= FTRACE_ITER_HASH;
 853
 854         return t_hash_next(m, p, pos);
 855 }
 856
 857 static int t_hash_show(struct seq_file *m, void *v)
 858 {
 859         struct ftrace_func_probe *rec;
 860         struct hlist_node *hnd = v;
 861         char str[KSYM_SYMBOL_LEN];
 862
 863         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
 864
 865         if (rec->ops->print)
 866                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
 867
 868         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 869         seq_printf(m, "%s:", str);
 870
 871         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
 872         seq_printf(m, "%s", str);
 873
 874         if (rec->data)
 875                 seq_printf(m, ":%p", rec->data);
 876         seq_putc(m, '\n');
 877
 878         return 0;
 879 }
 880
 881 static void *
 882 t_next(struct seq_file *m, void *v, loff_t *pos)
 883 {
 884         struct ftrace_iterator *iter = m->private;
 885         struct dyn_ftrace *rec = NULL;
 886
 887         if (iter->flags & FTRACE_ITER_HASH)
 888                 return t_hash_next(m, v, pos);
 889
 890         (*pos)++;
 891
 892         if (iter->flags & FTRACE_ITER_PRINTALL)
 893                 return NULL;
 894
 895  retry:
 896         if (iter->idx >= iter->pg->index) {
 897                 if (iter->pg->next) {
 898                         iter->pg = iter->pg->next;
 899                         iter->idx = 0;
 900                         goto retry;
 901                 } else {
 902                         iter->idx = -1;
 903                 }
 904         } else {
 905                 rec = &iter->pg->records[iter->idx++];
 906                 if ((rec->flags & FTRACE_FL_FREE) ||
 907
 908                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 909                      (rec->flags & FTRACE_FL_FAILED)) ||
 910
 911                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 912                      !(rec->flags & FTRACE_FL_FAILED)) ||
 913
 914                     ((iter->flags & FTRACE_ITER_FILTER) &&
 915                      !(rec->flags & FTRACE_FL_FILTER)) ||
 916
 917                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 918                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 919                         rec = NULL;
 920                         goto retry;
 921                 }
 922         }
 923
 924         return rec;
 925 }
 926
 927 static void *t_start(struct seq_file *m, loff_t *pos)
 928 {
 929         struct ftrace_iterator *iter = m->private;
 930         void *p = NULL;
 931
 932         mutex_lock(&ftrace_lock);
 933         /*
 934          * For set_ftrace_filter reading, if we have the filter
 935          * off, we can short cut and just print out that all
 936          * functions are enabled.
 937          */
 938         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
 939                 if (*pos > 0)
 940                         return t_hash_start(m, pos);
 941                 iter->flags |= FTRACE_ITER_PRINTALL;
 942                 (*pos)++;
 943                 return iter;
 944         }
 945
 946         if (iter->flags & FTRACE_ITER_HASH)
 947                 return t_hash_start(m, pos);
 948
 949         if (*pos > 0) {
 950                 if (iter->idx < 0)
 951                         return p;
 952                 (*pos)--;
 953                 iter->idx--;
 954         }
 955
 956         p = t_next(m, p, pos);
 957
 958         if (!p)
 959                 return t_hash_start(m, pos);
 960
 961         return p;
 962 }
 963
 964 static void t_stop(struct seq_file *m, void *p)
 965 {
 966         mutex_unlock(&ftrace_lock);
 967 }
 968
 969 static int t_show(struct seq_file *m, void *v)
 970 {
 971         struct ftrace_iterator *iter = m->private;
 972         struct dyn_ftrace *rec = v;
 973         char str[KSYM_SYMBOL_LEN];
 974
 975         if (iter->flags & FTRACE_ITER_HASH)
 976                 return t_hash_show(m, v);
 977
 978         if (iter->flags & FTRACE_ITER_PRINTALL) {
 979                 seq_printf(m, "#### all functions enabled ####\n");
 980                 return 0;
 981         }
 982
 983         if (!rec)
 984                 return 0;
 985
 986         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 987
 988         seq_printf(m, "%s\n", str);
 989
 990         return 0;
 991 }
 992
 993 static struct seq_operations show_ftrace_seq_ops = {
 994         .start = t_start,
 995         .next = t_next,
 996         .stop = t_stop,
 997         .show = t_show,
 998 };
 999
1000 static int
1001 ftrace_avail_open(struct inode *inode, struct file *file)
1002 {
1003         struct ftrace_iterator *iter;
1004         int ret;
1005
1006         if (unlikely(ftrace_disabled))
1007                 return -ENODEV;
1008
1009         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1010         if (!iter)
1011                 return -ENOMEM;
1012
1013         iter->pg = ftrace_pages_start;
1014
1015         ret = seq_open(file, &show_ftrace_seq_ops);
1016         if (!ret) {
1017                 struct seq_file *m = file->private_data;
1018
1019                 m->private = iter;
1020         } else {
1021                 kfree(iter);
1022         }
1023
1024         return ret;
1025 }
1026
1027 int ftrace_avail_release(struct inode *inode, struct file *file)
1028 {
1029         struct seq_file *m = (struct seq_file *)file->private_data;
1030         struct ftrace_iterator *iter = m->private;
1031
1032         seq_release(inode, file);
1033         kfree(iter);
1034
1035         return 0;
1036 }
1037
1038 static int
1039 ftrace_failures_open(struct inode *inode, struct file *file)
1040 {
1041         int ret;
1042         struct seq_file *m;
1043         struct ftrace_iterator *iter;
1044
1045         ret = ftrace_avail_open(inode, file);
1046         if (!ret) {
1047                 m = (struct seq_file *)file->private_data;
1048                 iter = (struct ftrace_iterator *)m->private;
1049                 iter->flags = FTRACE_ITER_FAILURES;
1050         }
1051
1052         return ret;
1053 }
1054
1055
1056 static void ftrace_filter_reset(int enable)
1057 {
1058         struct ftrace_page *pg;
1059         struct dyn_ftrace *rec;
1060         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1061
1062         mutex_lock(&ftrace_lock);
1063         if (enable)
1064                 ftrace_filtered = 0;
1065         do_for_each_ftrace_rec(pg, rec) {
1066                 if (rec->flags & FTRACE_FL_FAILED)
1067                         continue;
1068                 rec->flags &= ~type;
1069         } while_for_each_ftrace_rec();
1070         mutex_unlock(&ftrace_lock);
1071 }
1072
1073 static int
1074 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1075 {
1076         struct ftrace_iterator *iter;
1077         int ret = 0;
1078
1079         if (unlikely(ftrace_disabled))
1080                 return -ENODEV;
1081
1082         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1083         if (!iter)
1084                 return -ENOMEM;
1085
1086         mutex_lock(&ftrace_regex_lock);
1087         if ((file->f_mode & FMODE_WRITE) &&
1088             !(file->f_flags & O_APPEND))
1089                 ftrace_filter_reset(enable);
1090
1091         if (file->f_mode & FMODE_READ) {
1092                 iter->pg = ftrace_pages_start;
1093                 iter->flags = enable ? FTRACE_ITER_FILTER :
1094                         FTRACE_ITER_NOTRACE;
1095
1096                 ret = seq_open(file, &show_ftrace_seq_ops);
1097                 if (!ret) {
1098                         struct seq_file *m = file->private_data;
1099                         m->private = iter;
1100                 } else
1101                         kfree(iter);
1102         } else
1103                 file->private_data = iter;
1104         mutex_unlock(&ftrace_regex_lock);
1105
1106         return ret;
1107 }
1108
1109 static int
1110 ftrace_filter_open(struct inode *inode, struct file *file)
1111 {
1112         return ftrace_regex_open(inode, file, 1);
1113 }
1114
1115 static int
1116 ftrace_notrace_open(struct inode *inode, struct file *file)
1117 {
1118         return ftrace_regex_open(inode, file, 0);
1119 }
1120
1121 static ssize_t
1122 ftrace_regex_read(struct file *file, char __user *ubuf,
1123                        size_t cnt, loff_t *ppos)
1124 {
1125         if (file->f_mode & FMODE_READ)
1126                 return seq_read(file, ubuf, cnt, ppos);
1127         else
1128                 return -EPERM;
1129 }
1130
1131 static loff_t
1132 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1133 {
1134         loff_t ret;
1135
1136         if (file->f_mode & FMODE_READ)
1137                 ret = seq_lseek(file, offset, origin);
1138         else
1139                 file->f_pos = ret = 1;
1140
1141         return ret;
1142 }
1143
1144 enum {
1145         MATCH_FULL,
1146         MATCH_FRONT_ONLY,
1147         MATCH_MIDDLE_ONLY,
1148         MATCH_END_ONLY,
1149 };
1150
1151 /*
1152  * (static function - no need for kernel doc)
1153  *
1154  * Pass in a buffer containing a glob and this function will
1155  * set search to point to the search part of the buffer and
1156  * return the type of search it is (see enum above).
1157  * This does modify buff.
1158  *
1159  * Returns enum type.
1160  *  search returns the pointer to use for comparison.
1161  *  not returns 1 if buff started with a '!'
1162  *     0 otherwise.
1163  */
1164 static int
1165 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1166 {
1167         int type = MATCH_FULL;
1168         int i;
1169
1170         if (buff[0] == '!') {
1171                 *not = 1;
1172                 buff++;
1173                 len--;
1174         } else
1175                 *not = 0;
1176
1177         *search = buff;
1178
1179         for (i = 0; i < len; i++) {
1180                 if (buff[i] == '*') {
1181                         if (!i) {
1182                                 *search = buff + 1;
1183                                 type = MATCH_END_ONLY;
1184                         } else {
1185                                 if (type == MATCH_END_ONLY)
1186                                         type = MATCH_MIDDLE_ONLY;
1187                                 else
1188                                         type = MATCH_FRONT_ONLY;
1189                                 buff[i] = 0;
1190                                 break;
1191                         }
1192                 }
1193         }
1194
1195         return type;
1196 }
1197
1198 static int ftrace_match(char *str, char *regex, int len, int type)
1199 {
1200         int matched = 0;
1201         char *ptr;
1202
1203         switch (type) {
1204         case MATCH_FULL:
1205                 if (strcmp(str, regex) == 0)
1206                         matched = 1;
1207                 break;
1208         case MATCH_FRONT_ONLY:
1209                 if (strncmp(str, regex, len) == 0)
1210                         matched = 1;
1211                 break;
1212         case MATCH_MIDDLE_ONLY:
1213                 if (strstr(str, regex))
1214                         matched = 1;
1215                 break;
1216         case MATCH_END_ONLY:
1217                 ptr = strstr(str, regex);
1218                 if (ptr && (ptr[len] == 0))
1219                         matched = 1;
1220                 break;
1221         }
1222
1223         return matched;
1224 }
1225
1226 static int
1227 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1228 {
1229         char str[KSYM_SYMBOL_LEN];
1230
1231         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1232         return ftrace_match(str, regex, len, type);
1233 }
1234
1235 static void ftrace_match_records(char *buff, int len, int enable)
1236 {
1237         unsigned int search_len;
1238         struct ftrace_page *pg;
1239         struct dyn_ftrace *rec;
1240         unsigned long flag;
1241         char *search;
1242         int type;
1243         int not;
1244
1245         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1246         type = ftrace_setup_glob(buff, len, &search, &not);
1247
1248         search_len = strlen(search);
1249
1250         mutex_lock(&ftrace_lock);
1251         do_for_each_ftrace_rec(pg, rec) {
1252
1253                 if (rec->flags & FTRACE_FL_FAILED)
1254                         continue;
1255
1256                 if (ftrace_match_record(rec, search, search_len, type)) {
1257                         if (not)
1258                                 rec->flags &= ~flag;
1259                         else
1260                                 rec->flags |= flag;
1261                 }
1262                 /*
1263                  * Only enable filtering if we have a function that
1264                  * is filtered on.
1265                  */
1266                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1267                         ftrace_filtered = 1;
1268         } while_for_each_ftrace_rec();
1269         mutex_unlock(&ftrace_lock);
1270 }
1271
1272 static int
1273 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1274                            char *regex, int len, int type)
1275 {
1276         char str[KSYM_SYMBOL_LEN];
1277         char *modname;
1278
1279         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1280
1281         if (!modname || strcmp(modname, mod))
1282                 return 0;
1283
1284         /* blank search means to match all funcs in the mod */
1285         if (len)
1286                 return ftrace_match(str, regex, len, type);
1287         else
1288                 return 1;
1289 }
1290
1291 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1292 {
1293         unsigned search_len = 0;
1294         struct ftrace_page *pg;
1295         struct dyn_ftrace *rec;
1296         int type = MATCH_FULL;
1297         char *search = buff;
1298         unsigned long flag;
1299         int not = 0;
1300
1301         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1302
1303         /* blank or '*' mean the same */
1304         if (strcmp(buff, "*") == 0)
1305                 buff[0] = 0;
1306
1307         /* handle the case of 'dont filter this module' */
1308         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1309                 buff[0] = 0;
1310                 not = 1;
1311         }
1312
1313         if (strlen(buff)) {
1314                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1315                 search_len = strlen(search);
1316         }
1317
1318         mutex_lock(&ftrace_lock);
1319         do_for_each_ftrace_rec(pg, rec) {
1320
1321                 if (rec->flags & FTRACE_FL_FAILED)
1322                         continue;
1323
1324                 if (ftrace_match_module_record(rec, mod,
1325                                                search, search_len, type)) {
1326                         if (not)
1327                                 rec->flags &= ~flag;
1328                         else
1329                                 rec->flags |= flag;
1330                 }
1331                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1332                         ftrace_filtered = 1;
1333
1334         } while_for_each_ftrace_rec();
1335         mutex_unlock(&ftrace_lock);
1336 }
1337
1338 /*
1339  * We register the module command as a template to show others how
1340  * to register the a command as well.
1341  */
1342
1343 static int
1344 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1345 {
1346         char *mod;
1347
1348         /*
1349          * cmd == 'mod' because we only registered this func
1350          * for the 'mod' ftrace_func_command.
1351          * But if you register one func with multiple commands,
1352          * you can tell which command was used by the cmd
1353          * parameter.
1354          */
1355
1356         /* we must have a module name */
1357         if (!param)
1358                 return -EINVAL;
1359
1360         mod = strsep(&param, ":");
1361         if (!strlen(mod))
1362                 return -EINVAL;
1363
1364         ftrace_match_module_records(func, mod, enable);
1365         return 0;
1366 }
1367
1368 static struct ftrace_func_command ftrace_mod_cmd = {
1369         .name                   = "mod",
1370         .func                   = ftrace_mod_callback,
1371 };
1372
1373 static int __init ftrace_mod_cmd_init(void)
1374 {
1375         return register_ftrace_command(&ftrace_mod_cmd);
1376 }
1377 device_initcall(ftrace_mod_cmd_init);
1378
1379 static void
1380 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1381 {
1382         struct ftrace_func_probe *entry;
1383         struct hlist_head *hhd;
1384         struct hlist_node *n;
1385         unsigned long key;
1386         int resched;
1387
1388         key = hash_long(ip, FTRACE_HASH_BITS);
1389
1390         hhd = &ftrace_func_hash[key];
1391
1392         if (hlist_empty(hhd))
1393                 return;
1394
1395         /*
1396          * Disable preemption for these calls to prevent a RCU grace
1397          * period. This syncs the hash iteration and freeing of items
1398          * on the hash. rcu_read_lock is too dangerous here.
1399          */
1400         resched = ftrace_preempt_disable();
1401         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1402                 if (entry->ip == ip)
1403                         entry->ops->func(ip, parent_ip, &entry->data);
1404         }
1405         ftrace_preempt_enable(resched);
1406 }
1407
1408 static struct ftrace_ops trace_probe_ops __read_mostly =
1409 {
1410         .func = function_trace_probe_call,
1411 };
1412
1413 static int ftrace_probe_registered;
1414
1415 static void __enable_ftrace_function_probe(void)
1416 {
1417         int i;
1418
1419         if (ftrace_probe_registered)
1420                 return;
1421
1422         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1423                 struct hlist_head *hhd = &ftrace_func_hash[i];
1424                 if (hhd->first)
1425                         break;
1426         }
1427         /* Nothing registered? */
1428         if (i == FTRACE_FUNC_HASHSIZE)
1429                 return;
1430
1431         __register_ftrace_function(&trace_probe_ops);
1432         ftrace_startup(0);
1433         ftrace_probe_registered = 1;
1434 }
1435
1436 static void __disable_ftrace_function_probe(void)
1437 {
1438         int i;
1439
1440         if (!ftrace_probe_registered)
1441                 return;
1442
1443         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1444                 struct hlist_head *hhd = &ftrace_func_hash[i];
1445                 if (hhd->first)
1446                         return;
1447         }
1448
1449         /* no more funcs left */
1450         __unregister_ftrace_function(&trace_probe_ops);
1451         ftrace_shutdown(0);
1452         ftrace_probe_registered = 0;
1453 }
1454
1455
1456 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1457 {
1458         struct ftrace_func_probe *entry =
1459                 container_of(rhp, struct ftrace_func_probe, rcu);
1460
1461         if (entry->ops->free)
1462                 entry->ops->free(&entry->data);
1463         kfree(entry);
1464 }
1465
1466
1467 int
1468 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1469                               void *data)
1470 {
1471         struct ftrace_func_probe *entry;
1472         struct ftrace_page *pg;
1473         struct dyn_ftrace *rec;
1474         int type, len, not;
1475         unsigned long key;
1476         int count = 0;
1477         char *search;
1478
1479         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1480         len = strlen(search);
1481
1482         /* we do not support '!' for function probes */
1483         if (WARN_ON(not))
1484                 return -EINVAL;
1485
1486         mutex_lock(&ftrace_lock);
1487         do_for_each_ftrace_rec(pg, rec) {
1488
1489                 if (rec->flags & FTRACE_FL_FAILED)
1490                         continue;
1491
1492                 if (!ftrace_match_record(rec, search, len, type))
1493                         continue;
1494
1495                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1496                 if (!entry) {
1497                         /* If we did not process any, then return error */
1498                         if (!count)
1499                                 count = -ENOMEM;
1500                         goto out_unlock;
1501                 }
1502
1503                 count++;
1504
1505                 entry->data = data;
1506
1507                 /*
1508                  * The caller might want to do something special
1509                  * for each function we find. We call the callback
1510                  * to give the caller an opportunity to do so.
1511                  */
1512                 if (ops->callback) {
1513                         if (ops->callback(rec->ip, &entry->data) < 0) {
1514                                 /* caller does not like this func */
1515                                 kfree(entry);
1516                                 continue;
1517                         }
1518                 }
1519
1520                 entry->ops = ops;
1521                 entry->ip = rec->ip;
1522
1523                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1524                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1525
1526         } while_for_each_ftrace_rec();
1527         __enable_ftrace_function_probe();
1528
1529  out_unlock:
1530         mutex_unlock(&ftrace_lock);
1531
1532         return count;
1533 }
1534
1535 enum {
1536         PROBE_TEST_FUNC         = 1,
1537         PROBE_TEST_DATA         = 2
1538 };
1539
1540 static void
1541 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1542                                   void *data, int flags)
1543 {
1544         struct ftrace_func_probe *entry;
1545         struct hlist_node *n, *tmp;
1546         char str[KSYM_SYMBOL_LEN];
1547         int type = MATCH_FULL;
1548         int i, len = 0;
1549         char *search;
1550
1551         if (glob && (strcmp(glob, "*") || !strlen(glob)))
1552                 glob = NULL;
1553         else {
1554                 int not;
1555
1556                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1557                 len = strlen(search);
1558
1559                 /* we do not support '!' for function probes */
1560                 if (WARN_ON(not))
1561                         return;
1562         }
1563
1564         mutex_lock(&ftrace_lock);
1565         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1566                 struct hlist_head *hhd = &ftrace_func_hash[i];
1567
1568                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1569
1570                         /* break up if statements for readability */
1571                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1572                                 continue;
1573
1574                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
1575                                 continue;
1576
1577                         /* do this last, since it is the most expensive */
1578                         if (glob) {
1579                                 kallsyms_lookup(entry->ip, NULL, NULL,
1580                                                 NULL, str);
1581                                 if (!ftrace_match(str, glob, len, type))
1582                                         continue;
1583                         }
1584
1585                         hlist_del(&entry->node);
1586                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1587                 }
1588         }
1589         __disable_ftrace_function_probe();
1590         mutex_unlock(&ftrace_lock);
1591 }
1592
1593 void
1594 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1595                                 void *data)
1596 {
1597         __unregister_ftrace_function_probe(glob, ops, data,
1598                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
1599 }
1600
1601 void
1602 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1603 {
1604         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1605 }
1606
1607 void unregister_ftrace_function_probe_all(char *glob)
1608 {
1609         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1610 }
1611
1612 static LIST_HEAD(ftrace_commands);
1613 static DEFINE_MUTEX(ftrace_cmd_mutex);
1614
1615 int register_ftrace_command(struct ftrace_func_command *cmd)
1616 {
1617         struct ftrace_func_command *p;
1618         int ret = 0;
1619
1620         mutex_lock(&ftrace_cmd_mutex);
1621         list_for_each_entry(p, &ftrace_commands, list) {
1622                 if (strcmp(cmd->name, p->name) == 0) {
1623                         ret = -EBUSY;
1624                         goto out_unlock;
1625                 }
1626         }
1627         list_add(&cmd->list, &ftrace_commands);
1628  out_unlock:
1629         mutex_unlock(&ftrace_cmd_mutex);
1630
1631         return ret;
1632 }
1633
1634 int unregister_ftrace_command(struct ftrace_func_command *cmd)
1635 {
1636         struct ftrace_func_command *p, *n;
1637         int ret = -ENODEV;
1638
1639         mutex_lock(&ftrace_cmd_mutex);
1640         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1641                 if (strcmp(cmd->name, p->name) == 0) {
1642                         ret = 0;
1643                         list_del_init(&p->list);
1644                         goto out_unlock;
1645                 }
1646         }
1647  out_unlock:
1648         mutex_unlock(&ftrace_cmd_mutex);
1649
1650         return ret;
1651 }
1652
1653 static int ftrace_process_regex(char *buff, int len, int enable)
1654 {
1655         char *func, *command, *next = buff;
1656         struct ftrace_func_command *p;
1657         int ret = -EINVAL;
1658
1659         func = strsep(&next, ":");
1660
1661         if (!next) {
1662                 ftrace_match_records(func, len, enable);
1663                 return 0;
1664         }
1665
1666         /* command found */
1667
1668         command = strsep(&next, ":");
1669
1670         mutex_lock(&ftrace_cmd_mutex);
1671         list_for_each_entry(p, &ftrace_commands, list) {
1672                 if (strcmp(p->name, command) == 0) {
1673                         ret = p->func(func, command, next, enable);
1674                         goto out_unlock;
1675                 }
1676         }
1677  out_unlock:
1678         mutex_unlock(&ftrace_cmd_mutex);
1679
1680         return ret;
1681 }
1682
1683 static ssize_t
1684 ftrace_regex_write(struct file *file, const char __user *ubuf,
1685                    size_t cnt, loff_t *ppos, int enable)
1686 {
1687         struct ftrace_iterator *iter;
1688         char ch;
1689         size_t read = 0;
1690         ssize_t ret;
1691
1692         if (!cnt || cnt < 0)
1693                 return 0;
1694
1695         mutex_lock(&ftrace_regex_lock);
1696
1697         if (file->f_mode & FMODE_READ) {
1698                 struct seq_file *m = file->private_data;
1699                 iter = m->private;
1700         } else
1701                 iter = file->private_data;
1702
1703         if (!*ppos) {
1704                 iter->flags &= ~FTRACE_ITER_CONT;
1705                 iter->buffer_idx = 0;
1706         }
1707
1708         ret = get_user(ch, ubuf++);
1709         if (ret)
1710                 goto out;
1711         read++;
1712         cnt--;
1713
1714         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1715                 /* skip white space */
1716                 while (cnt && isspace(ch)) {
1717                         ret = get_user(ch, ubuf++);
1718                         if (ret)
1719                                 goto out;
1720                         read++;
1721                         cnt--;
1722                 }
1723
1724                 if (isspace(ch)) {
1725                         file->f_pos += read;
1726                         ret = read;
1727                         goto out;
1728                 }
1729
1730                 iter->buffer_idx = 0;
1731         }
1732
1733         while (cnt && !isspace(ch)) {
1734                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1735                         iter->buffer[iter->buffer_idx++] = ch;
1736                 else {
1737                         ret = -EINVAL;
1738                         goto out;
1739                 }
1740                 ret = get_user(ch, ubuf++);
1741                 if (ret)
1742                         goto out;
1743                 read++;
1744                 cnt--;
1745         }
1746
1747         if (isspace(ch)) {
1748                 iter->filtered++;
1749                 iter->buffer[iter->buffer_idx] = 0;
1750                 ret = ftrace_process_regex(iter->buffer,
1751                                            iter->buffer_idx, enable);
1752                 if (ret)
1753                         goto out;
1754                 iter->buffer_idx = 0;
1755         } else
1756                 iter->flags |= FTRACE_ITER_CONT;
1757
1758
1759         file->f_pos += read;
1760
1761         ret = read;
1762  out:
1763         mutex_unlock(&ftrace_regex_lock);
1764
1765         return ret;
1766 }
1767
1768 static ssize_t
1769 ftrace_filter_write(struct file *file, const char __user *ubuf,
1770                     size_t cnt, loff_t *ppos)
1771 {
1772         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1773 }
1774
1775 static ssize_t
1776 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1777                      size_t cnt, loff_t *ppos)
1778 {
1779         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1780 }
1781
1782 static void
1783 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1784 {
1785         if (unlikely(ftrace_disabled))
1786                 return;
1787
1788         mutex_lock(&ftrace_regex_lock);
1789         if (reset)
1790                 ftrace_filter_reset(enable);
1791         if (buf)
1792                 ftrace_match_records(buf, len, enable);
1793         mutex_unlock(&ftrace_regex_lock);
1794 }
1795
1796 /**
1797  * ftrace_set_filter - set a function to filter on in ftrace
1798  * @buf - the string that holds the function filter text.
1799  * @len - the length of the string.
1800  * @reset - non zero to reset all filters before applying this filter.
1801  *
1802  * Filters denote which functions should be enabled when tracing is enabled.
1803  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1804  */
1805 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1806 {
1807         ftrace_set_regex(buf, len, reset, 1);
1808 }
1809
1810 /**
1811  * ftrace_set_notrace - set a function to not trace in ftrace
1812  * @buf - the string that holds the function notrace text.
1813  * @len - the length of the string.
1814  * @reset - non zero to reset all filters before applying this filter.
1815  *
1816  * Notrace Filters denote which functions should not be enabled when tracing
1817  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1818  * for tracing.
1819  */
1820 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1821 {
1822         ftrace_set_regex(buf, len, reset, 0);
1823 }
1824
1825 static int
1826 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1827 {
1828         struct seq_file *m = (struct seq_file *)file->private_data;
1829         struct ftrace_iterator *iter;
1830
1831         mutex_lock(&ftrace_regex_lock);
1832         if (file->f_mode & FMODE_READ) {
1833                 iter = m->private;
1834
1835                 seq_release(inode, file);
1836         } else
1837                 iter = file->private_data;
1838
1839         if (iter->buffer_idx) {
1840                 iter->filtered++;
1841                 iter->buffer[iter->buffer_idx] = 0;
1842                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1843         }
1844
1845         mutex_lock(&ftrace_lock);
1846         if (ftrace_start_up && ftrace_enabled)
1847                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1848         mutex_unlock(&ftrace_lock);
1849
1850         kfree(iter);
1851         mutex_unlock(&ftrace_regex_lock);
1852         return 0;
1853 }
1854
1855 static int
1856 ftrace_filter_release(struct inode *inode, struct file *file)
1857 {
1858         return ftrace_regex_release(inode, file, 1);
1859 }
1860
1861 static int
1862 ftrace_notrace_release(struct inode *inode, struct file *file)
1863 {
1864         return ftrace_regex_release(inode, file, 0);
1865 }
1866
1867 static const struct file_operations ftrace_avail_fops = {
1868         .open = ftrace_avail_open,
1869         .read = seq_read,
1870         .llseek = seq_lseek,
1871         .release = ftrace_avail_release,
1872 };
1873
1874 static const struct file_operations ftrace_failures_fops = {
1875         .open = ftrace_failures_open,
1876         .read = seq_read,
1877         .llseek = seq_lseek,
1878         .release = ftrace_avail_release,
1879 };
1880
1881 static const struct file_operations ftrace_filter_fops = {
1882         .open = ftrace_filter_open,
1883         .read = ftrace_regex_read,
1884         .write = ftrace_filter_write,
1885         .llseek = ftrace_regex_lseek,
1886         .release = ftrace_filter_release,
1887 };
1888
1889 static const struct file_operations ftrace_notrace_fops = {
1890         .open = ftrace_notrace_open,
1891         .read = ftrace_regex_read,
1892         .write = ftrace_notrace_write,
1893         .llseek = ftrace_regex_lseek,
1894         .release = ftrace_notrace_release,
1895 };
1896
1897 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1898
1899 static DEFINE_MUTEX(graph_lock);
1900
1901 int ftrace_graph_count;
1902 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1903
1904 static void *
1905 g_next(struct seq_file *m, void *v, loff_t *pos)
1906 {
1907         unsigned long *array = m->private;
1908         int index = *pos;
1909
1910         (*pos)++;
1911
1912         if (index >= ftrace_graph_count)
1913                 return NULL;
1914
1915         return &array[index];
1916 }
1917
1918 static void *g_start(struct seq_file *m, loff_t *pos)
1919 {
1920         void *p = NULL;
1921
1922         mutex_lock(&graph_lock);
1923
1924         /* Nothing, tell g_show to print all functions are enabled */
1925         if (!ftrace_graph_count && !*pos)
1926                 return (void *)1;
1927
1928         p = g_next(m, p, pos);
1929
1930         return p;
1931 }
1932
1933 static void g_stop(struct seq_file *m, void *p)
1934 {
1935         mutex_unlock(&graph_lock);
1936 }
1937
1938 static int g_show(struct seq_file *m, void *v)
1939 {
1940         unsigned long *ptr = v;
1941         char str[KSYM_SYMBOL_LEN];
1942
1943         if (!ptr)
1944                 return 0;
1945
1946         if (ptr == (unsigned long *)1) {
1947                 seq_printf(m, "#### all functions enabled ####\n");
1948                 return 0;
1949         }
1950
1951         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1952
1953         seq_printf(m, "%s\n", str);
1954
1955         return 0;
1956 }
1957
1958 static struct seq_operations ftrace_graph_seq_ops = {
1959         .start = g_start,
1960         .next = g_next,
1961         .stop = g_stop,
1962         .show = g_show,
1963 };
1964
1965 static int
1966 ftrace_graph_open(struct inode *inode, struct file *file)
1967 {
1968         int ret = 0;
1969
1970         if (unlikely(ftrace_disabled))
1971                 return -ENODEV;
1972
1973         mutex_lock(&graph_lock);
1974         if ((file->f_mode & FMODE_WRITE) &&
1975             !(file->f_flags & O_APPEND)) {
1976                 ftrace_graph_count = 0;
1977                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1978         }
1979
1980         if (file->f_mode & FMODE_READ) {
1981                 ret = seq_open(file, &ftrace_graph_seq_ops);
1982                 if (!ret) {
1983                         struct seq_file *m = file->private_data;
1984                         m->private = ftrace_graph_funcs;
1985                 }
1986         } else
1987                 file->private_data = ftrace_graph_funcs;
1988         mutex_unlock(&graph_lock);
1989
1990         return ret;
1991 }
1992
1993 static ssize_t
1994 ftrace_graph_read(struct file *file, char __user *ubuf,
1995                        size_t cnt, loff_t *ppos)
1996 {
1997         if (file->f_mode & FMODE_READ)
1998                 return seq_read(file, ubuf, cnt, ppos);
1999         else
2000                 return -EPERM;
2001 }
2002
2003 static int
2004 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2005 {
2006         struct dyn_ftrace *rec;
2007         struct ftrace_page *pg;
2008         int search_len;
2009         int found = 0;
2010         int type, not;
2011         char *search;
2012         bool exists;
2013         int i;
2014
2015         if (ftrace_disabled)
2016                 return -ENODEV;
2017
2018         /* decode regex */
2019         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2020         if (not)
2021                 return -EINVAL;
2022
2023         search_len = strlen(search);
2024
2025         mutex_lock(&ftrace_lock);
2026         do_for_each_ftrace_rec(pg, rec) {
2027
2028                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2029                         break;
2030
2031                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2032                         continue;
2033
2034                 if (ftrace_match_record(rec, search, search_len, type)) {
2035                         /* ensure it is not already in the array */
2036                         exists = false;
2037                         for (i = 0; i < *idx; i++)
2038                                 if (array[i] == rec->ip) {
2039                                         exists = true;
2040                                         break;
2041                                 }
2042                         if (!exists) {
2043                                 array[(*idx)++] = rec->ip;
2044                                 found = 1;
2045                         }
2046                 }
2047         } while_for_each_ftrace_rec();
2048
2049         mutex_unlock(&ftrace_lock);
2050
2051         return found ? 0 : -EINVAL;
2052 }
2053
2054 static ssize_t
2055 ftrace_graph_write(struct file *file, const char __user *ubuf,
2056                    size_t cnt, loff_t *ppos)
2057 {
2058         unsigned char buffer[FTRACE_BUFF_MAX+1];
2059         unsigned long *array;
2060         size_t read = 0;
2061         ssize_t ret;
2062         int index = 0;
2063         char ch;
2064
2065         if (!cnt || cnt < 0)
2066                 return 0;
2067
2068         mutex_lock(&graph_lock);
2069
2070         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2071                 ret = -EBUSY;
2072                 goto out;
2073         }
2074
2075         if (file->f_mode & FMODE_READ) {
2076                 struct seq_file *m = file->private_data;
2077                 array = m->private;
2078         } else
2079                 array = file->private_data;
2080
2081         ret = get_user(ch, ubuf++);
2082         if (ret)
2083                 goto out;
2084         read++;
2085         cnt--;
2086
2087         /* skip white space */
2088         while (cnt && isspace(ch)) {
2089                 ret = get_user(ch, ubuf++);
2090                 if (ret)
2091                         goto out;
2092                 read++;
2093                 cnt--;
2094         }
2095
2096         if (isspace(ch)) {
2097                 *ppos += read;
2098                 ret = read;
2099                 goto out;
2100         }
2101
2102         while (cnt && !isspace(ch)) {
2103                 if (index < FTRACE_BUFF_MAX)
2104                         buffer[index++] = ch;
2105                 else {
2106                         ret = -EINVAL;
2107                         goto out;
2108                 }
2109                 ret = get_user(ch, ubuf++);
2110                 if (ret)
2111                         goto out;
2112                 read++;
2113                 cnt--;
2114         }
2115         buffer[index] = 0;
2116
2117         /* we allow only one expression at a time */
2118         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2119         if (ret)
2120                 goto out;
2121
2122         file->f_pos += read;
2123
2124         ret = read;
2125  out:
2126         mutex_unlock(&graph_lock);
2127
2128         return ret;
2129 }
2130
2131 static const struct file_operations ftrace_graph_fops = {
2132         .open = ftrace_graph_open,
2133         .read = ftrace_graph_read,
2134         .write = ftrace_graph_write,
2135 };
2136 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2137
2138 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2139 {
2140         struct dentry *entry;
2141
2142         entry = debugfs_create_file("available_filter_functions", 0444,
2143                                     d_tracer, NULL, &ftrace_avail_fops);
2144         if (!entry)
2145                 pr_warning("Could not create debugfs "
2146                            "'available_filter_functions' entry\n");
2147
2148         entry = debugfs_create_file("failures", 0444,
2149                                     d_tracer, NULL, &ftrace_failures_fops);
2150         if (!entry)
2151                 pr_warning("Could not create debugfs 'failures' entry\n");
2152
2153         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2154                                     NULL, &ftrace_filter_fops);
2155         if (!entry)
2156                 pr_warning("Could not create debugfs "
2157                            "'set_ftrace_filter' entry\n");
2158
2159         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2160                                     NULL, &ftrace_notrace_fops);
2161         if (!entry)
2162                 pr_warning("Could not create debugfs "
2163                            "'set_ftrace_notrace' entry\n");
2164
2165 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2166         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2167                                     NULL,
2168                                     &ftrace_graph_fops);
2169         if (!entry)
2170                 pr_warning("Could not create debugfs "
2171                            "'set_graph_function' entry\n");
2172 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2173
2174         return 0;
2175 }
2176
2177 static int ftrace_convert_nops(struct module *mod,
2178                                unsigned long *start,
2179                                unsigned long *end)
2180 {
2181         unsigned long *p;
2182         unsigned long addr;
2183         unsigned long flags;
2184
2185         mutex_lock(&ftrace_lock);
2186         p = start;
2187         while (p < end) {
2188                 addr = ftrace_call_adjust(*p++);
2189                 /*
2190                  * Some architecture linkers will pad between
2191                  * the different mcount_loc sections of different
2192                  * object files to satisfy alignments.
2193                  * Skip any NULL pointers.
2194                  */
2195                 if (!addr)
2196                         continue;
2197                 ftrace_record_ip(addr);
2198         }
2199
2200         /* disable interrupts to prevent kstop machine */
2201         local_irq_save(flags);
2202         ftrace_update_code(mod);
2203         local_irq_restore(flags);
2204         mutex_unlock(&ftrace_lock);
2205
2206         return 0;
2207 }
2208
2209 void ftrace_init_module(struct module *mod,
2210                         unsigned long *start, unsigned long *end)
2211 {
2212         if (ftrace_disabled || start == end)
2213                 return;
2214         ftrace_convert_nops(mod, start, end);
2215 }
2216
2217 extern unsigned long __start_mcount_loc[];
2218 extern unsigned long __stop_mcount_loc[];
2219
2220 void __init ftrace_init(void)
2221 {
2222         unsigned long count, addr, flags;
2223         int ret;
2224
2225         /* Keep the ftrace pointer to the stub */
2226         addr = (unsigned long)ftrace_stub;
2227
2228         local_irq_save(flags);
2229         ftrace_dyn_arch_init(&addr);
2230         local_irq_restore(flags);
2231
2232         /* ftrace_dyn_arch_init places the return code in addr */
2233         if (addr)
2234                 goto failed;
2235
2236         count = __stop_mcount_loc - __start_mcount_loc;
2237
2238         ret = ftrace_dyn_table_alloc(count);
2239         if (ret)
2240                 goto failed;
2241
2242         last_ftrace_enabled = ftrace_enabled = 1;
2243
2244         ret = ftrace_convert_nops(NULL,
2245                                   __start_mcount_loc,
2246                                   __stop_mcount_loc);
2247
2248         return;
2249  failed:
2250         ftrace_disabled = 1;
2251 }
2252
2253 #else
2254
2255 static int __init ftrace_nodyn_init(void)
2256 {
2257         ftrace_enabled = 1;
2258         return 0;
2259 }
2260 device_initcall(ftrace_nodyn_init);
2261
2262 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2263 static inline void ftrace_startup_enable(int command) { }
2264 /* Keep as macros so we do not need to define the commands */
2265 # define ftrace_startup(command)        do { } while (0)
2266 # define ftrace_shutdown(command)       do { } while (0)
2267 # define ftrace_startup_sysctl()        do { } while (0)
2268 # define ftrace_shutdown_sysctl()       do { } while (0)
2269 #endif /* CONFIG_DYNAMIC_FTRACE */
2270
2271 static ssize_t
2272 ftrace_pid_read(struct file *file, char __user *ubuf,
2273                        size_t cnt, loff_t *ppos)
2274 {
2275         char buf[64];
2276         int r;
2277
2278         if (ftrace_pid_trace == ftrace_swapper_pid)
2279                 r = sprintf(buf, "swapper tasks\n");
2280         else if (ftrace_pid_trace)
2281                 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
2282         else
2283                 r = sprintf(buf, "no pid\n");
2284
2285         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2286 }
2287
2288 static void clear_ftrace_swapper(void)
2289 {
2290         struct task_struct *p;
2291         int cpu;
2292
2293         get_online_cpus();
2294         for_each_online_cpu(cpu) {
2295                 p = idle_task(cpu);
2296                 clear_tsk_trace_trace(p);
2297         }
2298         put_online_cpus();
2299 }
2300
2301 static void set_ftrace_swapper(void)
2302 {
2303         struct task_struct *p;
2304         int cpu;
2305
2306         get_online_cpus();
2307         for_each_online_cpu(cpu) {
2308                 p = idle_task(cpu);
2309                 set_tsk_trace_trace(p);
2310         }
2311         put_online_cpus();
2312 }
2313
2314 static void clear_ftrace_pid(struct pid *pid)
2315 {
2316         struct task_struct *p;
2317
2318         rcu_read_lock();
2319         do_each_pid_task(pid, PIDTYPE_PID, p) {
2320                 clear_tsk_trace_trace(p);
2321         } while_each_pid_task(pid, PIDTYPE_PID, p);
2322         rcu_read_unlock();
2323
2324         put_pid(pid);
2325 }
2326
2327 static void set_ftrace_pid(struct pid *pid)
2328 {
2329         struct task_struct *p;
2330
2331         rcu_read_lock();
2332         do_each_pid_task(pid, PIDTYPE_PID, p) {
2333                 set_tsk_trace_trace(p);
2334         } while_each_pid_task(pid, PIDTYPE_PID, p);
2335         rcu_read_unlock();
2336 }
2337
2338 static void clear_ftrace_pid_task(struct pid **pid)
2339 {
2340         if (*pid == ftrace_swapper_pid)
2341                 clear_ftrace_swapper();
2342         else
2343                 clear_ftrace_pid(*pid);
2344
2345         *pid = NULL;
2346 }
2347
2348 static void set_ftrace_pid_task(struct pid *pid)
2349 {
2350         if (pid == ftrace_swapper_pid)
2351                 set_ftrace_swapper();
2352         else
2353                 set_ftrace_pid(pid);
2354 }
2355
2356 static ssize_t
2357 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2358                    size_t cnt, loff_t *ppos)
2359 {
2360         struct pid *pid;
2361         char buf[64];
2362         long val;
2363         int ret;
2364
2365         if (cnt >= sizeof(buf))
2366                 return -EINVAL;
2367
2368         if (copy_from_user(&buf, ubuf, cnt))
2369                 return -EFAULT;
2370
2371         buf[cnt] = 0;
2372
2373         ret = strict_strtol(buf, 10, &val);
2374         if (ret < 0)
2375                 return ret;
2376
2377         mutex_lock(&ftrace_lock);
2378         if (val < 0) {
2379                 /* disable pid tracing */
2380                 if (!ftrace_pid_trace)
2381                         goto out;
2382
2383                 clear_ftrace_pid_task(&ftrace_pid_trace);
2384
2385         } else {
2386                 /* swapper task is special */
2387                 if (!val) {
2388                         pid = ftrace_swapper_pid;
2389                         if (pid == ftrace_pid_trace)
2390                                 goto out;
2391                 } else {
2392                         pid = find_get_pid(val);
2393
2394                         if (pid == ftrace_pid_trace) {
2395                                 put_pid(pid);
2396                                 goto out;
2397                         }
2398                 }
2399
2400                 if (ftrace_pid_trace)
2401                         clear_ftrace_pid_task(&ftrace_pid_trace);
2402
2403                 if (!pid)
2404                         goto out;
2405
2406                 ftrace_pid_trace = pid;
2407
2408                 set_ftrace_pid_task(ftrace_pid_trace);
2409         }
2410
2411         /* update the function call */
2412         ftrace_update_pid_func();
2413         ftrace_startup_enable(0);
2414
2415  out:
2416         mutex_unlock(&ftrace_lock);
2417
2418         return cnt;
2419 }
2420
2421 static const struct file_operations ftrace_pid_fops = {
2422         .read = ftrace_pid_read,
2423         .write = ftrace_pid_write,
2424 };
2425
2426 static __init int ftrace_init_debugfs(void)
2427 {
2428         struct dentry *d_tracer;
2429         struct dentry *entry;
2430
2431         d_tracer = tracing_init_dentry();
2432         if (!d_tracer)
2433                 return 0;
2434
2435         ftrace_init_dyn_debugfs(d_tracer);
2436
2437         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2438                                     NULL, &ftrace_pid_fops);
2439         if (!entry)
2440                 pr_warning("Could not create debugfs "
2441                            "'set_ftrace_pid' entry\n");
2442         return 0;
2443 }
2444 fs_initcall(ftrace_init_debugfs);
2445
2446 /**
2447  * ftrace_kill - kill ftrace
2448  *
2449  * This function should be used by panic code. It stops ftrace
2450  * but in a not so nice way. If you need to simply kill ftrace
2451  * from a non-atomic section, use ftrace_kill.
2452  */
2453 void ftrace_kill(void)
2454 {
2455         ftrace_disabled = 1;
2456         ftrace_enabled = 0;
2457         clear_ftrace_function();
2458 }
2459
2460 /**
2461  * register_ftrace_function - register a function for profiling
2462  * @ops - ops structure that holds the function for profiling.
2463  *
2464  * Register a function to be called by all functions in the
2465  * kernel.
2466  *
2467  * Note: @ops->func and all the functions it calls must be labeled
2468  *       with "notrace", otherwise it will go into a
2469  *       recursive loop.
2470  */
2471 int register_ftrace_function(struct ftrace_ops *ops)
2472 {
2473         int ret;
2474
2475         if (unlikely(ftrace_disabled))
2476                 return -1;
2477
2478         mutex_lock(&ftrace_lock);
2479
2480         ret = __register_ftrace_function(ops);
2481         ftrace_startup(0);
2482
2483         mutex_unlock(&ftrace_lock);
2484         return ret;
2485 }
2486
2487 /**
2488  * unregister_ftrace_function - unregister a function for profiling.
2489  * @ops - ops structure that holds the function to unregister
2490  *
2491  * Unregister a function that was added to be called by ftrace profiling.
2492  */
2493 int unregister_ftrace_function(struct ftrace_ops *ops)
2494 {
2495         int ret;
2496
2497         mutex_lock(&ftrace_lock);
2498         ret = __unregister_ftrace_function(ops);
2499         ftrace_shutdown(0);
2500         mutex_unlock(&ftrace_lock);
2501
2502         return ret;
2503 }
2504
2505 int
2506 ftrace_enable_sysctl(struct ctl_table *table, int write,
2507                      struct file *file, void __user *buffer, size_t *lenp,
2508                      loff_t *ppos)
2509 {
2510         int ret;
2511
2512         if (unlikely(ftrace_disabled))
2513                 return -ENODEV;
2514
2515         mutex_lock(&ftrace_lock);
2516
2517         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
2518
2519         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
2520                 goto out;
2521
2522         last_ftrace_enabled = ftrace_enabled;
2523
2524         if (ftrace_enabled) {
2525
2526                 ftrace_startup_sysctl();
2527
2528                 /* we are starting ftrace again */
2529                 if (ftrace_list != &ftrace_list_end) {
2530                         if (ftrace_list->next == &ftrace_list_end)
2531                                 ftrace_trace_function = ftrace_list->func;
2532                         else
2533                                 ftrace_trace_function = ftrace_list_func;
2534                 }
2535
2536         } else {
2537                 /* stopping ftrace calls (just send to ftrace_stub) */
2538                 ftrace_trace_function = ftrace_stub;
2539
2540                 ftrace_shutdown_sysctl();
2541         }
2542
2543  out:
2544         mutex_unlock(&ftrace_lock);
2545         return ret;
2546 }
2547
2548 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2549
2550 static atomic_t ftrace_graph_active;
2551 static struct notifier_block ftrace_suspend_notifier;
2552
2553 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
2554 {
2555         return 0;
2556 }
2557
2558 /* The callbacks that hook a function */
2559 trace_func_graph_ret_t ftrace_graph_return =
2560                         (trace_func_graph_ret_t)ftrace_stub;
2561 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
2562
2563 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2564 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
2565 {
2566         int i;
2567         int ret = 0;
2568         unsigned long flags;
2569         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
2570         struct task_struct *g, *t;
2571
2572         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
2573                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
2574                                         * sizeof(struct ftrace_ret_stack),
2575                                         GFP_KERNEL);
2576                 if (!ret_stack_list[i]) {
2577                         start = 0;
2578                         end = i;
2579                         ret = -ENOMEM;
2580                         goto free;
2581                 }
2582         }
2583
2584         read_lock_irqsave(&tasklist_lock, flags);
2585         do_each_thread(g, t) {
2586                 if (start == end) {
2587                         ret = -EAGAIN;
2588                         goto unlock;
2589                 }
2590
2591                 if (t->ret_stack == NULL) {
2592                         t->curr_ret_stack = -1;
2593                         /* Make sure IRQs see the -1 first: */
2594                         barrier();
2595                         t->ret_stack = ret_stack_list[start++];
2596                         atomic_set(&t->tracing_graph_pause, 0);
2597                         atomic_set(&t->trace_overrun, 0);
2598                 }
2599         } while_each_thread(g, t);
2600
2601 unlock:
2602         read_unlock_irqrestore(&tasklist_lock, flags);
2603 free:
2604         for (i = start; i < end; i++)
2605                 kfree(ret_stack_list[i]);
2606         return ret;
2607 }
2608
2609 /* Allocate a return stack for each task */
2610 static int start_graph_tracing(void)
2611 {
2612         struct ftrace_ret_stack **ret_stack_list;
2613         int ret, cpu;
2614
2615         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2616                                 sizeof(struct ftrace_ret_stack *),
2617                                 GFP_KERNEL);
2618
2619         if (!ret_stack_list)
2620                 return -ENOMEM;
2621
2622         /* The cpu_boot init_task->ret_stack will never be freed */
2623         for_each_online_cpu(cpu)
2624                 ftrace_graph_init_task(idle_task(cpu));
2625
2626         do {
2627                 ret = alloc_retstack_tasklist(ret_stack_list);
2628         } while (ret == -EAGAIN);
2629
2630         kfree(ret_stack_list);
2631         return ret;
2632 }
2633
2634 /*
2635  * Hibernation protection.
2636  * The state of the current task is too much unstable during
2637  * suspend/restore to disk. We want to protect against that.
2638  */
2639 static int
2640 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2641                                                         void *unused)
2642 {
2643         switch (state) {
2644         case PM_HIBERNATION_PREPARE:
2645                 pause_graph_tracing();
2646                 break;
2647
2648         case PM_POST_HIBERNATION:
2649                 unpause_graph_tracing();
2650                 break;
2651         }
2652         return NOTIFY_DONE;
2653 }
2654
2655 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2656                         trace_func_graph_ent_t entryfunc)
2657 {
2658         int ret = 0;
2659
2660         mutex_lock(&ftrace_lock);
2661
2662         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2663         register_pm_notifier(&ftrace_suspend_notifier);
2664
2665         atomic_inc(&ftrace_graph_active);
2666         ret = start_graph_tracing();
2667         if (ret) {
2668                 atomic_dec(&ftrace_graph_active);
2669                 goto out;
2670         }
2671
2672         ftrace_graph_return = retfunc;
2673         ftrace_graph_entry = entryfunc;
2674
2675         ftrace_startup(FTRACE_START_FUNC_RET);
2676
2677 out:
2678         mutex_unlock(&ftrace_lock);
2679         return ret;
2680 }
2681
2682 void unregister_ftrace_graph(void)
2683 {
2684         mutex_lock(&ftrace_lock);
2685
2686         atomic_dec(&ftrace_graph_active);
2687         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2688         ftrace_graph_entry = ftrace_graph_entry_stub;
2689         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2690         unregister_pm_notifier(&ftrace_suspend_notifier);
2691
2692         mutex_unlock(&ftrace_lock);
2693 }
2694
2695 /* Allocate a return stack for newly created task */
2696 void ftrace_graph_init_task(struct task_struct *t)
2697 {
2698         if (atomic_read(&ftrace_graph_active)) {
2699                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2700                                 * sizeof(struct ftrace_ret_stack),
2701                                 GFP_KERNEL);
2702                 if (!t->ret_stack)
2703                         return;
2704                 t->curr_ret_stack = -1;
2705                 atomic_set(&t->tracing_graph_pause, 0);
2706                 atomic_set(&t->trace_overrun, 0);
2707         } else
2708                 t->ret_stack = NULL;
2709 }
2710
2711 void ftrace_graph_exit_task(struct task_struct *t)
2712 {
2713         struct ftrace_ret_stack *ret_stack = t->ret_stack;
2714
2715         t->ret_stack = NULL;
2716         /* NULL must become visible to IRQs before we free it: */
2717         barrier();
2718
2719         kfree(ret_stack);
2720 }
2721
2722 void ftrace_graph_stop(void)
2723 {
2724         ftrace_stop();
2725 }
2726 #endif
2727