git.oblomov.eu Git - linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/debugfs.h>
  21 #include <linux/hardirq.h>
  22 #include <linux/kthread.h>
  23 #include <linux/uaccess.h>
  24 #include <linux/kprobes.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/ctype.h>
  28 #include <linux/hash.h>
  29 #include <linux/list.h>
  30
  31 #include <asm/ftrace.h>
  32
  33 #include "trace.h"
  34
  35 /* ftrace_enabled is a method to turn ftrace on or off */
  36 int ftrace_enabled __read_mostly;
  37 static int last_ftrace_enabled;
  38
  39 /*
  40  * ftrace_disabled is set when an anomaly is discovered.
  41  * ftrace_disabled is much stronger than ftrace_enabled.
  42  */
  43 static int ftrace_disabled __read_mostly;
  44
  45 static DEFINE_SPINLOCK(ftrace_lock);
  46 static DEFINE_MUTEX(ftrace_sysctl_lock);
  47
  48 static struct ftrace_ops ftrace_list_end __read_mostly =
  49 {
  50         .func = ftrace_stub,
  51 };
  52
  53 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  54 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  55
  56 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  57 {
  58         struct ftrace_ops *op = ftrace_list;
  59
  60         /* in case someone actually ports this to alpha! */
  61         read_barrier_depends();
  62
  63         while (op != &ftrace_list_end) {
  64                 /* silly alpha */
  65                 read_barrier_depends();
  66                 op->func(ip, parent_ip);
  67                 op = op->next;
  68         };
  69 }
  70
  71 /**
  72  * clear_ftrace_function - reset the ftrace function
  73  *
  74  * This NULLs the ftrace function and in essence stops
  75  * tracing.  There may be lag
  76  */
  77 void clear_ftrace_function(void)
  78 {
  79         ftrace_trace_function = ftrace_stub;
  80 }
  81
  82 static int __register_ftrace_function(struct ftrace_ops *ops)
  83 {
  84         /* should not be called from interrupt context */
  85         spin_lock(&ftrace_lock);
  86
  87         ops->next = ftrace_list;
  88         /*
  89          * We are entering ops into the ftrace_list but another
  90          * CPU might be walking that list. We need to make sure
  91          * the ops->next pointer is valid before another CPU sees
  92          * the ops pointer included into the ftrace_list.
  93          */
  94         smp_wmb();
  95         ftrace_list = ops;
  96
  97         if (ftrace_enabled) {
  98                 /*
  99                  * For one func, simply call it directly.
 100                  * For more than one func, call the chain.
 101                  */
 102                 if (ops->next == &ftrace_list_end)
 103                         ftrace_trace_function = ops->func;
 104                 else
 105                         ftrace_trace_function = ftrace_list_func;
 106         }
 107
 108         spin_unlock(&ftrace_lock);
 109
 110         return 0;
 111 }
 112
 113 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 114 {
 115         struct ftrace_ops **p;
 116         int ret = 0;
 117
 118         /* should not be called from interrupt context */
 119         spin_lock(&ftrace_lock);
 120
 121         /*
 122          * If we are removing the last function, then simply point
 123          * to the ftrace_stub.
 124          */
 125         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 126                 ftrace_trace_function = ftrace_stub;
 127                 ftrace_list = &ftrace_list_end;
 128                 goto out;
 129         }
 130
 131         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 132                 if (*p == ops)
 133                         break;
 134
 135         if (*p != ops) {
 136                 ret = -1;
 137                 goto out;
 138         }
 139
 140         *p = (*p)->next;
 141
 142         if (ftrace_enabled) {
 143                 /* If we only have one func left, then call that directly */
 144                 if (ftrace_list == &ftrace_list_end ||
 145                     ftrace_list->next == &ftrace_list_end)
 146                         ftrace_trace_function = ftrace_list->func;
 147         }
 148
 149  out:
 150         spin_unlock(&ftrace_lock);
 151
 152         return ret;
 153 }
 154
 155 #ifdef CONFIG_DYNAMIC_FTRACE
 156
 157 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 158 /*
 159  * The hash lock is only needed when the recording of the mcount
 160  * callers are dynamic. That is, by the caller themselves and
 161  * not recorded via the compilation.
 162  */
 163 static DEFINE_SPINLOCK(ftrace_hash_lock);
 164 #define ftrace_hash_lock(flags)   spin_lock_irqsave(&ftrace_hash_lock, flags)
 165 #define ftrace_hash_unlock(flags) \
 166                         spin_unlock_irqrestore(&ftrace_hash_lock, flags)
 167 #else
 168 /* This is protected via the ftrace_lock with MCOUNT_RECORD. */
 169 #define ftrace_hash_lock(flags)   do { (void)(flags); } while (0)
 170 #define ftrace_hash_unlock(flags) do { } while(0)
 171 #endif
 172
 173 /*
 174  * Since MCOUNT_ADDR may point to mcount itself, we do not want
 175  * to get it confused by reading a reference in the code as we
 176  * are parsing on objcopy output of text. Use a variable for
 177  * it instead.
 178  */
 179 static unsigned long mcount_addr = MCOUNT_ADDR;
 180
 181 static struct task_struct *ftraced_task;
 182
 183 enum {
 184         FTRACE_ENABLE_CALLS             = (1 << 0),
 185         FTRACE_DISABLE_CALLS            = (1 << 1),
 186         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 187         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 188         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 189 };
 190
 191 static int ftrace_filtered;
 192 static int tracing_on;
 193 static int frozen_record_count;
 194
 195 static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
 196
 197 static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
 198
 199 static DEFINE_MUTEX(ftraced_lock);
 200 static DEFINE_MUTEX(ftrace_regex_lock);
 201
 202 struct ftrace_page {
 203         struct ftrace_page      *next;
 204         unsigned long           index;
 205         struct dyn_ftrace       records[];
 206 };
 207
 208 #define ENTRIES_PER_PAGE \
 209   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 210
 211 /* estimate from running different kernels */
 212 #define NR_TO_INIT              10000
 213
 214 static struct ftrace_page       *ftrace_pages_start;
 215 static struct ftrace_page       *ftrace_pages;
 216
 217 static int ftraced_trigger;
 218 static int ftraced_suspend;
 219 static int ftraced_stop;
 220
 221 static int ftrace_record_suspend;
 222
 223 static struct dyn_ftrace *ftrace_free_records;
 224
 225
 226 #ifdef CONFIG_KPROBES
 227 static inline void freeze_record(struct dyn_ftrace *rec)
 228 {
 229         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 230                 rec->flags |= FTRACE_FL_FROZEN;
 231                 frozen_record_count++;
 232         }
 233 }
 234
 235 static inline void unfreeze_record(struct dyn_ftrace *rec)
 236 {
 237         if (rec->flags & FTRACE_FL_FROZEN) {
 238                 rec->flags &= ~FTRACE_FL_FROZEN;
 239                 frozen_record_count--;
 240         }
 241 }
 242
 243 static inline int record_frozen(struct dyn_ftrace *rec)
 244 {
 245         return rec->flags & FTRACE_FL_FROZEN;
 246 }
 247 #else
 248 # define freeze_record(rec)                     ({ 0; })
 249 # define unfreeze_record(rec)                   ({ 0; })
 250 # define record_frozen(rec)                     ({ 0; })
 251 #endif /* CONFIG_KPROBES */
 252
 253 int skip_trace(unsigned long ip)
 254 {
 255         unsigned long fl;
 256         struct dyn_ftrace *rec;
 257         struct hlist_node *t;
 258         struct hlist_head *head;
 259
 260         if (frozen_record_count == 0)
 261                 return 0;
 262
 263         head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
 264         hlist_for_each_entry_rcu(rec, t, head, node) {
 265                 if (rec->ip == ip) {
 266                         if (record_frozen(rec)) {
 267                                 if (rec->flags & FTRACE_FL_FAILED)
 268                                         return 1;
 269
 270                                 if (!(rec->flags & FTRACE_FL_CONVERTED))
 271                                         return 1;
 272
 273                                 if (!tracing_on || !ftrace_enabled)
 274                                         return 1;
 275
 276                                 if (ftrace_filtered) {
 277                                         fl = rec->flags & (FTRACE_FL_FILTER |
 278                                                            FTRACE_FL_NOTRACE);
 279                                         if (!fl || (fl & FTRACE_FL_NOTRACE))
 280                                                 return 1;
 281                                 }
 282                         }
 283                         break;
 284                 }
 285         }
 286
 287         return 0;
 288 }
 289
 290 static inline int
 291 ftrace_ip_in_hash(unsigned long ip, unsigned long key)
 292 {
 293         struct dyn_ftrace *p;
 294         struct hlist_node *t;
 295         int found = 0;
 296
 297         hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
 298                 if (p->ip == ip) {
 299                         found = 1;
 300                         break;
 301                 }
 302         }
 303
 304         return found;
 305 }
 306
 307 static inline void
 308 ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
 309 {
 310         hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
 311 }
 312
 313 /* called from kstop_machine */
 314 static inline void ftrace_del_hash(struct dyn_ftrace *node)
 315 {
 316         hlist_del(&node->node);
 317 }
 318
 319 static void ftrace_free_rec(struct dyn_ftrace *rec)
 320 {
 321         rec->ip = (unsigned long)ftrace_free_records;
 322         ftrace_free_records = rec;
 323         rec->flags |= FTRACE_FL_FREE;
 324 }
 325
 326 void ftrace_release(void *start, unsigned long size)
 327 {
 328         struct dyn_ftrace *rec;
 329         struct ftrace_page *pg;
 330         unsigned long s = (unsigned long)start;
 331         unsigned long e = s + size;
 332         int i;
 333
 334         if (ftrace_disabled || !start)
 335                 return;
 336
 337         /* should not be called from interrupt context */
 338         spin_lock(&ftrace_lock);
 339
 340         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 341                 for (i = 0; i < pg->index; i++) {
 342                         rec = &pg->records[i];
 343
 344                         if ((rec->ip >= s) && (rec->ip < e))
 345                                 ftrace_free_rec(rec);
 346                 }
 347         }
 348         spin_unlock(&ftrace_lock);
 349
 350 }
 351
 352 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 353 {
 354         struct dyn_ftrace *rec;
 355
 356         /* First check for freed records */
 357         if (ftrace_free_records) {
 358                 rec = ftrace_free_records;
 359
 360                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 361                         WARN_ON_ONCE(1);
 362                         ftrace_free_records = NULL;
 363                         ftrace_disabled = 1;
 364                         ftrace_enabled = 0;
 365                         return NULL;
 366                 }
 367
 368                 ftrace_free_records = (void *)rec->ip;
 369                 memset(rec, 0, sizeof(*rec));
 370                 return rec;
 371         }
 372
 373         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 374                 if (!ftrace_pages->next)
 375                         return NULL;
 376                 ftrace_pages = ftrace_pages->next;
 377         }
 378
 379         return &ftrace_pages->records[ftrace_pages->index++];
 380 }
 381
 382 static void
 383 ftrace_record_ip(unsigned long ip)
 384 {
 385         struct dyn_ftrace *node;
 386         unsigned long flags;
 387         unsigned long key;
 388         int resched;
 389         int cpu;
 390
 391         if (!ftrace_enabled || ftrace_disabled)
 392                 return;
 393
 394         resched = need_resched();
 395         preempt_disable_notrace();
 396
 397         /*
 398          * We simply need to protect against recursion.
 399          * Use the the raw version of smp_processor_id and not
 400          * __get_cpu_var which can call debug hooks that can
 401          * cause a recursive crash here.
 402          */
 403         cpu = raw_smp_processor_id();
 404         per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
 405         if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
 406                 goto out;
 407
 408         if (unlikely(ftrace_record_suspend))
 409                 goto out;
 410
 411         key = hash_long(ip, FTRACE_HASHBITS);
 412
 413         WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
 414
 415         if (ftrace_ip_in_hash(ip, key))
 416                 goto out;
 417
 418         ftrace_hash_lock(flags);
 419
 420         /* This ip may have hit the hash before the lock */
 421         if (ftrace_ip_in_hash(ip, key))
 422                 goto out_unlock;
 423
 424         node = ftrace_alloc_dyn_node(ip);
 425         if (!node)
 426                 goto out_unlock;
 427
 428         node->ip = ip;
 429
 430         ftrace_add_hash(node, key);
 431
 432         ftraced_trigger = 1;
 433
 434  out_unlock:
 435         ftrace_hash_unlock(flags);
 436  out:
 437         per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
 438
 439         /* prevent recursion with scheduler */
 440         if (resched)
 441                 preempt_enable_no_resched_notrace();
 442         else
 443                 preempt_enable_notrace();
 444 }
 445
 446 #define FTRACE_ADDR ((long)(ftrace_caller))
 447
 448 static int
 449 __ftrace_replace_code(struct dyn_ftrace *rec,
 450                       unsigned char *old, unsigned char *new, int enable)
 451 {
 452         unsigned long ip, fl;
 453
 454         ip = rec->ip;
 455
 456         if (ftrace_filtered && enable) {
 457                 /*
 458                  * If filtering is on:
 459                  *
 460                  * If this record is set to be filtered and
 461                  * is enabled then do nothing.
 462                  *
 463                  * If this record is set to be filtered and
 464                  * it is not enabled, enable it.
 465                  *
 466                  * If this record is not set to be filtered
 467                  * and it is not enabled do nothing.
 468                  *
 469                  * If this record is set not to trace then
 470                  * do nothing.
 471                  *
 472                  * If this record is set not to trace and
 473                  * it is enabled then disable it.
 474                  *
 475                  * If this record is not set to be filtered and
 476                  * it is enabled, disable it.
 477                  */
 478
 479                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
 480                                    FTRACE_FL_ENABLED);
 481
 482                 if ((fl ==  (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
 483                     (fl ==  (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
 484                     !fl || (fl == FTRACE_FL_NOTRACE))
 485                         return 0;
 486
 487                 /*
 488                  * If it is enabled disable it,
 489                  * otherwise enable it!
 490                  */
 491                 if (fl & FTRACE_FL_ENABLED) {
 492                         /* swap new and old */
 493                         new = old;
 494                         old = ftrace_call_replace(ip, FTRACE_ADDR);
 495                         rec->flags &= ~FTRACE_FL_ENABLED;
 496                 } else {
 497                         new = ftrace_call_replace(ip, FTRACE_ADDR);
 498                         rec->flags |= FTRACE_FL_ENABLED;
 499                 }
 500         } else {
 501
 502                 if (enable) {
 503                         /*
 504                          * If this record is set not to trace and is
 505                          * not enabled, do nothing.
 506                          */
 507                         fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
 508                         if (fl == FTRACE_FL_NOTRACE)
 509                                 return 0;
 510
 511                         new = ftrace_call_replace(ip, FTRACE_ADDR);
 512                 } else
 513                         old = ftrace_call_replace(ip, FTRACE_ADDR);
 514
 515                 if (enable) {
 516                         if (rec->flags & FTRACE_FL_ENABLED)
 517                                 return 0;
 518                         rec->flags |= FTRACE_FL_ENABLED;
 519                 } else {
 520                         if (!(rec->flags & FTRACE_FL_ENABLED))
 521                                 return 0;
 522                         rec->flags &= ~FTRACE_FL_ENABLED;
 523                 }
 524         }
 525
 526         return ftrace_modify_code(ip, old, new);
 527 }
 528
 529 static void ftrace_replace_code(int enable)
 530 {
 531         int i, failed;
 532         unsigned char *new = NULL, *old = NULL;
 533         struct dyn_ftrace *rec;
 534         struct ftrace_page *pg;
 535
 536         if (enable)
 537                 old = ftrace_nop_replace();
 538         else
 539                 new = ftrace_nop_replace();
 540
 541         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 542                 for (i = 0; i < pg->index; i++) {
 543                         rec = &pg->records[i];
 544
 545                         /* don't modify code that has already faulted */
 546                         if (rec->flags & FTRACE_FL_FAILED)
 547                                 continue;
 548
 549                         /* ignore updates to this record's mcount site */
 550                         if (get_kprobe((void *)rec->ip)) {
 551                                 freeze_record(rec);
 552                                 continue;
 553                         } else {
 554                                 unfreeze_record(rec);
 555                         }
 556
 557                         failed = __ftrace_replace_code(rec, old, new, enable);
 558                         if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 559                                 rec->flags |= FTRACE_FL_FAILED;
 560                                 if ((system_state == SYSTEM_BOOTING) ||
 561                                     !core_kernel_text(rec->ip)) {
 562                                         ftrace_del_hash(rec);
 563                                         ftrace_free_rec(rec);
 564                                 }
 565                         }
 566                 }
 567         }
 568 }
 569
 570 static void ftrace_shutdown_replenish(void)
 571 {
 572         if (ftrace_pages->next)
 573                 return;
 574
 575         /* allocate another page */
 576         ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
 577 }
 578
 579 static void print_ip_ins(const char *fmt, unsigned char *p)
 580 {
 581         int i;
 582
 583         printk(KERN_CONT "%s", fmt);
 584
 585         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 586                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 587 }
 588
 589 static int
 590 ftrace_code_disable(struct dyn_ftrace *rec)
 591 {
 592         unsigned long ip;
 593         unsigned char *nop, *call;
 594         int failed;
 595
 596         ip = rec->ip;
 597
 598         nop = ftrace_nop_replace();
 599         call = ftrace_call_replace(ip, mcount_addr);
 600
 601         failed = ftrace_modify_code(ip, call, nop);
 602         if (failed) {
 603                 switch (failed) {
 604                 case 1:
 605                         WARN_ON_ONCE(1);
 606                         pr_info("ftrace faulted on modifying ");
 607                         print_ip_sym(ip);
 608                         break;
 609                 case 2:
 610                         WARN_ON_ONCE(1);
 611                         pr_info("ftrace failed to modify ");
 612                         print_ip_sym(ip);
 613                         print_ip_ins(" expected: ", call);
 614                         print_ip_ins(" actual: ", (unsigned char *)ip);
 615                         print_ip_ins(" replace: ", nop);
 616                         printk(KERN_CONT "\n");
 617                         break;
 618                 }
 619
 620                 rec->flags |= FTRACE_FL_FAILED;
 621                 return 0;
 622         }
 623         return 1;
 624 }
 625
 626 static int __ftrace_update_code(void *ignore);
 627
 628 static int __ftrace_modify_code(void *data)
 629 {
 630         unsigned long addr;
 631         int *command = data;
 632
 633         if (*command & FTRACE_ENABLE_CALLS) {
 634                 /*
 635                  * Update any recorded ips now that we have the
 636                  * machine stopped
 637                  */
 638                 __ftrace_update_code(NULL);
 639                 ftrace_replace_code(1);
 640                 tracing_on = 1;
 641         } else if (*command & FTRACE_DISABLE_CALLS) {
 642                 ftrace_replace_code(0);
 643                 tracing_on = 0;
 644         }
 645
 646         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 647                 ftrace_update_ftrace_func(ftrace_trace_function);
 648
 649         if (*command & FTRACE_ENABLE_MCOUNT) {
 650                 addr = (unsigned long)ftrace_record_ip;
 651                 ftrace_mcount_set(&addr);
 652         } else if (*command & FTRACE_DISABLE_MCOUNT) {
 653                 addr = (unsigned long)ftrace_stub;
 654                 ftrace_mcount_set(&addr);
 655         }
 656
 657         return 0;
 658 }
 659
 660 static void ftrace_run_update_code(int command)
 661 {
 662         stop_machine(__ftrace_modify_code, &command, NULL);
 663 }
 664
 665 void ftrace_disable_daemon(void)
 666 {
 667         /* Stop the daemon from calling kstop_machine */
 668         mutex_lock(&ftraced_lock);
 669         ftraced_stop = 1;
 670         mutex_unlock(&ftraced_lock);
 671
 672         ftrace_force_update();
 673 }
 674
 675 void ftrace_enable_daemon(void)
 676 {
 677         mutex_lock(&ftraced_lock);
 678         ftraced_stop = 0;
 679         mutex_unlock(&ftraced_lock);
 680
 681         ftrace_force_update();
 682 }
 683
 684 static ftrace_func_t saved_ftrace_func;
 685
 686 static void ftrace_startup(void)
 687 {
 688         int command = 0;
 689
 690         if (unlikely(ftrace_disabled))
 691                 return;
 692
 693         mutex_lock(&ftraced_lock);
 694         ftraced_suspend++;
 695         if (ftraced_suspend == 1)
 696                 command |= FTRACE_ENABLE_CALLS;
 697
 698         if (saved_ftrace_func != ftrace_trace_function) {
 699                 saved_ftrace_func = ftrace_trace_function;
 700                 command |= FTRACE_UPDATE_TRACE_FUNC;
 701         }
 702
 703         if (!command || !ftrace_enabled)
 704                 goto out;
 705
 706         ftrace_run_update_code(command);
 707  out:
 708         mutex_unlock(&ftraced_lock);
 709 }
 710
 711 static void ftrace_shutdown(void)
 712 {
 713         int command = 0;
 714
 715         if (unlikely(ftrace_disabled))
 716                 return;
 717
 718         mutex_lock(&ftraced_lock);
 719         ftraced_suspend--;
 720         if (!ftraced_suspend)
 721                 command |= FTRACE_DISABLE_CALLS;
 722
 723         if (saved_ftrace_func != ftrace_trace_function) {
 724                 saved_ftrace_func = ftrace_trace_function;
 725                 command |= FTRACE_UPDATE_TRACE_FUNC;
 726         }
 727
 728         if (!command || !ftrace_enabled)
 729                 goto out;
 730
 731         ftrace_run_update_code(command);
 732  out:
 733         mutex_unlock(&ftraced_lock);
 734 }
 735
 736 static void ftrace_startup_sysctl(void)
 737 {
 738         int command = FTRACE_ENABLE_MCOUNT;
 739
 740         if (unlikely(ftrace_disabled))
 741                 return;
 742
 743         mutex_lock(&ftraced_lock);
 744         /* Force update next time */
 745         saved_ftrace_func = NULL;
 746         /* ftraced_suspend is true if we want ftrace running */
 747         if (ftraced_suspend)
 748                 command |= FTRACE_ENABLE_CALLS;
 749
 750         ftrace_run_update_code(command);
 751         mutex_unlock(&ftraced_lock);
 752 }
 753
 754 static void ftrace_shutdown_sysctl(void)
 755 {
 756         int command = FTRACE_DISABLE_MCOUNT;
 757
 758         if (unlikely(ftrace_disabled))
 759                 return;
 760
 761         mutex_lock(&ftraced_lock);
 762         /* ftraced_suspend is true if ftrace is running */
 763         if (ftraced_suspend)
 764                 command |= FTRACE_DISABLE_CALLS;
 765
 766         ftrace_run_update_code(command);
 767         mutex_unlock(&ftraced_lock);
 768 }
 769
 770 static cycle_t          ftrace_update_time;
 771 static unsigned long    ftrace_update_cnt;
 772 unsigned long           ftrace_update_tot_cnt;
 773
 774 static int __ftrace_update_code(void *ignore)
 775 {
 776         int i, save_ftrace_enabled;
 777         cycle_t start, stop;
 778         struct dyn_ftrace *p;
 779         struct hlist_node *t, *n;
 780         struct hlist_head *head, temp_list;
 781
 782         /* Don't be recording funcs now */
 783         ftrace_record_suspend++;
 784         save_ftrace_enabled = ftrace_enabled;
 785         ftrace_enabled = 0;
 786
 787         start = ftrace_now(raw_smp_processor_id());
 788         ftrace_update_cnt = 0;
 789
 790         /* No locks needed, the machine is stopped! */
 791         for (i = 0; i < FTRACE_HASHSIZE; i++) {
 792                 INIT_HLIST_HEAD(&temp_list);
 793                 head = &ftrace_hash[i];
 794
 795                 /* all CPUS are stopped, we are safe to modify code */
 796                 hlist_for_each_entry_safe(p, t, n, head, node) {
 797                         /* Skip over failed records which have not been
 798                          * freed. */
 799                         if (p->flags & FTRACE_FL_FAILED)
 800                                 continue;
 801
 802                         /* Unconverted records are always at the head of the
 803                          * hash bucket. Once we encounter a converted record,
 804                          * simply skip over to the next bucket. Saves ftraced
 805                          * some processor cycles (ftrace does its bid for
 806                          * global warming :-p ). */
 807                         if (p->flags & (FTRACE_FL_CONVERTED))
 808                                 break;
 809
 810                         /* Ignore updates to this record's mcount site.
 811                          * Reintroduce this record at the head of this
 812                          * bucket to attempt to "convert" it again if
 813                          * the kprobe on it is unregistered before the
 814                          * next run. */
 815                         if (get_kprobe((void *)p->ip)) {
 816                                 ftrace_del_hash(p);
 817                                 INIT_HLIST_NODE(&p->node);
 818                                 hlist_add_head(&p->node, &temp_list);
 819                                 freeze_record(p);
 820                                 continue;
 821                         } else {
 822                                 unfreeze_record(p);
 823                         }
 824
 825                         /* convert record (i.e, patch mcount-call with NOP) */
 826                         if (ftrace_code_disable(p)) {
 827                                 p->flags |= FTRACE_FL_CONVERTED;
 828                                 ftrace_update_cnt++;
 829                         } else {
 830                                 if ((system_state == SYSTEM_BOOTING) ||
 831                                     !core_kernel_text(p->ip)) {
 832                                         ftrace_del_hash(p);
 833                                         ftrace_free_rec(p);
 834                                 }
 835                         }
 836                 }
 837
 838                 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
 839                         hlist_del(&p->node);
 840                         INIT_HLIST_NODE(&p->node);
 841                         hlist_add_head(&p->node, head);
 842                 }
 843         }
 844
 845         stop = ftrace_now(raw_smp_processor_id());
 846         ftrace_update_time = stop - start;
 847         ftrace_update_tot_cnt += ftrace_update_cnt;
 848         ftraced_trigger = 0;
 849
 850         ftrace_enabled = save_ftrace_enabled;
 851         ftrace_record_suspend--;
 852
 853         return 0;
 854 }
 855
 856 static int ftrace_update_code(void)
 857 {
 858         if (unlikely(ftrace_disabled) ||
 859             !ftrace_enabled || !ftraced_trigger)
 860                 return 0;
 861
 862         stop_machine(__ftrace_update_code, NULL, NULL);
 863
 864         return 1;
 865 }
 866
 867 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 868 {
 869         struct ftrace_page *pg;
 870         int cnt;
 871         int i;
 872
 873         /* allocate a few pages */
 874         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 875         if (!ftrace_pages_start)
 876                 return -1;
 877
 878         /*
 879          * Allocate a few more pages.
 880          *
 881          * TODO: have some parser search vmlinux before
 882          *   final linking to find all calls to ftrace.
 883          *   Then we can:
 884          *    a) know how many pages to allocate.
 885          *     and/or
 886          *    b) set up the table then.
 887          *
 888          *  The dynamic code is still necessary for
 889          *  modules.
 890          */
 891
 892         pg = ftrace_pages = ftrace_pages_start;
 893
 894         cnt = num_to_init / ENTRIES_PER_PAGE;
 895         pr_info("ftrace: allocating %ld hash entries in %d pages\n",
 896                 num_to_init, cnt);
 897
 898         for (i = 0; i < cnt; i++) {
 899                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 900
 901                 /* If we fail, we'll try later anyway */
 902                 if (!pg->next)
 903                         break;
 904
 905                 pg = pg->next;
 906         }
 907
 908         return 0;
 909 }
 910
 911 enum {
 912         FTRACE_ITER_FILTER      = (1 << 0),
 913         FTRACE_ITER_CONT        = (1 << 1),
 914         FTRACE_ITER_NOTRACE     = (1 << 2),
 915         FTRACE_ITER_FAILURES    = (1 << 3),
 916 };
 917
 918 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 919
 920 struct ftrace_iterator {
 921         loff_t                  pos;
 922         struct ftrace_page      *pg;
 923         unsigned                idx;
 924         unsigned                flags;
 925         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 926         unsigned                buffer_idx;
 927         unsigned                filtered;
 928 };
 929
 930 static void *
 931 t_next(struct seq_file *m, void *v, loff_t *pos)
 932 {
 933         struct ftrace_iterator *iter = m->private;
 934         struct dyn_ftrace *rec = NULL;
 935
 936         (*pos)++;
 937
 938         /* should not be called from interrupt context */
 939         spin_lock(&ftrace_lock);
 940  retry:
 941         if (iter->idx >= iter->pg->index) {
 942                 if (iter->pg->next) {
 943                         iter->pg = iter->pg->next;
 944                         iter->idx = 0;
 945                         goto retry;
 946                 }
 947         } else {
 948                 rec = &iter->pg->records[iter->idx++];
 949                 if ((rec->flags & FTRACE_FL_FREE) ||
 950
 951                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 952                      (rec->flags & FTRACE_FL_FAILED)) ||
 953
 954                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 955                      !(rec->flags & FTRACE_FL_FAILED)) ||
 956
 957                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 958                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 959                         rec = NULL;
 960                         goto retry;
 961                 }
 962         }
 963         spin_unlock(&ftrace_lock);
 964
 965         iter->pos = *pos;
 966
 967         return rec;
 968 }
 969
 970 static void *t_start(struct seq_file *m, loff_t *pos)
 971 {
 972         struct ftrace_iterator *iter = m->private;
 973         void *p = NULL;
 974         loff_t l = -1;
 975
 976         if (*pos != iter->pos) {
 977                 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
 978                         ;
 979         } else {
 980                 l = *pos;
 981                 p = t_next(m, p, &l);
 982         }
 983
 984         return p;
 985 }
 986
 987 static void t_stop(struct seq_file *m, void *p)
 988 {
 989 }
 990
 991 static int t_show(struct seq_file *m, void *v)
 992 {
 993         struct dyn_ftrace *rec = v;
 994         char str[KSYM_SYMBOL_LEN];
 995
 996         if (!rec)
 997                 return 0;
 998
 999         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1000
1001         seq_printf(m, "%s\n", str);
1002
1003         return 0;
1004 }
1005
1006 static struct seq_operations show_ftrace_seq_ops = {
1007         .start = t_start,
1008         .next = t_next,
1009         .stop = t_stop,
1010         .show = t_show,
1011 };
1012
1013 static int
1014 ftrace_avail_open(struct inode *inode, struct file *file)
1015 {
1016         struct ftrace_iterator *iter;
1017         int ret;
1018
1019         if (unlikely(ftrace_disabled))
1020                 return -ENODEV;
1021
1022         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1023         if (!iter)
1024                 return -ENOMEM;
1025
1026         iter->pg = ftrace_pages_start;
1027         iter->pos = -1;
1028
1029         ret = seq_open(file, &show_ftrace_seq_ops);
1030         if (!ret) {
1031                 struct seq_file *m = file->private_data;
1032
1033                 m->private = iter;
1034         } else {
1035                 kfree(iter);
1036         }
1037
1038         return ret;
1039 }
1040
1041 int ftrace_avail_release(struct inode *inode, struct file *file)
1042 {
1043         struct seq_file *m = (struct seq_file *)file->private_data;
1044         struct ftrace_iterator *iter = m->private;
1045
1046         seq_release(inode, file);
1047         kfree(iter);
1048
1049         return 0;
1050 }
1051
1052 static int
1053 ftrace_failures_open(struct inode *inode, struct file *file)
1054 {
1055         int ret;
1056         struct seq_file *m;
1057         struct ftrace_iterator *iter;
1058
1059         ret = ftrace_avail_open(inode, file);
1060         if (!ret) {
1061                 m = (struct seq_file *)file->private_data;
1062                 iter = (struct ftrace_iterator *)m->private;
1063                 iter->flags = FTRACE_ITER_FAILURES;
1064         }
1065
1066         return ret;
1067 }
1068
1069
1070 static void ftrace_filter_reset(int enable)
1071 {
1072         struct ftrace_page *pg;
1073         struct dyn_ftrace *rec;
1074         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1075         unsigned i;
1076
1077         /* should not be called from interrupt context */
1078         spin_lock(&ftrace_lock);
1079         if (enable)
1080                 ftrace_filtered = 0;
1081         pg = ftrace_pages_start;
1082         while (pg) {
1083                 for (i = 0; i < pg->index; i++) {
1084                         rec = &pg->records[i];
1085                         if (rec->flags & FTRACE_FL_FAILED)
1086                                 continue;
1087                         rec->flags &= ~type;
1088                 }
1089                 pg = pg->next;
1090         }
1091         spin_unlock(&ftrace_lock);
1092 }
1093
1094 static int
1095 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1096 {
1097         struct ftrace_iterator *iter;
1098         int ret = 0;
1099
1100         if (unlikely(ftrace_disabled))
1101                 return -ENODEV;
1102
1103         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1104         if (!iter)
1105                 return -ENOMEM;
1106
1107         mutex_lock(&ftrace_regex_lock);
1108         if ((file->f_mode & FMODE_WRITE) &&
1109             !(file->f_flags & O_APPEND))
1110                 ftrace_filter_reset(enable);
1111
1112         if (file->f_mode & FMODE_READ) {
1113                 iter->pg = ftrace_pages_start;
1114                 iter->pos = -1;
1115                 iter->flags = enable ? FTRACE_ITER_FILTER :
1116                         FTRACE_ITER_NOTRACE;
1117
1118                 ret = seq_open(file, &show_ftrace_seq_ops);
1119                 if (!ret) {
1120                         struct seq_file *m = file->private_data;
1121                         m->private = iter;
1122                 } else
1123                         kfree(iter);
1124         } else
1125                 file->private_data = iter;
1126         mutex_unlock(&ftrace_regex_lock);
1127
1128         return ret;
1129 }
1130
1131 static int
1132 ftrace_filter_open(struct inode *inode, struct file *file)
1133 {
1134         return ftrace_regex_open(inode, file, 1);
1135 }
1136
1137 static int
1138 ftrace_notrace_open(struct inode *inode, struct file *file)
1139 {
1140         return ftrace_regex_open(inode, file, 0);
1141 }
1142
1143 static ssize_t
1144 ftrace_regex_read(struct file *file, char __user *ubuf,
1145                        size_t cnt, loff_t *ppos)
1146 {
1147         if (file->f_mode & FMODE_READ)
1148                 return seq_read(file, ubuf, cnt, ppos);
1149         else
1150                 return -EPERM;
1151 }
1152
1153 static loff_t
1154 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1155 {
1156         loff_t ret;
1157
1158         if (file->f_mode & FMODE_READ)
1159                 ret = seq_lseek(file, offset, origin);
1160         else
1161                 file->f_pos = ret = 1;
1162
1163         return ret;
1164 }
1165
1166 enum {
1167         MATCH_FULL,
1168         MATCH_FRONT_ONLY,
1169         MATCH_MIDDLE_ONLY,
1170         MATCH_END_ONLY,
1171 };
1172
1173 static void
1174 ftrace_match(unsigned char *buff, int len, int enable)
1175 {
1176         char str[KSYM_SYMBOL_LEN];
1177         char *search = NULL;
1178         struct ftrace_page *pg;
1179         struct dyn_ftrace *rec;
1180         int type = MATCH_FULL;
1181         unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1182         unsigned i, match = 0, search_len = 0;
1183
1184         for (i = 0; i < len; i++) {
1185                 if (buff[i] == '*') {
1186                         if (!i) {
1187                                 search = buff + i + 1;
1188                                 type = MATCH_END_ONLY;
1189                                 search_len = len - (i + 1);
1190                         } else {
1191                                 if (type == MATCH_END_ONLY) {
1192                                         type = MATCH_MIDDLE_ONLY;
1193                                 } else {
1194                                         match = i;
1195                                         type = MATCH_FRONT_ONLY;
1196                                 }
1197                                 buff[i] = 0;
1198                                 break;
1199                         }
1200                 }
1201         }
1202
1203         /* should not be called from interrupt context */
1204         spin_lock(&ftrace_lock);
1205         if (enable)
1206                 ftrace_filtered = 1;
1207         pg = ftrace_pages_start;
1208         while (pg) {
1209                 for (i = 0; i < pg->index; i++) {
1210                         int matched = 0;
1211                         char *ptr;
1212
1213                         rec = &pg->records[i];
1214                         if (rec->flags & FTRACE_FL_FAILED)
1215                                 continue;
1216                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1217                         switch (type) {
1218                         case MATCH_FULL:
1219                                 if (strcmp(str, buff) == 0)
1220                                         matched = 1;
1221                                 break;
1222                         case MATCH_FRONT_ONLY:
1223                                 if (memcmp(str, buff, match) == 0)
1224                                         matched = 1;
1225                                 break;
1226                         case MATCH_MIDDLE_ONLY:
1227                                 if (strstr(str, search))
1228                                         matched = 1;
1229                                 break;
1230                         case MATCH_END_ONLY:
1231                                 ptr = strstr(str, search);
1232                                 if (ptr && (ptr[search_len] == 0))
1233                                         matched = 1;
1234                                 break;
1235                         }
1236                         if (matched)
1237                                 rec->flags |= flag;
1238                 }
1239                 pg = pg->next;
1240         }
1241         spin_unlock(&ftrace_lock);
1242 }
1243
1244 static ssize_t
1245 ftrace_regex_write(struct file *file, const char __user *ubuf,
1246                    size_t cnt, loff_t *ppos, int enable)
1247 {
1248         struct ftrace_iterator *iter;
1249         char ch;
1250         size_t read = 0;
1251         ssize_t ret;
1252
1253         if (!cnt || cnt < 0)
1254                 return 0;
1255
1256         mutex_lock(&ftrace_regex_lock);
1257
1258         if (file->f_mode & FMODE_READ) {
1259                 struct seq_file *m = file->private_data;
1260                 iter = m->private;
1261         } else
1262                 iter = file->private_data;
1263
1264         if (!*ppos) {
1265                 iter->flags &= ~FTRACE_ITER_CONT;
1266                 iter->buffer_idx = 0;
1267         }
1268
1269         ret = get_user(ch, ubuf++);
1270         if (ret)
1271                 goto out;
1272         read++;
1273         cnt--;
1274
1275         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1276                 /* skip white space */
1277                 while (cnt && isspace(ch)) {
1278                         ret = get_user(ch, ubuf++);
1279                         if (ret)
1280                                 goto out;
1281                         read++;
1282                         cnt--;
1283                 }
1284
1285                 if (isspace(ch)) {
1286                         file->f_pos += read;
1287                         ret = read;
1288                         goto out;
1289                 }
1290
1291                 iter->buffer_idx = 0;
1292         }
1293
1294         while (cnt && !isspace(ch)) {
1295                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1296                         iter->buffer[iter->buffer_idx++] = ch;
1297                 else {
1298                         ret = -EINVAL;
1299                         goto out;
1300                 }
1301                 ret = get_user(ch, ubuf++);
1302                 if (ret)
1303                         goto out;
1304                 read++;
1305                 cnt--;
1306         }
1307
1308         if (isspace(ch)) {
1309                 iter->filtered++;
1310                 iter->buffer[iter->buffer_idx] = 0;
1311                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1312                 iter->buffer_idx = 0;
1313         } else
1314                 iter->flags |= FTRACE_ITER_CONT;
1315
1316
1317         file->f_pos += read;
1318
1319         ret = read;
1320  out:
1321         mutex_unlock(&ftrace_regex_lock);
1322
1323         return ret;
1324 }
1325
1326 static ssize_t
1327 ftrace_filter_write(struct file *file, const char __user *ubuf,
1328                     size_t cnt, loff_t *ppos)
1329 {
1330         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1331 }
1332
1333 static ssize_t
1334 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1335                      size_t cnt, loff_t *ppos)
1336 {
1337         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1338 }
1339
1340 static void
1341 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1342 {
1343         if (unlikely(ftrace_disabled))
1344                 return;
1345
1346         mutex_lock(&ftrace_regex_lock);
1347         if (reset)
1348                 ftrace_filter_reset(enable);
1349         if (buf)
1350                 ftrace_match(buf, len, enable);
1351         mutex_unlock(&ftrace_regex_lock);
1352 }
1353
1354 /**
1355  * ftrace_set_filter - set a function to filter on in ftrace
1356  * @buf - the string that holds the function filter text.
1357  * @len - the length of the string.
1358  * @reset - non zero to reset all filters before applying this filter.
1359  *
1360  * Filters denote which functions should be enabled when tracing is enabled.
1361  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1362  */
1363 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1364 {
1365         ftrace_set_regex(buf, len, reset, 1);
1366 }
1367
1368 /**
1369  * ftrace_set_notrace - set a function to not trace in ftrace
1370  * @buf - the string that holds the function notrace text.
1371  * @len - the length of the string.
1372  * @reset - non zero to reset all filters before applying this filter.
1373  *
1374  * Notrace Filters denote which functions should not be enabled when tracing
1375  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1376  * for tracing.
1377  */
1378 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1379 {
1380         ftrace_set_regex(buf, len, reset, 0);
1381 }
1382
1383 static int
1384 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1385 {
1386         struct seq_file *m = (struct seq_file *)file->private_data;
1387         struct ftrace_iterator *iter;
1388
1389         mutex_lock(&ftrace_regex_lock);
1390         if (file->f_mode & FMODE_READ) {
1391                 iter = m->private;
1392
1393                 seq_release(inode, file);
1394         } else
1395                 iter = file->private_data;
1396
1397         if (iter->buffer_idx) {
1398                 iter->filtered++;
1399                 iter->buffer[iter->buffer_idx] = 0;
1400                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1401         }
1402
1403         mutex_lock(&ftrace_sysctl_lock);
1404         mutex_lock(&ftraced_lock);
1405         if (iter->filtered && ftraced_suspend && ftrace_enabled)
1406                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1407         mutex_unlock(&ftraced_lock);
1408         mutex_unlock(&ftrace_sysctl_lock);
1409
1410         kfree(iter);
1411         mutex_unlock(&ftrace_regex_lock);
1412         return 0;
1413 }
1414
1415 static int
1416 ftrace_filter_release(struct inode *inode, struct file *file)
1417 {
1418         return ftrace_regex_release(inode, file, 1);
1419 }
1420
1421 static int
1422 ftrace_notrace_release(struct inode *inode, struct file *file)
1423 {
1424         return ftrace_regex_release(inode, file, 0);
1425 }
1426
1427 static ssize_t
1428 ftraced_read(struct file *filp, char __user *ubuf,
1429                      size_t cnt, loff_t *ppos)
1430 {
1431         /* don't worry about races */
1432         char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1433         int r = strlen(buf);
1434
1435         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1436 }
1437
1438 static ssize_t
1439 ftraced_write(struct file *filp, const char __user *ubuf,
1440                       size_t cnt, loff_t *ppos)
1441 {
1442         char buf[64];
1443         long val;
1444         int ret;
1445
1446         if (cnt >= sizeof(buf))
1447                 return -EINVAL;
1448
1449         if (copy_from_user(&buf, ubuf, cnt))
1450                 return -EFAULT;
1451
1452         if (strncmp(buf, "enable", 6) == 0)
1453                 val = 1;
1454         else if (strncmp(buf, "disable", 7) == 0)
1455                 val = 0;
1456         else {
1457                 buf[cnt] = 0;
1458
1459                 ret = strict_strtoul(buf, 10, &val);
1460                 if (ret < 0)
1461                         return ret;
1462
1463                 val = !!val;
1464         }
1465
1466         if (val)
1467                 ftrace_enable_daemon();
1468         else
1469                 ftrace_disable_daemon();
1470
1471         filp->f_pos += cnt;
1472
1473         return cnt;
1474 }
1475
1476 static struct file_operations ftrace_avail_fops = {
1477         .open = ftrace_avail_open,
1478         .read = seq_read,
1479         .llseek = seq_lseek,
1480         .release = ftrace_avail_release,
1481 };
1482
1483 static struct file_operations ftrace_failures_fops = {
1484         .open = ftrace_failures_open,
1485         .read = seq_read,
1486         .llseek = seq_lseek,
1487         .release = ftrace_avail_release,
1488 };
1489
1490 static struct file_operations ftrace_filter_fops = {
1491         .open = ftrace_filter_open,
1492         .read = ftrace_regex_read,
1493         .write = ftrace_filter_write,
1494         .llseek = ftrace_regex_lseek,
1495         .release = ftrace_filter_release,
1496 };
1497
1498 static struct file_operations ftrace_notrace_fops = {
1499         .open = ftrace_notrace_open,
1500         .read = ftrace_regex_read,
1501         .write = ftrace_notrace_write,
1502         .llseek = ftrace_regex_lseek,
1503         .release = ftrace_notrace_release,
1504 };
1505
1506 static struct file_operations ftraced_fops = {
1507         .open = tracing_open_generic,
1508         .read = ftraced_read,
1509         .write = ftraced_write,
1510 };
1511
1512 /**
1513  * ftrace_force_update - force an update to all recording ftrace functions
1514  */
1515 int ftrace_force_update(void)
1516 {
1517         int ret = 0;
1518
1519         if (unlikely(ftrace_disabled))
1520                 return -ENODEV;
1521
1522         mutex_lock(&ftrace_sysctl_lock);
1523         mutex_lock(&ftraced_lock);
1524
1525         /*
1526          * If ftraced_trigger is not set, then there is nothing
1527          * to update.
1528          */
1529         if (ftraced_trigger && !ftrace_update_code())
1530                 ret = -EBUSY;
1531
1532         mutex_unlock(&ftraced_lock);
1533         mutex_unlock(&ftrace_sysctl_lock);
1534
1535         return ret;
1536 }
1537
1538 static void ftrace_force_shutdown(void)
1539 {
1540         struct task_struct *task;
1541         int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1542
1543         mutex_lock(&ftraced_lock);
1544         task = ftraced_task;
1545         ftraced_task = NULL;
1546         ftraced_suspend = -1;
1547         ftrace_run_update_code(command);
1548         mutex_unlock(&ftraced_lock);
1549
1550         if (task)
1551                 kthread_stop(task);
1552 }
1553
1554 static __init int ftrace_init_debugfs(void)
1555 {
1556         struct dentry *d_tracer;
1557         struct dentry *entry;
1558
1559         d_tracer = tracing_init_dentry();
1560
1561         entry = debugfs_create_file("available_filter_functions", 0444,
1562                                     d_tracer, NULL, &ftrace_avail_fops);
1563         if (!entry)
1564                 pr_warning("Could not create debugfs "
1565                            "'available_filter_functions' entry\n");
1566
1567         entry = debugfs_create_file("failures", 0444,
1568                                     d_tracer, NULL, &ftrace_failures_fops);
1569         if (!entry)
1570                 pr_warning("Could not create debugfs 'failures' entry\n");
1571
1572         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1573                                     NULL, &ftrace_filter_fops);
1574         if (!entry)
1575                 pr_warning("Could not create debugfs "
1576                            "'set_ftrace_filter' entry\n");
1577
1578         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1579                                     NULL, &ftrace_notrace_fops);
1580         if (!entry)
1581                 pr_warning("Could not create debugfs "
1582                            "'set_ftrace_notrace' entry\n");
1583
1584         entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1585                                     NULL, &ftraced_fops);
1586         if (!entry)
1587                 pr_warning("Could not create debugfs "
1588                            "'ftraced_enabled' entry\n");
1589         return 0;
1590 }
1591
1592 fs_initcall(ftrace_init_debugfs);
1593
1594 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
1595 static int ftrace_convert_nops(unsigned long *start,
1596                                unsigned long *end)
1597 {
1598         unsigned long *p;
1599         unsigned long addr;
1600         unsigned long flags;
1601
1602         p = start;
1603         while (p < end) {
1604                 addr = ftrace_call_adjust(*p++);
1605                 /* should not be called from interrupt context */
1606                 spin_lock(&ftrace_lock);
1607                 ftrace_record_ip(addr);
1608                 spin_unlock(&ftrace_lock);
1609                 ftrace_shutdown_replenish();
1610         }
1611
1612         /* p is ignored */
1613         local_irq_save(flags);
1614         __ftrace_update_code(p);
1615         local_irq_restore(flags);
1616
1617         return 0;
1618 }
1619
1620 void ftrace_init_module(unsigned long *start, unsigned long *end)
1621 {
1622         if (ftrace_disabled || start == end)
1623                 return;
1624         ftrace_convert_nops(start, end);
1625 }
1626
1627 extern unsigned long __start_mcount_loc[];
1628 extern unsigned long __stop_mcount_loc[];
1629
1630 void __init ftrace_init(void)
1631 {
1632         unsigned long count, addr, flags;
1633         int ret;
1634
1635         /* Keep the ftrace pointer to the stub */
1636         addr = (unsigned long)ftrace_stub;
1637
1638         local_irq_save(flags);
1639         ftrace_dyn_arch_init(&addr);
1640         local_irq_restore(flags);
1641
1642         /* ftrace_dyn_arch_init places the return code in addr */
1643         if (addr)
1644                 goto failed;
1645
1646         count = __stop_mcount_loc - __start_mcount_loc;
1647
1648         ret = ftrace_dyn_table_alloc(count);
1649         if (ret)
1650                 goto failed;
1651
1652         last_ftrace_enabled = ftrace_enabled = 1;
1653
1654         ret = ftrace_convert_nops(__start_mcount_loc,
1655                                   __stop_mcount_loc);
1656
1657         return;
1658  failed:
1659         ftrace_disabled = 1;
1660 }
1661 #else /* CONFIG_FTRACE_MCOUNT_RECORD */
1662 static int ftraced(void *ignore)
1663 {
1664         unsigned long usecs;
1665
1666         while (!kthread_should_stop()) {
1667
1668                 set_current_state(TASK_INTERRUPTIBLE);
1669
1670                 /* check once a second */
1671                 schedule_timeout(HZ);
1672
1673                 if (unlikely(ftrace_disabled))
1674                         continue;
1675
1676                 mutex_lock(&ftrace_sysctl_lock);
1677                 mutex_lock(&ftraced_lock);
1678                 if (!ftraced_suspend && !ftraced_stop &&
1679                     ftrace_update_code()) {
1680                         usecs = nsecs_to_usecs(ftrace_update_time);
1681                         if (ftrace_update_tot_cnt > 100000) {
1682                                 ftrace_update_tot_cnt = 0;
1683                                 pr_info("hm, dftrace overflow: %lu change%s"
1684                                         " (%lu total) in %lu usec%s\n",
1685                                         ftrace_update_cnt,
1686                                         ftrace_update_cnt != 1 ? "s" : "",
1687                                         ftrace_update_tot_cnt,
1688                                         usecs, usecs != 1 ? "s" : "");
1689                                 ftrace_disabled = 1;
1690                                 WARN_ON_ONCE(1);
1691                         }
1692                 }
1693                 mutex_unlock(&ftraced_lock);
1694                 mutex_unlock(&ftrace_sysctl_lock);
1695
1696                 ftrace_shutdown_replenish();
1697         }
1698         __set_current_state(TASK_RUNNING);
1699         return 0;
1700 }
1701
1702 static int __init ftrace_dynamic_init(void)
1703 {
1704         struct task_struct *p;
1705         unsigned long addr;
1706         int ret;
1707
1708         addr = (unsigned long)ftrace_record_ip;
1709
1710         stop_machine(ftrace_dyn_arch_init, &addr, NULL);
1711
1712         /* ftrace_dyn_arch_init places the return code in addr */
1713         if (addr) {
1714                 ret = (int)addr;
1715                 goto failed;
1716         }
1717
1718         ret = ftrace_dyn_table_alloc(NR_TO_INIT);
1719         if (ret)
1720                 goto failed;
1721
1722         p = kthread_run(ftraced, NULL, "ftraced");
1723         if (IS_ERR(p)) {
1724                 ret = -1;
1725                 goto failed;
1726         }
1727
1728         last_ftrace_enabled = ftrace_enabled = 1;
1729         ftraced_task = p;
1730
1731         return 0;
1732
1733  failed:
1734         ftrace_disabled = 1;
1735         return ret;
1736 }
1737
1738 core_initcall(ftrace_dynamic_init);
1739 #endif /* CONFIG_FTRACE_MCOUNT_RECORD */
1740
1741 #else
1742 # define ftrace_startup()               do { } while (0)
1743 # define ftrace_shutdown()              do { } while (0)
1744 # define ftrace_startup_sysctl()        do { } while (0)
1745 # define ftrace_shutdown_sysctl()       do { } while (0)
1746 # define ftrace_force_shutdown()        do { } while (0)
1747 #endif /* CONFIG_DYNAMIC_FTRACE */
1748
1749 /**
1750  * ftrace_kill_atomic - kill ftrace from critical sections
1751  *
1752  * This function should be used by panic code. It stops ftrace
1753  * but in a not so nice way. If you need to simply kill ftrace
1754  * from a non-atomic section, use ftrace_kill.
1755  */
1756 void ftrace_kill_atomic(void)
1757 {
1758         ftrace_disabled = 1;
1759         ftrace_enabled = 0;
1760 #ifdef CONFIG_DYNAMIC_FTRACE
1761         ftraced_suspend = -1;
1762 #endif
1763         clear_ftrace_function();
1764 }
1765
1766 /**
1767  * ftrace_kill - totally shutdown ftrace
1768  *
1769  * This is a safety measure. If something was detected that seems
1770  * wrong, calling this function will keep ftrace from doing
1771  * any more modifications, and updates.
1772  * used when something went wrong.
1773  */
1774 void ftrace_kill(void)
1775 {
1776         mutex_lock(&ftrace_sysctl_lock);
1777         ftrace_disabled = 1;
1778         ftrace_enabled = 0;
1779
1780         clear_ftrace_function();
1781         mutex_unlock(&ftrace_sysctl_lock);
1782
1783         /* Try to totally disable ftrace */
1784         ftrace_force_shutdown();
1785 }
1786
1787 /**
1788  * register_ftrace_function - register a function for profiling
1789  * @ops - ops structure that holds the function for profiling.
1790  *
1791  * Register a function to be called by all functions in the
1792  * kernel.
1793  *
1794  * Note: @ops->func and all the functions it calls must be labeled
1795  *       with "notrace", otherwise it will go into a
1796  *       recursive loop.
1797  */
1798 int register_ftrace_function(struct ftrace_ops *ops)
1799 {
1800         int ret;
1801
1802         if (unlikely(ftrace_disabled))
1803                 return -1;
1804
1805         mutex_lock(&ftrace_sysctl_lock);
1806         ret = __register_ftrace_function(ops);
1807         ftrace_startup();
1808         mutex_unlock(&ftrace_sysctl_lock);
1809
1810         return ret;
1811 }
1812
1813 /**
1814  * unregister_ftrace_function - unresgister a function for profiling.
1815  * @ops - ops structure that holds the function to unregister
1816  *
1817  * Unregister a function that was added to be called by ftrace profiling.
1818  */
1819 int unregister_ftrace_function(struct ftrace_ops *ops)
1820 {
1821         int ret;
1822
1823         mutex_lock(&ftrace_sysctl_lock);
1824         ret = __unregister_ftrace_function(ops);
1825         ftrace_shutdown();
1826         mutex_unlock(&ftrace_sysctl_lock);
1827
1828         return ret;
1829 }
1830
1831 int
1832 ftrace_enable_sysctl(struct ctl_table *table, int write,
1833                      struct file *file, void __user *buffer, size_t *lenp,
1834                      loff_t *ppos)
1835 {
1836         int ret;
1837
1838         if (unlikely(ftrace_disabled))
1839                 return -ENODEV;
1840
1841         mutex_lock(&ftrace_sysctl_lock);
1842
1843         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
1844
1845         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1846                 goto out;
1847
1848         last_ftrace_enabled = ftrace_enabled;
1849
1850         if (ftrace_enabled) {
1851
1852                 ftrace_startup_sysctl();
1853
1854                 /* we are starting ftrace again */
1855                 if (ftrace_list != &ftrace_list_end) {
1856                         if (ftrace_list->next == &ftrace_list_end)
1857                                 ftrace_trace_function = ftrace_list->func;
1858                         else
1859                                 ftrace_trace_function = ftrace_list_func;
1860                 }
1861
1862         } else {
1863                 /* stopping ftrace calls (just send to ftrace_stub) */
1864                 ftrace_trace_function = ftrace_stub;
1865
1866                 ftrace_shutdown_sysctl();
1867         }
1868
1869  out:
1870         mutex_unlock(&ftrace_sysctl_lock);
1871         return ret;
1872 }