Merge branch 'tracing-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6] / kernel / trace / ftrace.c
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally ported from the -rt patch by:
8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 William Lee Irwin III
14  */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
31
32 #include <trace/sched.h>
33
34 #include <asm/ftrace.h>
35
36 #include "trace.h"
37
38 #define FTRACE_WARN_ON(cond)                    \
39         do {                                    \
40                 if (WARN_ON(cond))              \
41                         ftrace_kill();          \
42         } while (0)
43
44 #define FTRACE_WARN_ON_ONCE(cond)               \
45         do {                                    \
46                 if (WARN_ON_ONCE(cond))         \
47                         ftrace_kill();          \
48         } while (0)
49
50 /* hash bits for specific function selection */
51 #define FTRACE_HASH_BITS 7
52 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
53
54 /* ftrace_enabled is a method to turn ftrace on or off */
55 int ftrace_enabled __read_mostly;
56 static int last_ftrace_enabled;
57
58 /* Quick disabling of function tracer. */
59 int function_trace_stop;
60
61 /*
62  * ftrace_disabled is set when an anomaly is discovered.
63  * ftrace_disabled is much stronger than ftrace_enabled.
64  */
65 static int ftrace_disabled __read_mostly;
66
67 static DEFINE_MUTEX(ftrace_lock);
68
69 static struct ftrace_ops ftrace_list_end __read_mostly =
70 {
71         .func = ftrace_stub,
72 };
73
74 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
75 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
76 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
77 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
78
79 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
80 {
81         struct ftrace_ops *op = ftrace_list;
82
83         /* in case someone actually ports this to alpha! */
84         read_barrier_depends();
85
86         while (op != &ftrace_list_end) {
87                 /* silly alpha */
88                 read_barrier_depends();
89                 op->func(ip, parent_ip);
90                 op = op->next;
91         };
92 }
93
94 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
95 {
96         if (!test_tsk_trace_trace(current))
97                 return;
98
99         ftrace_pid_function(ip, parent_ip);
100 }
101
102 static void set_ftrace_pid_function(ftrace_func_t func)
103 {
104         /* do not set ftrace_pid_function to itself! */
105         if (func != ftrace_pid_func)
106                 ftrace_pid_function = func;
107 }
108
109 /**
110  * clear_ftrace_function - reset the ftrace function
111  *
112  * This NULLs the ftrace function and in essence stops
113  * tracing.  There may be lag
114  */
115 void clear_ftrace_function(void)
116 {
117         ftrace_trace_function = ftrace_stub;
118         __ftrace_trace_function = ftrace_stub;
119         ftrace_pid_function = ftrace_stub;
120 }
121
122 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
123 /*
124  * For those archs that do not test ftrace_trace_stop in their
125  * mcount call site, we need to do it from C.
126  */
127 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
128 {
129         if (function_trace_stop)
130                 return;
131
132         __ftrace_trace_function(ip, parent_ip);
133 }
134 #endif
135
136 static int __register_ftrace_function(struct ftrace_ops *ops)
137 {
138         ops->next = ftrace_list;
139         /*
140          * We are entering ops into the ftrace_list but another
141          * CPU might be walking that list. We need to make sure
142          * the ops->next pointer is valid before another CPU sees
143          * the ops pointer included into the ftrace_list.
144          */
145         smp_wmb();
146         ftrace_list = ops;
147
148         if (ftrace_enabled) {
149                 ftrace_func_t func;
150
151                 if (ops->next == &ftrace_list_end)
152                         func = ops->func;
153                 else
154                         func = ftrace_list_func;
155
156                 if (ftrace_pid_trace) {
157                         set_ftrace_pid_function(func);
158                         func = ftrace_pid_func;
159                 }
160
161                 /*
162                  * For one func, simply call it directly.
163                  * For more than one func, call the chain.
164                  */
165 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
166                 ftrace_trace_function = func;
167 #else
168                 __ftrace_trace_function = func;
169                 ftrace_trace_function = ftrace_test_stop_func;
170 #endif
171         }
172
173         return 0;
174 }
175
176 static int __unregister_ftrace_function(struct ftrace_ops *ops)
177 {
178         struct ftrace_ops **p;
179
180         /*
181          * If we are removing the last function, then simply point
182          * to the ftrace_stub.
183          */
184         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
185                 ftrace_trace_function = ftrace_stub;
186                 ftrace_list = &ftrace_list_end;
187                 return 0;
188         }
189
190         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
191                 if (*p == ops)
192                         break;
193
194         if (*p != ops)
195                 return -1;
196
197         *p = (*p)->next;
198
199         if (ftrace_enabled) {
200                 /* If we only have one func left, then call that directly */
201                 if (ftrace_list->next == &ftrace_list_end) {
202                         ftrace_func_t func = ftrace_list->func;
203
204                         if (ftrace_pid_trace) {
205                                 set_ftrace_pid_function(func);
206                                 func = ftrace_pid_func;
207                         }
208 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
209                         ftrace_trace_function = func;
210 #else
211                         __ftrace_trace_function = func;
212 #endif
213                 }
214         }
215
216         return 0;
217 }
218
219 static void ftrace_update_pid_func(void)
220 {
221         ftrace_func_t func;
222
223         if (ftrace_trace_function == ftrace_stub)
224                 return;
225
226         func = ftrace_trace_function;
227
228         if (ftrace_pid_trace) {
229                 set_ftrace_pid_function(func);
230                 func = ftrace_pid_func;
231         } else {
232                 if (func == ftrace_pid_func)
233                         func = ftrace_pid_function;
234         }
235
236 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
237         ftrace_trace_function = func;
238 #else
239         __ftrace_trace_function = func;
240 #endif
241 }
242
243 /* set when tracing only a pid */
244 struct pid *ftrace_pid_trace;
245 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
246
247 #ifdef CONFIG_DYNAMIC_FTRACE
248
249 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
250 # error Dynamic ftrace depends on MCOUNT_RECORD
251 #endif
252
253 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
254
255 struct ftrace_func_probe {
256         struct hlist_node       node;
257         struct ftrace_probe_ops *ops;
258         unsigned long           flags;
259         unsigned long           ip;
260         void                    *data;
261         struct rcu_head         rcu;
262 };
263
264
265 enum {
266         FTRACE_ENABLE_CALLS             = (1 << 0),
267         FTRACE_DISABLE_CALLS            = (1 << 1),
268         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
269         FTRACE_ENABLE_MCOUNT            = (1 << 3),
270         FTRACE_DISABLE_MCOUNT           = (1 << 4),
271         FTRACE_START_FUNC_RET           = (1 << 5),
272         FTRACE_STOP_FUNC_RET            = (1 << 6),
273 };
274
275 static int ftrace_filtered;
276
277 static struct dyn_ftrace *ftrace_new_addrs;
278
279 static DEFINE_MUTEX(ftrace_regex_lock);
280
281 struct ftrace_page {
282         struct ftrace_page      *next;
283         int                     index;
284         struct dyn_ftrace       records[];
285 };
286
287 #define ENTRIES_PER_PAGE \
288   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
289
290 /* estimate from running different kernels */
291 #define NR_TO_INIT              10000
292
293 static struct ftrace_page       *ftrace_pages_start;
294 static struct ftrace_page       *ftrace_pages;
295
296 static struct dyn_ftrace *ftrace_free_records;
297
298 /*
299  * This is a double for. Do not use 'break' to break out of the loop,
300  * you must use a goto.
301  */
302 #define do_for_each_ftrace_rec(pg, rec)                                 \
303         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
304                 int _____i;                                             \
305                 for (_____i = 0; _____i < pg->index; _____i++) {        \
306                         rec = &pg->records[_____i];
307
308 #define while_for_each_ftrace_rec()             \
309                 }                               \
310         }
311
312 #ifdef CONFIG_KPROBES
313
314 static int frozen_record_count;
315
316 static inline void freeze_record(struct dyn_ftrace *rec)
317 {
318         if (!(rec->flags & FTRACE_FL_FROZEN)) {
319                 rec->flags |= FTRACE_FL_FROZEN;
320                 frozen_record_count++;
321         }
322 }
323
324 static inline void unfreeze_record(struct dyn_ftrace *rec)
325 {
326         if (rec->flags & FTRACE_FL_FROZEN) {
327                 rec->flags &= ~FTRACE_FL_FROZEN;
328                 frozen_record_count--;
329         }
330 }
331
332 static inline int record_frozen(struct dyn_ftrace *rec)
333 {
334         return rec->flags & FTRACE_FL_FROZEN;
335 }
336 #else
337 # define freeze_record(rec)                     ({ 0; })
338 # define unfreeze_record(rec)                   ({ 0; })
339 # define record_frozen(rec)                     ({ 0; })
340 #endif /* CONFIG_KPROBES */
341
342 static void ftrace_free_rec(struct dyn_ftrace *rec)
343 {
344         rec->freelist = ftrace_free_records;
345         ftrace_free_records = rec;
346         rec->flags |= FTRACE_FL_FREE;
347 }
348
349 void ftrace_release(void *start, unsigned long size)
350 {
351         struct dyn_ftrace *rec;
352         struct ftrace_page *pg;
353         unsigned long s = (unsigned long)start;
354         unsigned long e = s + size;
355
356         if (ftrace_disabled || !start)
357                 return;
358
359         mutex_lock(&ftrace_lock);
360         do_for_each_ftrace_rec(pg, rec) {
361                 if ((rec->ip >= s) && (rec->ip < e)) {
362                         /*
363                          * rec->ip is changed in ftrace_free_rec()
364                          * It should not between s and e if record was freed.
365                          */
366                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
367                         ftrace_free_rec(rec);
368                 }
369         } while_for_each_ftrace_rec();
370         mutex_unlock(&ftrace_lock);
371 }
372
373 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
374 {
375         struct dyn_ftrace *rec;
376
377         /* First check for freed records */
378         if (ftrace_free_records) {
379                 rec = ftrace_free_records;
380
381                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
382                         FTRACE_WARN_ON_ONCE(1);
383                         ftrace_free_records = NULL;
384                         return NULL;
385                 }
386
387                 ftrace_free_records = rec->freelist;
388                 memset(rec, 0, sizeof(*rec));
389                 return rec;
390         }
391
392         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
393                 if (!ftrace_pages->next) {
394                         /* allocate another page */
395                         ftrace_pages->next =
396                                 (void *)get_zeroed_page(GFP_KERNEL);
397                         if (!ftrace_pages->next)
398                                 return NULL;
399                 }
400                 ftrace_pages = ftrace_pages->next;
401         }
402
403         return &ftrace_pages->records[ftrace_pages->index++];
404 }
405
406 static struct dyn_ftrace *
407 ftrace_record_ip(unsigned long ip)
408 {
409         struct dyn_ftrace *rec;
410
411         if (ftrace_disabled)
412                 return NULL;
413
414         rec = ftrace_alloc_dyn_node(ip);
415         if (!rec)
416                 return NULL;
417
418         rec->ip = ip;
419         rec->newlist = ftrace_new_addrs;
420         ftrace_new_addrs = rec;
421
422         return rec;
423 }
424
425 static void print_ip_ins(const char *fmt, unsigned char *p)
426 {
427         int i;
428
429         printk(KERN_CONT "%s", fmt);
430
431         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
432                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
433 }
434
435 static void ftrace_bug(int failed, unsigned long ip)
436 {
437         switch (failed) {
438         case -EFAULT:
439                 FTRACE_WARN_ON_ONCE(1);
440                 pr_info("ftrace faulted on modifying ");
441                 print_ip_sym(ip);
442                 break;
443         case -EINVAL:
444                 FTRACE_WARN_ON_ONCE(1);
445                 pr_info("ftrace failed to modify ");
446                 print_ip_sym(ip);
447                 print_ip_ins(" actual: ", (unsigned char *)ip);
448                 printk(KERN_CONT "\n");
449                 break;
450         case -EPERM:
451                 FTRACE_WARN_ON_ONCE(1);
452                 pr_info("ftrace faulted on writing ");
453                 print_ip_sym(ip);
454                 break;
455         default:
456                 FTRACE_WARN_ON_ONCE(1);
457                 pr_info("ftrace faulted on unknown error ");
458                 print_ip_sym(ip);
459         }
460 }
461
462
463 static int
464 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
465 {
466         unsigned long ftrace_addr;
467         unsigned long ip, fl;
468
469         ftrace_addr = (unsigned long)FTRACE_ADDR;
470
471         ip = rec->ip;
472
473         /*
474          * If this record is not to be traced and
475          * it is not enabled then do nothing.
476          *
477          * If this record is not to be traced and
478          * it is enabled then disable it.
479          *
480          */
481         if (rec->flags & FTRACE_FL_NOTRACE) {
482                 if (rec->flags & FTRACE_FL_ENABLED)
483                         rec->flags &= ~FTRACE_FL_ENABLED;
484                 else
485                         return 0;
486
487         } else if (ftrace_filtered && enable) {
488                 /*
489                  * Filtering is on:
490                  */
491
492                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
493
494                 /* Record is filtered and enabled, do nothing */
495                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
496                         return 0;
497
498                 /* Record is not filtered or enabled, do nothing */
499                 if (!fl)
500                         return 0;
501
502                 /* Record is not filtered but enabled, disable it */
503                 if (fl == FTRACE_FL_ENABLED)
504                         rec->flags &= ~FTRACE_FL_ENABLED;
505                 else
506                 /* Otherwise record is filtered but not enabled, enable it */
507                         rec->flags |= FTRACE_FL_ENABLED;
508         } else {
509                 /* Disable or not filtered */
510
511                 if (enable) {
512                         /* if record is enabled, do nothing */
513                         if (rec->flags & FTRACE_FL_ENABLED)
514                                 return 0;
515
516                         rec->flags |= FTRACE_FL_ENABLED;
517
518                 } else {
519
520                         /* if record is not enabled, do nothing */
521                         if (!(rec->flags & FTRACE_FL_ENABLED))
522                                 return 0;
523
524                         rec->flags &= ~FTRACE_FL_ENABLED;
525                 }
526         }
527
528         if (rec->flags & FTRACE_FL_ENABLED)
529                 return ftrace_make_call(rec, ftrace_addr);
530         else
531                 return ftrace_make_nop(NULL, rec, ftrace_addr);
532 }
533
534 static void ftrace_replace_code(int enable)
535 {
536         struct dyn_ftrace *rec;
537         struct ftrace_page *pg;
538         int failed;
539
540         do_for_each_ftrace_rec(pg, rec) {
541                 /*
542                  * Skip over free records, records that have
543                  * failed and not converted.
544                  */
545                 if (rec->flags & FTRACE_FL_FREE ||
546                     rec->flags & FTRACE_FL_FAILED ||
547                     !(rec->flags & FTRACE_FL_CONVERTED))
548                         continue;
549
550                 /* ignore updates to this record's mcount site */
551                 if (get_kprobe((void *)rec->ip)) {
552                         freeze_record(rec);
553                         continue;
554                 } else {
555                         unfreeze_record(rec);
556                 }
557
558                 failed = __ftrace_replace_code(rec, enable);
559                 if (failed) {
560                         rec->flags |= FTRACE_FL_FAILED;
561                         if ((system_state == SYSTEM_BOOTING) ||
562                             !core_kernel_text(rec->ip)) {
563                                 ftrace_free_rec(rec);
564                                 } else {
565                                 ftrace_bug(failed, rec->ip);
566                                         /* Stop processing */
567                                         return;
568                                 }
569                 }
570         } while_for_each_ftrace_rec();
571 }
572
573 static int
574 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
575 {
576         unsigned long ip;
577         int ret;
578
579         ip = rec->ip;
580
581         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
582         if (ret) {
583                 ftrace_bug(ret, ip);
584                 rec->flags |= FTRACE_FL_FAILED;
585                 return 0;
586         }
587         return 1;
588 }
589
590 /*
591  * archs can override this function if they must do something
592  * before the modifying code is performed.
593  */
594 int __weak ftrace_arch_code_modify_prepare(void)
595 {
596         return 0;
597 }
598
599 /*
600  * archs can override this function if they must do something
601  * after the modifying code is performed.
602  */
603 int __weak ftrace_arch_code_modify_post_process(void)
604 {
605         return 0;
606 }
607
608 static int __ftrace_modify_code(void *data)
609 {
610         int *command = data;
611
612         if (*command & FTRACE_ENABLE_CALLS)
613                 ftrace_replace_code(1);
614         else if (*command & FTRACE_DISABLE_CALLS)
615                 ftrace_replace_code(0);
616
617         if (*command & FTRACE_UPDATE_TRACE_FUNC)
618                 ftrace_update_ftrace_func(ftrace_trace_function);
619
620         if (*command & FTRACE_START_FUNC_RET)
621                 ftrace_enable_ftrace_graph_caller();
622         else if (*command & FTRACE_STOP_FUNC_RET)
623                 ftrace_disable_ftrace_graph_caller();
624
625         return 0;
626 }
627
628 static void ftrace_run_update_code(int command)
629 {
630         int ret;
631
632         ret = ftrace_arch_code_modify_prepare();
633         FTRACE_WARN_ON(ret);
634         if (ret)
635                 return;
636
637         stop_machine(__ftrace_modify_code, &command, NULL);
638
639         ret = ftrace_arch_code_modify_post_process();
640         FTRACE_WARN_ON(ret);
641 }
642
643 static ftrace_func_t saved_ftrace_func;
644 static int ftrace_start_up;
645
646 static void ftrace_startup_enable(int command)
647 {
648         if (saved_ftrace_func != ftrace_trace_function) {
649                 saved_ftrace_func = ftrace_trace_function;
650                 command |= FTRACE_UPDATE_TRACE_FUNC;
651         }
652
653         if (!command || !ftrace_enabled)
654                 return;
655
656         ftrace_run_update_code(command);
657 }
658
659 static void ftrace_startup(int command)
660 {
661         if (unlikely(ftrace_disabled))
662                 return;
663
664         ftrace_start_up++;
665         command |= FTRACE_ENABLE_CALLS;
666
667         ftrace_startup_enable(command);
668 }
669
670 static void ftrace_shutdown(int command)
671 {
672         if (unlikely(ftrace_disabled))
673                 return;
674
675         ftrace_start_up--;
676         if (!ftrace_start_up)
677                 command |= FTRACE_DISABLE_CALLS;
678
679         if (saved_ftrace_func != ftrace_trace_function) {
680                 saved_ftrace_func = ftrace_trace_function;
681                 command |= FTRACE_UPDATE_TRACE_FUNC;
682         }
683
684         if (!command || !ftrace_enabled)
685                 return;
686
687         ftrace_run_update_code(command);
688 }
689
690 static void ftrace_startup_sysctl(void)
691 {
692         int command = FTRACE_ENABLE_MCOUNT;
693
694         if (unlikely(ftrace_disabled))
695                 return;
696
697         /* Force update next time */
698         saved_ftrace_func = NULL;
699         /* ftrace_start_up is true if we want ftrace running */
700         if (ftrace_start_up)
701                 command |= FTRACE_ENABLE_CALLS;
702
703         ftrace_run_update_code(command);
704 }
705
706 static void ftrace_shutdown_sysctl(void)
707 {
708         int command = FTRACE_DISABLE_MCOUNT;
709
710         if (unlikely(ftrace_disabled))
711                 return;
712
713         /* ftrace_start_up is true if ftrace is running */
714         if (ftrace_start_up)
715                 command |= FTRACE_DISABLE_CALLS;
716
717         ftrace_run_update_code(command);
718 }
719
720 static cycle_t          ftrace_update_time;
721 static unsigned long    ftrace_update_cnt;
722 unsigned long           ftrace_update_tot_cnt;
723
724 static int ftrace_update_code(struct module *mod)
725 {
726         struct dyn_ftrace *p;
727         cycle_t start, stop;
728
729         start = ftrace_now(raw_smp_processor_id());
730         ftrace_update_cnt = 0;
731
732         while (ftrace_new_addrs) {
733
734                 /* If something went wrong, bail without enabling anything */
735                 if (unlikely(ftrace_disabled))
736                         return -1;
737
738                 p = ftrace_new_addrs;
739                 ftrace_new_addrs = p->newlist;
740                 p->flags = 0L;
741
742                 /* convert record (i.e, patch mcount-call with NOP) */
743                 if (ftrace_code_disable(mod, p)) {
744                         p->flags |= FTRACE_FL_CONVERTED;
745                         ftrace_update_cnt++;
746                 } else
747                         ftrace_free_rec(p);
748         }
749
750         stop = ftrace_now(raw_smp_processor_id());
751         ftrace_update_time = stop - start;
752         ftrace_update_tot_cnt += ftrace_update_cnt;
753
754         return 0;
755 }
756
757 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
758 {
759         struct ftrace_page *pg;
760         int cnt;
761         int i;
762
763         /* allocate a few pages */
764         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
765         if (!ftrace_pages_start)
766                 return -1;
767
768         /*
769          * Allocate a few more pages.
770          *
771          * TODO: have some parser search vmlinux before
772          *   final linking to find all calls to ftrace.
773          *   Then we can:
774          *    a) know how many pages to allocate.
775          *     and/or
776          *    b) set up the table then.
777          *
778          *  The dynamic code is still necessary for
779          *  modules.
780          */
781
782         pg = ftrace_pages = ftrace_pages_start;
783
784         cnt = num_to_init / ENTRIES_PER_PAGE;
785         pr_info("ftrace: allocating %ld entries in %d pages\n",
786                 num_to_init, cnt + 1);
787
788         for (i = 0; i < cnt; i++) {
789                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
790
791                 /* If we fail, we'll try later anyway */
792                 if (!pg->next)
793                         break;
794
795                 pg = pg->next;
796         }
797
798         return 0;
799 }
800
801 enum {
802         FTRACE_ITER_FILTER      = (1 << 0),
803         FTRACE_ITER_CONT        = (1 << 1),
804         FTRACE_ITER_NOTRACE     = (1 << 2),
805         FTRACE_ITER_FAILURES    = (1 << 3),
806         FTRACE_ITER_PRINTALL    = (1 << 4),
807         FTRACE_ITER_HASH        = (1 << 5),
808 };
809
810 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
811
812 struct ftrace_iterator {
813         struct ftrace_page      *pg;
814         int                     hidx;
815         int                     idx;
816         unsigned                flags;
817         unsigned char           buffer[FTRACE_BUFF_MAX+1];
818         unsigned                buffer_idx;
819         unsigned                filtered;
820 };
821
822 static void *
823 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
824 {
825         struct ftrace_iterator *iter = m->private;
826         struct hlist_node *hnd = v;
827         struct hlist_head *hhd;
828
829         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
830
831         (*pos)++;
832
833  retry:
834         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
835                 return NULL;
836
837         hhd = &ftrace_func_hash[iter->hidx];
838
839         if (hlist_empty(hhd)) {
840                 iter->hidx++;
841                 hnd = NULL;
842                 goto retry;
843         }
844
845         if (!hnd)
846                 hnd = hhd->first;
847         else {
848                 hnd = hnd->next;
849                 if (!hnd) {
850                         iter->hidx++;
851                         goto retry;
852                 }
853         }
854
855         return hnd;
856 }
857
858 static void *t_hash_start(struct seq_file *m, loff_t *pos)
859 {
860         struct ftrace_iterator *iter = m->private;
861         void *p = NULL;
862
863         iter->flags |= FTRACE_ITER_HASH;
864
865         return t_hash_next(m, p, pos);
866 }
867
868 static int t_hash_show(struct seq_file *m, void *v)
869 {
870         struct ftrace_func_probe *rec;
871         struct hlist_node *hnd = v;
872         char str[KSYM_SYMBOL_LEN];
873
874         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
875
876         if (rec->ops->print)
877                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
878
879         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
880         seq_printf(m, "%s:", str);
881
882         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
883         seq_printf(m, "%s", str);
884
885         if (rec->data)
886                 seq_printf(m, ":%p", rec->data);
887         seq_putc(m, '\n');
888
889         return 0;
890 }
891
892 static void *
893 t_next(struct seq_file *m, void *v, loff_t *pos)
894 {
895         struct ftrace_iterator *iter = m->private;
896         struct dyn_ftrace *rec = NULL;
897
898         if (iter->flags & FTRACE_ITER_HASH)
899                 return t_hash_next(m, v, pos);
900
901         (*pos)++;
902
903         if (iter->flags & FTRACE_ITER_PRINTALL)
904                 return NULL;
905
906  retry:
907         if (iter->idx >= iter->pg->index) {
908                 if (iter->pg->next) {
909                         iter->pg = iter->pg->next;
910                         iter->idx = 0;
911                         goto retry;
912                 } else {
913                         iter->idx = -1;
914                 }
915         } else {
916                 rec = &iter->pg->records[iter->idx++];
917                 if ((rec->flags & FTRACE_FL_FREE) ||
918
919                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
920                      (rec->flags & FTRACE_FL_FAILED)) ||
921
922                     ((iter->flags & FTRACE_ITER_FAILURES) &&
923                      !(rec->flags & FTRACE_FL_FAILED)) ||
924
925                     ((iter->flags & FTRACE_ITER_FILTER) &&
926                      !(rec->flags & FTRACE_FL_FILTER)) ||
927
928                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
929                      !(rec->flags & FTRACE_FL_NOTRACE))) {
930                         rec = NULL;
931                         goto retry;
932                 }
933         }
934
935         return rec;
936 }
937
938 static void *t_start(struct seq_file *m, loff_t *pos)
939 {
940         struct ftrace_iterator *iter = m->private;
941         void *p = NULL;
942
943         mutex_lock(&ftrace_lock);
944         /*
945          * For set_ftrace_filter reading, if we have the filter
946          * off, we can short cut and just print out that all
947          * functions are enabled.
948          */
949         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
950                 if (*pos > 0)
951                         return t_hash_start(m, pos);
952                 iter->flags |= FTRACE_ITER_PRINTALL;
953                 (*pos)++;
954                 return iter;
955         }
956
957         if (iter->flags & FTRACE_ITER_HASH)
958                 return t_hash_start(m, pos);
959
960         if (*pos > 0) {
961                 if (iter->idx < 0)
962                         return p;
963                 (*pos)--;
964                 iter->idx--;
965         }
966
967         p = t_next(m, p, pos);
968
969         if (!p)
970                 return t_hash_start(m, pos);
971
972         return p;
973 }
974
975 static void t_stop(struct seq_file *m, void *p)
976 {
977         mutex_unlock(&ftrace_lock);
978 }
979
980 static int t_show(struct seq_file *m, void *v)
981 {
982         struct ftrace_iterator *iter = m->private;
983         struct dyn_ftrace *rec = v;
984         char str[KSYM_SYMBOL_LEN];
985
986         if (iter->flags & FTRACE_ITER_HASH)
987                 return t_hash_show(m, v);
988
989         if (iter->flags & FTRACE_ITER_PRINTALL) {
990                 seq_printf(m, "#### all functions enabled ####\n");
991                 return 0;
992         }
993
994         if (!rec)
995                 return 0;
996
997         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
998
999         seq_printf(m, "%s\n", str);
1000
1001         return 0;
1002 }
1003
1004 static struct seq_operations show_ftrace_seq_ops = {
1005         .start = t_start,
1006         .next = t_next,
1007         .stop = t_stop,
1008         .show = t_show,
1009 };
1010
1011 static int
1012 ftrace_avail_open(struct inode *inode, struct file *file)
1013 {
1014         struct ftrace_iterator *iter;
1015         int ret;
1016
1017         if (unlikely(ftrace_disabled))
1018                 return -ENODEV;
1019
1020         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1021         if (!iter)
1022                 return -ENOMEM;
1023
1024         iter->pg = ftrace_pages_start;
1025
1026         ret = seq_open(file, &show_ftrace_seq_ops);
1027         if (!ret) {
1028                 struct seq_file *m = file->private_data;
1029
1030                 m->private = iter;
1031         } else {
1032                 kfree(iter);
1033         }
1034
1035         return ret;
1036 }
1037
1038 int ftrace_avail_release(struct inode *inode, struct file *file)
1039 {
1040         struct seq_file *m = (struct seq_file *)file->private_data;
1041         struct ftrace_iterator *iter = m->private;
1042
1043         seq_release(inode, file);
1044         kfree(iter);
1045
1046         return 0;
1047 }
1048
1049 static int
1050 ftrace_failures_open(struct inode *inode, struct file *file)
1051 {
1052         int ret;
1053         struct seq_file *m;
1054         struct ftrace_iterator *iter;
1055
1056         ret = ftrace_avail_open(inode, file);
1057         if (!ret) {
1058                 m = (struct seq_file *)file->private_data;
1059                 iter = (struct ftrace_iterator *)m->private;
1060                 iter->flags = FTRACE_ITER_FAILURES;
1061         }
1062
1063         return ret;
1064 }
1065
1066
1067 static void ftrace_filter_reset(int enable)
1068 {
1069         struct ftrace_page *pg;
1070         struct dyn_ftrace *rec;
1071         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1072
1073         mutex_lock(&ftrace_lock);
1074         if (enable)
1075                 ftrace_filtered = 0;
1076         do_for_each_ftrace_rec(pg, rec) {
1077                 if (rec->flags & FTRACE_FL_FAILED)
1078                         continue;
1079                 rec->flags &= ~type;
1080         } while_for_each_ftrace_rec();
1081         mutex_unlock(&ftrace_lock);
1082 }
1083
1084 static int
1085 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1086 {
1087         struct ftrace_iterator *iter;
1088         int ret = 0;
1089
1090         if (unlikely(ftrace_disabled))
1091                 return -ENODEV;
1092
1093         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1094         if (!iter)
1095                 return -ENOMEM;
1096
1097         mutex_lock(&ftrace_regex_lock);
1098         if ((file->f_mode & FMODE_WRITE) &&
1099             !(file->f_flags & O_APPEND))
1100                 ftrace_filter_reset(enable);
1101
1102         if (file->f_mode & FMODE_READ) {
1103                 iter->pg = ftrace_pages_start;
1104                 iter->flags = enable ? FTRACE_ITER_FILTER :
1105                         FTRACE_ITER_NOTRACE;
1106
1107                 ret = seq_open(file, &show_ftrace_seq_ops);
1108                 if (!ret) {
1109                         struct seq_file *m = file->private_data;
1110                         m->private = iter;
1111                 } else
1112                         kfree(iter);
1113         } else
1114                 file->private_data = iter;
1115         mutex_unlock(&ftrace_regex_lock);
1116
1117         return ret;
1118 }
1119
1120 static int
1121 ftrace_filter_open(struct inode *inode, struct file *file)
1122 {
1123         return ftrace_regex_open(inode, file, 1);
1124 }
1125
1126 static int
1127 ftrace_notrace_open(struct inode *inode, struct file *file)
1128 {
1129         return ftrace_regex_open(inode, file, 0);
1130 }
1131
1132 static loff_t
1133 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1134 {
1135         loff_t ret;
1136
1137         if (file->f_mode & FMODE_READ)
1138                 ret = seq_lseek(file, offset, origin);
1139         else
1140                 file->f_pos = ret = 1;
1141
1142         return ret;
1143 }
1144
1145 enum {
1146         MATCH_FULL,
1147         MATCH_FRONT_ONLY,
1148         MATCH_MIDDLE_ONLY,
1149         MATCH_END_ONLY,
1150 };
1151
1152 /*
1153  * (static function - no need for kernel doc)
1154  *
1155  * Pass in a buffer containing a glob and this function will
1156  * set search to point to the search part of the buffer and
1157  * return the type of search it is (see enum above).
1158  * This does modify buff.
1159  *
1160  * Returns enum type.
1161  *  search returns the pointer to use for comparison.
1162  *  not returns 1 if buff started with a '!'
1163  *     0 otherwise.
1164  */
1165 static int
1166 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1167 {
1168         int type = MATCH_FULL;
1169         int i;
1170
1171         if (buff[0] == '!') {
1172                 *not = 1;
1173                 buff++;
1174                 len--;
1175         } else
1176                 *not = 0;
1177
1178         *search = buff;
1179
1180         for (i = 0; i < len; i++) {
1181                 if (buff[i] == '*') {
1182                         if (!i) {
1183                                 *search = buff + 1;
1184                                 type = MATCH_END_ONLY;
1185                         } else {
1186                                 if (type == MATCH_END_ONLY)
1187                                         type = MATCH_MIDDLE_ONLY;
1188                                 else
1189                                         type = MATCH_FRONT_ONLY;
1190                                 buff[i] = 0;
1191                                 break;
1192                         }
1193                 }
1194         }
1195
1196         return type;
1197 }
1198
1199 static int ftrace_match(char *str, char *regex, int len, int type)
1200 {
1201         int matched = 0;
1202         char *ptr;
1203
1204         switch (type) {
1205         case MATCH_FULL:
1206                 if (strcmp(str, regex) == 0)
1207                         matched = 1;
1208                 break;
1209         case MATCH_FRONT_ONLY:
1210                 if (strncmp(str, regex, len) == 0)
1211                         matched = 1;
1212                 break;
1213         case MATCH_MIDDLE_ONLY:
1214                 if (strstr(str, regex))
1215                         matched = 1;
1216                 break;
1217         case MATCH_END_ONLY:
1218                 ptr = strstr(str, regex);
1219                 if (ptr && (ptr[len] == 0))
1220                         matched = 1;
1221                 break;
1222         }
1223
1224         return matched;
1225 }
1226
1227 static int
1228 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1229 {
1230         char str[KSYM_SYMBOL_LEN];
1231
1232         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1233         return ftrace_match(str, regex, len, type);
1234 }
1235
1236 static void ftrace_match_records(char *buff, int len, int enable)
1237 {
1238         unsigned int search_len;
1239         struct ftrace_page *pg;
1240         struct dyn_ftrace *rec;
1241         unsigned long flag;
1242         char *search;
1243         int type;
1244         int not;
1245
1246         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1247         type = ftrace_setup_glob(buff, len, &search, &not);
1248
1249         search_len = strlen(search);
1250
1251         mutex_lock(&ftrace_lock);
1252         do_for_each_ftrace_rec(pg, rec) {
1253
1254                 if (rec->flags & FTRACE_FL_FAILED)
1255                         continue;
1256
1257                 if (ftrace_match_record(rec, search, search_len, type)) {
1258                         if (not)
1259                                 rec->flags &= ~flag;
1260                         else
1261                                 rec->flags |= flag;
1262                 }
1263                 /*
1264                  * Only enable filtering if we have a function that
1265                  * is filtered on.
1266                  */
1267                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1268                         ftrace_filtered = 1;
1269         } while_for_each_ftrace_rec();
1270         mutex_unlock(&ftrace_lock);
1271 }
1272
1273 static int
1274 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1275                            char *regex, int len, int type)
1276 {
1277         char str[KSYM_SYMBOL_LEN];
1278         char *modname;
1279
1280         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1281
1282         if (!modname || strcmp(modname, mod))
1283                 return 0;
1284
1285         /* blank search means to match all funcs in the mod */
1286         if (len)
1287                 return ftrace_match(str, regex, len, type);
1288         else
1289                 return 1;
1290 }
1291
1292 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1293 {
1294         unsigned search_len = 0;
1295         struct ftrace_page *pg;
1296         struct dyn_ftrace *rec;
1297         int type = MATCH_FULL;
1298         char *search = buff;
1299         unsigned long flag;
1300         int not = 0;
1301
1302         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1303
1304         /* blank or '*' mean the same */
1305         if (strcmp(buff, "*") == 0)
1306                 buff[0] = 0;
1307
1308         /* handle the case of 'dont filter this module' */
1309         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1310                 buff[0] = 0;
1311                 not = 1;
1312         }
1313
1314         if (strlen(buff)) {
1315                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1316                 search_len = strlen(search);
1317         }
1318
1319         mutex_lock(&ftrace_lock);
1320         do_for_each_ftrace_rec(pg, rec) {
1321
1322                 if (rec->flags & FTRACE_FL_FAILED)
1323                         continue;
1324
1325                 if (ftrace_match_module_record(rec, mod,
1326                                                search, search_len, type)) {
1327                         if (not)
1328                                 rec->flags &= ~flag;
1329                         else
1330                                 rec->flags |= flag;
1331                 }
1332                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1333                         ftrace_filtered = 1;
1334
1335         } while_for_each_ftrace_rec();
1336         mutex_unlock(&ftrace_lock);
1337 }
1338
1339 /*
1340  * We register the module command as a template to show others how
1341  * to register the a command as well.
1342  */
1343
1344 static int
1345 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1346 {
1347         char *mod;
1348
1349         /*
1350          * cmd == 'mod' because we only registered this func
1351          * for the 'mod' ftrace_func_command.
1352          * But if you register one func with multiple commands,
1353          * you can tell which command was used by the cmd
1354          * parameter.
1355          */
1356
1357         /* we must have a module name */
1358         if (!param)
1359                 return -EINVAL;
1360
1361         mod = strsep(&param, ":");
1362         if (!strlen(mod))
1363                 return -EINVAL;
1364
1365         ftrace_match_module_records(func, mod, enable);
1366         return 0;
1367 }
1368
1369 static struct ftrace_func_command ftrace_mod_cmd = {
1370         .name                   = "mod",
1371         .func                   = ftrace_mod_callback,
1372 };
1373
1374 static int __init ftrace_mod_cmd_init(void)
1375 {
1376         return register_ftrace_command(&ftrace_mod_cmd);
1377 }
1378 device_initcall(ftrace_mod_cmd_init);
1379
1380 static void
1381 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1382 {
1383         struct ftrace_func_probe *entry;
1384         struct hlist_head *hhd;
1385         struct hlist_node *n;
1386         unsigned long key;
1387         int resched;
1388
1389         key = hash_long(ip, FTRACE_HASH_BITS);
1390
1391         hhd = &ftrace_func_hash[key];
1392
1393         if (hlist_empty(hhd))
1394                 return;
1395
1396         /*
1397          * Disable preemption for these calls to prevent a RCU grace
1398          * period. This syncs the hash iteration and freeing of items
1399          * on the hash. rcu_read_lock is too dangerous here.
1400          */
1401         resched = ftrace_preempt_disable();
1402         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1403                 if (entry->ip == ip)
1404                         entry->ops->func(ip, parent_ip, &entry->data);
1405         }
1406         ftrace_preempt_enable(resched);
1407 }
1408
1409 static struct ftrace_ops trace_probe_ops __read_mostly =
1410 {
1411         .func = function_trace_probe_call,
1412 };
1413
1414 static int ftrace_probe_registered;
1415
1416 static void __enable_ftrace_function_probe(void)
1417 {
1418         int i;
1419
1420         if (ftrace_probe_registered)
1421                 return;
1422
1423         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1424                 struct hlist_head *hhd = &ftrace_func_hash[i];
1425                 if (hhd->first)
1426                         break;
1427         }
1428         /* Nothing registered? */
1429         if (i == FTRACE_FUNC_HASHSIZE)
1430                 return;
1431
1432         __register_ftrace_function(&trace_probe_ops);
1433         ftrace_startup(0);
1434         ftrace_probe_registered = 1;
1435 }
1436
1437 static void __disable_ftrace_function_probe(void)
1438 {
1439         int i;
1440
1441         if (!ftrace_probe_registered)
1442                 return;
1443
1444         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1445                 struct hlist_head *hhd = &ftrace_func_hash[i];
1446                 if (hhd->first)
1447                         return;
1448         }
1449
1450         /* no more funcs left */
1451         __unregister_ftrace_function(&trace_probe_ops);
1452         ftrace_shutdown(0);
1453         ftrace_probe_registered = 0;
1454 }
1455
1456
1457 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1458 {
1459         struct ftrace_func_probe *entry =
1460                 container_of(rhp, struct ftrace_func_probe, rcu);
1461
1462         if (entry->ops->free)
1463                 entry->ops->free(&entry->data);
1464         kfree(entry);
1465 }
1466
1467
1468 int
1469 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1470                               void *data)
1471 {
1472         struct ftrace_func_probe *entry;
1473         struct ftrace_page *pg;
1474         struct dyn_ftrace *rec;
1475         int type, len, not;
1476         unsigned long key;
1477         int count = 0;
1478         char *search;
1479
1480         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1481         len = strlen(search);
1482
1483         /* we do not support '!' for function probes */
1484         if (WARN_ON(not))
1485                 return -EINVAL;
1486
1487         mutex_lock(&ftrace_lock);
1488         do_for_each_ftrace_rec(pg, rec) {
1489
1490                 if (rec->flags & FTRACE_FL_FAILED)
1491                         continue;
1492
1493                 if (!ftrace_match_record(rec, search, len, type))
1494                         continue;
1495
1496                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1497                 if (!entry) {
1498                         /* If we did not process any, then return error */
1499                         if (!count)
1500                                 count = -ENOMEM;
1501                         goto out_unlock;
1502                 }
1503
1504                 count++;
1505
1506                 entry->data = data;
1507
1508                 /*
1509                  * The caller might want to do something special
1510                  * for each function we find. We call the callback
1511                  * to give the caller an opportunity to do so.
1512                  */
1513                 if (ops->callback) {
1514                         if (ops->callback(rec->ip, &entry->data) < 0) {
1515                                 /* caller does not like this func */
1516                                 kfree(entry);
1517                                 continue;
1518                         }
1519                 }
1520
1521                 entry->ops = ops;
1522                 entry->ip = rec->ip;
1523
1524                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1525                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1526
1527         } while_for_each_ftrace_rec();
1528         __enable_ftrace_function_probe();
1529
1530  out_unlock:
1531         mutex_unlock(&ftrace_lock);
1532
1533         return count;
1534 }
1535
1536 enum {
1537         PROBE_TEST_FUNC         = 1,
1538         PROBE_TEST_DATA         = 2
1539 };
1540
1541 static void
1542 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1543                                   void *data, int flags)
1544 {
1545         struct ftrace_func_probe *entry;
1546         struct hlist_node *n, *tmp;
1547         char str[KSYM_SYMBOL_LEN];
1548         int type = MATCH_FULL;
1549         int i, len = 0;
1550         char *search;
1551
1552         if (glob && (strcmp(glob, "*") || !strlen(glob)))
1553                 glob = NULL;
1554         else {
1555                 int not;
1556
1557                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1558                 len = strlen(search);
1559
1560                 /* we do not support '!' for function probes */
1561                 if (WARN_ON(not))
1562                         return;
1563         }
1564
1565         mutex_lock(&ftrace_lock);
1566         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1567                 struct hlist_head *hhd = &ftrace_func_hash[i];
1568
1569                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1570
1571                         /* break up if statements for readability */
1572                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1573                                 continue;
1574
1575                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
1576                                 continue;
1577
1578                         /* do this last, since it is the most expensive */
1579                         if (glob) {
1580                                 kallsyms_lookup(entry->ip, NULL, NULL,
1581                                                 NULL, str);
1582                                 if (!ftrace_match(str, glob, len, type))
1583                                         continue;
1584                         }
1585
1586                         hlist_del(&entry->node);
1587                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1588                 }
1589         }
1590         __disable_ftrace_function_probe();
1591         mutex_unlock(&ftrace_lock);
1592 }
1593
1594 void
1595 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1596                                 void *data)
1597 {
1598         __unregister_ftrace_function_probe(glob, ops, data,
1599                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
1600 }
1601
1602 void
1603 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1604 {
1605         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1606 }
1607
1608 void unregister_ftrace_function_probe_all(char *glob)
1609 {
1610         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1611 }
1612
1613 static LIST_HEAD(ftrace_commands);
1614 static DEFINE_MUTEX(ftrace_cmd_mutex);
1615
1616 int register_ftrace_command(struct ftrace_func_command *cmd)
1617 {
1618         struct ftrace_func_command *p;
1619         int ret = 0;
1620
1621         mutex_lock(&ftrace_cmd_mutex);
1622         list_for_each_entry(p, &ftrace_commands, list) {
1623                 if (strcmp(cmd->name, p->name) == 0) {
1624                         ret = -EBUSY;
1625                         goto out_unlock;
1626                 }
1627         }
1628         list_add(&cmd->list, &ftrace_commands);
1629  out_unlock:
1630         mutex_unlock(&ftrace_cmd_mutex);
1631
1632         return ret;
1633 }
1634
1635 int unregister_ftrace_command(struct ftrace_func_command *cmd)
1636 {
1637         struct ftrace_func_command *p, *n;
1638         int ret = -ENODEV;
1639
1640         mutex_lock(&ftrace_cmd_mutex);
1641         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1642                 if (strcmp(cmd->name, p->name) == 0) {
1643                         ret = 0;
1644                         list_del_init(&p->list);
1645                         goto out_unlock;
1646                 }
1647         }
1648  out_unlock:
1649         mutex_unlock(&ftrace_cmd_mutex);
1650
1651         return ret;
1652 }
1653
1654 static int ftrace_process_regex(char *buff, int len, int enable)
1655 {
1656         char *func, *command, *next = buff;
1657         struct ftrace_func_command *p;
1658         int ret = -EINVAL;
1659
1660         func = strsep(&next, ":");
1661
1662         if (!next) {
1663                 ftrace_match_records(func, len, enable);
1664                 return 0;
1665         }
1666
1667         /* command found */
1668
1669         command = strsep(&next, ":");
1670
1671         mutex_lock(&ftrace_cmd_mutex);
1672         list_for_each_entry(p, &ftrace_commands, list) {
1673                 if (strcmp(p->name, command) == 0) {
1674                         ret = p->func(func, command, next, enable);
1675                         goto out_unlock;
1676                 }
1677         }
1678  out_unlock:
1679         mutex_unlock(&ftrace_cmd_mutex);
1680
1681         return ret;
1682 }
1683
1684 static ssize_t
1685 ftrace_regex_write(struct file *file, const char __user *ubuf,
1686                    size_t cnt, loff_t *ppos, int enable)
1687 {
1688         struct ftrace_iterator *iter;
1689         char ch;
1690         size_t read = 0;
1691         ssize_t ret;
1692
1693         if (!cnt || cnt < 0)
1694                 return 0;
1695
1696         mutex_lock(&ftrace_regex_lock);
1697
1698         if (file->f_mode & FMODE_READ) {
1699                 struct seq_file *m = file->private_data;
1700                 iter = m->private;
1701         } else
1702                 iter = file->private_data;
1703
1704         if (!*ppos) {
1705                 iter->flags &= ~FTRACE_ITER_CONT;
1706                 iter->buffer_idx = 0;
1707         }
1708
1709         ret = get_user(ch, ubuf++);
1710         if (ret)
1711                 goto out;
1712         read++;
1713         cnt--;
1714
1715         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1716                 /* skip white space */
1717                 while (cnt && isspace(ch)) {
1718                         ret = get_user(ch, ubuf++);
1719                         if (ret)
1720                                 goto out;
1721                         read++;
1722                         cnt--;
1723                 }
1724
1725                 if (isspace(ch)) {
1726                         file->f_pos += read;
1727                         ret = read;
1728                         goto out;
1729                 }
1730
1731                 iter->buffer_idx = 0;
1732         }
1733
1734         while (cnt && !isspace(ch)) {
1735                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1736                         iter->buffer[iter->buffer_idx++] = ch;
1737                 else {
1738                         ret = -EINVAL;
1739                         goto out;
1740                 }
1741                 ret = get_user(ch, ubuf++);
1742                 if (ret)
1743                         goto out;
1744                 read++;
1745                 cnt--;
1746         }
1747
1748         if (isspace(ch)) {
1749                 iter->filtered++;
1750                 iter->buffer[iter->buffer_idx] = 0;
1751                 ret = ftrace_process_regex(iter->buffer,
1752                                            iter->buffer_idx, enable);
1753                 if (ret)
1754                         goto out;
1755                 iter->buffer_idx = 0;
1756         } else
1757                 iter->flags |= FTRACE_ITER_CONT;
1758
1759
1760         file->f_pos += read;
1761
1762         ret = read;
1763  out:
1764         mutex_unlock(&ftrace_regex_lock);
1765
1766         return ret;
1767 }
1768
1769 static ssize_t
1770 ftrace_filter_write(struct file *file, const char __user *ubuf,
1771                     size_t cnt, loff_t *ppos)
1772 {
1773         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1774 }
1775
1776 static ssize_t
1777 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1778                      size_t cnt, loff_t *ppos)
1779 {
1780         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1781 }
1782
1783 static void
1784 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1785 {
1786         if (unlikely(ftrace_disabled))
1787                 return;
1788
1789         mutex_lock(&ftrace_regex_lock);
1790         if (reset)
1791                 ftrace_filter_reset(enable);
1792         if (buf)
1793                 ftrace_match_records(buf, len, enable);
1794         mutex_unlock(&ftrace_regex_lock);
1795 }
1796
1797 /**
1798  * ftrace_set_filter - set a function to filter on in ftrace
1799  * @buf - the string that holds the function filter text.
1800  * @len - the length of the string.
1801  * @reset - non zero to reset all filters before applying this filter.
1802  *
1803  * Filters denote which functions should be enabled when tracing is enabled.
1804  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1805  */
1806 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1807 {
1808         ftrace_set_regex(buf, len, reset, 1);
1809 }
1810
1811 /**
1812  * ftrace_set_notrace - set a function to not trace in ftrace
1813  * @buf - the string that holds the function notrace text.
1814  * @len - the length of the string.
1815  * @reset - non zero to reset all filters before applying this filter.
1816  *
1817  * Notrace Filters denote which functions should not be enabled when tracing
1818  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1819  * for tracing.
1820  */
1821 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1822 {
1823         ftrace_set_regex(buf, len, reset, 0);
1824 }
1825
1826 static int
1827 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1828 {
1829         struct seq_file *m = (struct seq_file *)file->private_data;
1830         struct ftrace_iterator *iter;
1831
1832         mutex_lock(&ftrace_regex_lock);
1833         if (file->f_mode & FMODE_READ) {
1834                 iter = m->private;
1835
1836                 seq_release(inode, file);
1837         } else
1838                 iter = file->private_data;
1839
1840         if (iter->buffer_idx) {
1841                 iter->filtered++;
1842                 iter->buffer[iter->buffer_idx] = 0;
1843                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1844         }
1845
1846         mutex_lock(&ftrace_lock);
1847         if (ftrace_start_up && ftrace_enabled)
1848                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1849         mutex_unlock(&ftrace_lock);
1850
1851         kfree(iter);
1852         mutex_unlock(&ftrace_regex_lock);
1853         return 0;
1854 }
1855
1856 static int
1857 ftrace_filter_release(struct inode *inode, struct file *file)
1858 {
1859         return ftrace_regex_release(inode, file, 1);
1860 }
1861
1862 static int
1863 ftrace_notrace_release(struct inode *inode, struct file *file)
1864 {
1865         return ftrace_regex_release(inode, file, 0);
1866 }
1867
1868 static const struct file_operations ftrace_avail_fops = {
1869         .open = ftrace_avail_open,
1870         .read = seq_read,
1871         .llseek = seq_lseek,
1872         .release = ftrace_avail_release,
1873 };
1874
1875 static const struct file_operations ftrace_failures_fops = {
1876         .open = ftrace_failures_open,
1877         .read = seq_read,
1878         .llseek = seq_lseek,
1879         .release = ftrace_avail_release,
1880 };
1881
1882 static const struct file_operations ftrace_filter_fops = {
1883         .open = ftrace_filter_open,
1884         .read = seq_read,
1885         .write = ftrace_filter_write,
1886         .llseek = ftrace_regex_lseek,
1887         .release = ftrace_filter_release,
1888 };
1889
1890 static const struct file_operations ftrace_notrace_fops = {
1891         .open = ftrace_notrace_open,
1892         .read = seq_read,
1893         .write = ftrace_notrace_write,
1894         .llseek = ftrace_regex_lseek,
1895         .release = ftrace_notrace_release,
1896 };
1897
1898 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1899
1900 static DEFINE_MUTEX(graph_lock);
1901
1902 int ftrace_graph_count;
1903 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1904
1905 static void *
1906 g_next(struct seq_file *m, void *v, loff_t *pos)
1907 {
1908         unsigned long *array = m->private;
1909         int index = *pos;
1910
1911         (*pos)++;
1912
1913         if (index >= ftrace_graph_count)
1914                 return NULL;
1915
1916         return &array[index];
1917 }
1918
1919 static void *g_start(struct seq_file *m, loff_t *pos)
1920 {
1921         void *p = NULL;
1922
1923         mutex_lock(&graph_lock);
1924
1925         /* Nothing, tell g_show to print all functions are enabled */
1926         if (!ftrace_graph_count && !*pos)
1927                 return (void *)1;
1928
1929         p = g_next(m, p, pos);
1930
1931         return p;
1932 }
1933
1934 static void g_stop(struct seq_file *m, void *p)
1935 {
1936         mutex_unlock(&graph_lock);
1937 }
1938
1939 static int g_show(struct seq_file *m, void *v)
1940 {
1941         unsigned long *ptr = v;
1942         char str[KSYM_SYMBOL_LEN];
1943
1944         if (!ptr)
1945                 return 0;
1946
1947         if (ptr == (unsigned long *)1) {
1948                 seq_printf(m, "#### all functions enabled ####\n");
1949                 return 0;
1950         }
1951
1952         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1953
1954         seq_printf(m, "%s\n", str);
1955
1956         return 0;
1957 }
1958
1959 static struct seq_operations ftrace_graph_seq_ops = {
1960         .start = g_start,
1961         .next = g_next,
1962         .stop = g_stop,
1963         .show = g_show,
1964 };
1965
1966 static int
1967 ftrace_graph_open(struct inode *inode, struct file *file)
1968 {
1969         int ret = 0;
1970
1971         if (unlikely(ftrace_disabled))
1972                 return -ENODEV;
1973
1974         mutex_lock(&graph_lock);
1975         if ((file->f_mode & FMODE_WRITE) &&
1976             !(file->f_flags & O_APPEND)) {
1977                 ftrace_graph_count = 0;
1978                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1979         }
1980
1981         if (file->f_mode & FMODE_READ) {
1982                 ret = seq_open(file, &ftrace_graph_seq_ops);
1983                 if (!ret) {
1984                         struct seq_file *m = file->private_data;
1985                         m->private = ftrace_graph_funcs;
1986                 }
1987         } else
1988                 file->private_data = ftrace_graph_funcs;
1989         mutex_unlock(&graph_lock);
1990
1991         return ret;
1992 }
1993
1994 static int
1995 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
1996 {
1997         struct dyn_ftrace *rec;
1998         struct ftrace_page *pg;
1999         int search_len;
2000         int found = 0;
2001         int type, not;
2002         char *search;
2003         bool exists;
2004         int i;
2005
2006         if (ftrace_disabled)
2007                 return -ENODEV;
2008
2009         /* decode regex */
2010         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2011         if (not)
2012                 return -EINVAL;
2013
2014         search_len = strlen(search);
2015
2016         mutex_lock(&ftrace_lock);
2017         do_for_each_ftrace_rec(pg, rec) {
2018
2019                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2020                         break;
2021
2022                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2023                         continue;
2024
2025                 if (ftrace_match_record(rec, search, search_len, type)) {
2026                         /* ensure it is not already in the array */
2027                         exists = false;
2028                         for (i = 0; i < *idx; i++)
2029                                 if (array[i] == rec->ip) {
2030                                         exists = true;
2031                                         break;
2032                                 }
2033                         if (!exists) {
2034                                 array[(*idx)++] = rec->ip;
2035                                 found = 1;
2036                         }
2037                 }
2038         } while_for_each_ftrace_rec();
2039
2040         mutex_unlock(&ftrace_lock);
2041
2042         return found ? 0 : -EINVAL;
2043 }
2044
2045 static ssize_t
2046 ftrace_graph_write(struct file *file, const char __user *ubuf,
2047                    size_t cnt, loff_t *ppos)
2048 {
2049         unsigned char buffer[FTRACE_BUFF_MAX+1];
2050         unsigned long *array;
2051         size_t read = 0;
2052         ssize_t ret;
2053         int index = 0;
2054         char ch;
2055
2056         if (!cnt || cnt < 0)
2057                 return 0;
2058
2059         mutex_lock(&graph_lock);
2060
2061         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2062                 ret = -EBUSY;
2063                 goto out;
2064         }
2065
2066         if (file->f_mode & FMODE_READ) {
2067                 struct seq_file *m = file->private_data;
2068                 array = m->private;
2069         } else
2070                 array = file->private_data;
2071
2072         ret = get_user(ch, ubuf++);
2073         if (ret)
2074                 goto out;
2075         read++;
2076         cnt--;
2077
2078         /* skip white space */
2079         while (cnt && isspace(ch)) {
2080                 ret = get_user(ch, ubuf++);
2081                 if (ret)
2082                         goto out;
2083                 read++;
2084                 cnt--;
2085         }
2086
2087         if (isspace(ch)) {
2088                 *ppos += read;
2089                 ret = read;
2090                 goto out;
2091         }
2092
2093         while (cnt && !isspace(ch)) {
2094                 if (index < FTRACE_BUFF_MAX)
2095                         buffer[index++] = ch;
2096                 else {
2097                         ret = -EINVAL;
2098                         goto out;
2099                 }
2100                 ret = get_user(ch, ubuf++);
2101                 if (ret)
2102                         goto out;
2103                 read++;
2104                 cnt--;
2105         }
2106         buffer[index] = 0;
2107
2108         /* we allow only one expression at a time */
2109         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2110         if (ret)
2111                 goto out;
2112
2113         file->f_pos += read;
2114
2115         ret = read;
2116  out:
2117         mutex_unlock(&graph_lock);
2118
2119         return ret;
2120 }
2121
2122 static const struct file_operations ftrace_graph_fops = {
2123         .open = ftrace_graph_open,
2124         .read = seq_read,
2125         .write = ftrace_graph_write,
2126 };
2127 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2128
2129 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2130 {
2131         struct dentry *entry;
2132
2133         entry = debugfs_create_file("available_filter_functions", 0444,
2134                                     d_tracer, NULL, &ftrace_avail_fops);
2135         if (!entry)
2136                 pr_warning("Could not create debugfs "
2137                            "'available_filter_functions' entry\n");
2138
2139         entry = debugfs_create_file("failures", 0444,
2140                                     d_tracer, NULL, &ftrace_failures_fops);
2141         if (!entry)
2142                 pr_warning("Could not create debugfs 'failures' entry\n");
2143
2144         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2145                                     NULL, &ftrace_filter_fops);
2146         if (!entry)
2147                 pr_warning("Could not create debugfs "
2148                            "'set_ftrace_filter' entry\n");
2149
2150         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2151                                     NULL, &ftrace_notrace_fops);
2152         if (!entry)
2153                 pr_warning("Could not create debugfs "
2154                            "'set_ftrace_notrace' entry\n");
2155
2156 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2157         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2158                                     NULL,
2159                                     &ftrace_graph_fops);
2160         if (!entry)
2161                 pr_warning("Could not create debugfs "
2162                            "'set_graph_function' entry\n");
2163 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2164
2165         return 0;
2166 }
2167
2168 static int ftrace_convert_nops(struct module *mod,
2169                                unsigned long *start,
2170                                unsigned long *end)
2171 {
2172         unsigned long *p;
2173         unsigned long addr;
2174         unsigned long flags;
2175
2176         mutex_lock(&ftrace_lock);
2177         p = start;
2178         while (p < end) {
2179                 addr = ftrace_call_adjust(*p++);
2180                 /*
2181                  * Some architecture linkers will pad between
2182                  * the different mcount_loc sections of different
2183                  * object files to satisfy alignments.
2184                  * Skip any NULL pointers.
2185                  */
2186                 if (!addr)
2187                         continue;
2188                 ftrace_record_ip(addr);
2189         }
2190
2191         /* disable interrupts to prevent kstop machine */
2192         local_irq_save(flags);
2193         ftrace_update_code(mod);
2194         local_irq_restore(flags);
2195         mutex_unlock(&ftrace_lock);
2196
2197         return 0;
2198 }
2199
2200 void ftrace_init_module(struct module *mod,
2201                         unsigned long *start, unsigned long *end)
2202 {
2203         if (ftrace_disabled || start == end)
2204                 return;
2205         ftrace_convert_nops(mod, start, end);
2206 }
2207
2208 extern unsigned long __start_mcount_loc[];
2209 extern unsigned long __stop_mcount_loc[];
2210
2211 void __init ftrace_init(void)
2212 {
2213         unsigned long count, addr, flags;
2214         int ret;
2215
2216         /* Keep the ftrace pointer to the stub */
2217         addr = (unsigned long)ftrace_stub;
2218
2219         local_irq_save(flags);
2220         ftrace_dyn_arch_init(&addr);
2221         local_irq_restore(flags);
2222
2223         /* ftrace_dyn_arch_init places the return code in addr */
2224         if (addr)
2225                 goto failed;
2226
2227         count = __stop_mcount_loc - __start_mcount_loc;
2228
2229         ret = ftrace_dyn_table_alloc(count);
2230         if (ret)
2231                 goto failed;
2232
2233         last_ftrace_enabled = ftrace_enabled = 1;
2234
2235         ret = ftrace_convert_nops(NULL,
2236                                   __start_mcount_loc,
2237                                   __stop_mcount_loc);
2238
2239         return;
2240  failed:
2241         ftrace_disabled = 1;
2242 }
2243
2244 #else
2245
2246 static int __init ftrace_nodyn_init(void)
2247 {
2248         ftrace_enabled = 1;
2249         return 0;
2250 }
2251 device_initcall(ftrace_nodyn_init);
2252
2253 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2254 static inline void ftrace_startup_enable(int command) { }
2255 /* Keep as macros so we do not need to define the commands */
2256 # define ftrace_startup(command)        do { } while (0)
2257 # define ftrace_shutdown(command)       do { } while (0)
2258 # define ftrace_startup_sysctl()        do { } while (0)
2259 # define ftrace_shutdown_sysctl()       do { } while (0)
2260 #endif /* CONFIG_DYNAMIC_FTRACE */
2261
2262 static ssize_t
2263 ftrace_pid_read(struct file *file, char __user *ubuf,
2264                        size_t cnt, loff_t *ppos)
2265 {
2266         char buf[64];
2267         int r;
2268
2269         if (ftrace_pid_trace == ftrace_swapper_pid)
2270                 r = sprintf(buf, "swapper tasks\n");
2271         else if (ftrace_pid_trace)
2272                 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2273         else
2274                 r = sprintf(buf, "no pid\n");
2275
2276         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2277 }
2278
2279 static void clear_ftrace_swapper(void)
2280 {
2281         struct task_struct *p;
2282         int cpu;
2283
2284         get_online_cpus();
2285         for_each_online_cpu(cpu) {
2286                 p = idle_task(cpu);
2287                 clear_tsk_trace_trace(p);
2288         }
2289         put_online_cpus();
2290 }
2291
2292 static void set_ftrace_swapper(void)
2293 {
2294         struct task_struct *p;
2295         int cpu;
2296
2297         get_online_cpus();
2298         for_each_online_cpu(cpu) {
2299                 p = idle_task(cpu);
2300                 set_tsk_trace_trace(p);
2301         }
2302         put_online_cpus();
2303 }
2304
2305 static void clear_ftrace_pid(struct pid *pid)
2306 {
2307         struct task_struct *p;
2308
2309         rcu_read_lock();
2310         do_each_pid_task(pid, PIDTYPE_PID, p) {
2311                 clear_tsk_trace_trace(p);
2312         } while_each_pid_task(pid, PIDTYPE_PID, p);
2313         rcu_read_unlock();
2314
2315         put_pid(pid);
2316 }
2317
2318 static void set_ftrace_pid(struct pid *pid)
2319 {
2320         struct task_struct *p;
2321
2322         rcu_read_lock();
2323         do_each_pid_task(pid, PIDTYPE_PID, p) {
2324                 set_tsk_trace_trace(p);
2325         } while_each_pid_task(pid, PIDTYPE_PID, p);
2326         rcu_read_unlock();
2327 }
2328
2329 static void clear_ftrace_pid_task(struct pid **pid)
2330 {
2331         if (*pid == ftrace_swapper_pid)
2332                 clear_ftrace_swapper();
2333         else
2334                 clear_ftrace_pid(*pid);
2335
2336         *pid = NULL;
2337 }
2338
2339 static void set_ftrace_pid_task(struct pid *pid)
2340 {
2341         if (pid == ftrace_swapper_pid)
2342                 set_ftrace_swapper();
2343         else
2344                 set_ftrace_pid(pid);
2345 }
2346
2347 static ssize_t
2348 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2349                    size_t cnt, loff_t *ppos)
2350 {
2351         struct pid *pid;
2352         char buf[64];
2353         long val;
2354         int ret;
2355
2356         if (cnt >= sizeof(buf))
2357                 return -EINVAL;
2358
2359         if (copy_from_user(&buf, ubuf, cnt))
2360                 return -EFAULT;
2361
2362         buf[cnt] = 0;
2363
2364         ret = strict_strtol(buf, 10, &val);
2365         if (ret < 0)
2366                 return ret;
2367
2368         mutex_lock(&ftrace_lock);
2369         if (val < 0) {
2370                 /* disable pid tracing */
2371                 if (!ftrace_pid_trace)
2372                         goto out;
2373
2374                 clear_ftrace_pid_task(&ftrace_pid_trace);
2375
2376         } else {
2377                 /* swapper task is special */
2378                 if (!val) {
2379                         pid = ftrace_swapper_pid;
2380                         if (pid == ftrace_pid_trace)
2381                                 goto out;
2382                 } else {
2383                         pid = find_get_pid(val);
2384
2385                         if (pid == ftrace_pid_trace) {
2386                                 put_pid(pid);
2387                                 goto out;
2388                         }
2389                 }
2390
2391                 if (ftrace_pid_trace)
2392                         clear_ftrace_pid_task(&ftrace_pid_trace);
2393
2394                 if (!pid)
2395                         goto out;
2396
2397                 ftrace_pid_trace = pid;
2398
2399                 set_ftrace_pid_task(ftrace_pid_trace);
2400         }
2401
2402         /* update the function call */
2403         ftrace_update_pid_func();
2404         ftrace_startup_enable(0);
2405
2406  out:
2407         mutex_unlock(&ftrace_lock);
2408
2409         return cnt;
2410 }
2411
2412 static const struct file_operations ftrace_pid_fops = {
2413         .read = ftrace_pid_read,
2414         .write = ftrace_pid_write,
2415 };
2416
2417 static __init int ftrace_init_debugfs(void)
2418 {
2419         struct dentry *d_tracer;
2420         struct dentry *entry;
2421
2422         d_tracer = tracing_init_dentry();
2423         if (!d_tracer)
2424                 return 0;
2425
2426         ftrace_init_dyn_debugfs(d_tracer);
2427
2428         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2429                                     NULL, &ftrace_pid_fops);
2430         if (!entry)
2431                 pr_warning("Could not create debugfs "
2432                            "'set_ftrace_pid' entry\n");
2433         return 0;
2434 }
2435 fs_initcall(ftrace_init_debugfs);
2436
2437 /**
2438  * ftrace_kill - kill ftrace
2439  *
2440  * This function should be used by panic code. It stops ftrace
2441  * but in a not so nice way. If you need to simply kill ftrace
2442  * from a non-atomic section, use ftrace_kill.
2443  */
2444 void ftrace_kill(void)
2445 {
2446         ftrace_disabled = 1;
2447         ftrace_enabled = 0;
2448         clear_ftrace_function();
2449 }
2450
2451 /**
2452  * register_ftrace_function - register a function for profiling
2453  * @ops - ops structure that holds the function for profiling.
2454  *
2455  * Register a function to be called by all functions in the
2456  * kernel.
2457  *
2458  * Note: @ops->func and all the functions it calls must be labeled
2459  *       with "notrace", otherwise it will go into a
2460  *       recursive loop.
2461  */
2462 int register_ftrace_function(struct ftrace_ops *ops)
2463 {
2464         int ret;
2465
2466         if (unlikely(ftrace_disabled))
2467                 return -1;
2468
2469         mutex_lock(&ftrace_lock);
2470
2471         ret = __register_ftrace_function(ops);
2472         ftrace_startup(0);
2473
2474         mutex_unlock(&ftrace_lock);
2475         return ret;
2476 }
2477
2478 /**
2479  * unregister_ftrace_function - unregister a function for profiling.
2480  * @ops - ops structure that holds the function to unregister
2481  *
2482  * Unregister a function that was added to be called by ftrace profiling.
2483  */
2484 int unregister_ftrace_function(struct ftrace_ops *ops)
2485 {
2486         int ret;
2487
2488         mutex_lock(&ftrace_lock);
2489         ret = __unregister_ftrace_function(ops);
2490         ftrace_shutdown(0);
2491         mutex_unlock(&ftrace_lock);
2492
2493         return ret;
2494 }
2495
2496 int
2497 ftrace_enable_sysctl(struct ctl_table *table, int write,
2498                      struct file *file, void __user *buffer, size_t *lenp,
2499                      loff_t *ppos)
2500 {
2501         int ret;
2502
2503         if (unlikely(ftrace_disabled))
2504                 return -ENODEV;
2505
2506         mutex_lock(&ftrace_lock);
2507
2508         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
2509
2510         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
2511                 goto out;
2512
2513         last_ftrace_enabled = ftrace_enabled;
2514
2515         if (ftrace_enabled) {
2516
2517                 ftrace_startup_sysctl();
2518
2519                 /* we are starting ftrace again */
2520                 if (ftrace_list != &ftrace_list_end) {
2521                         if (ftrace_list->next == &ftrace_list_end)
2522                                 ftrace_trace_function = ftrace_list->func;
2523                         else
2524                                 ftrace_trace_function = ftrace_list_func;
2525                 }
2526
2527         } else {
2528                 /* stopping ftrace calls (just send to ftrace_stub) */
2529                 ftrace_trace_function = ftrace_stub;
2530
2531                 ftrace_shutdown_sysctl();
2532         }
2533
2534  out:
2535         mutex_unlock(&ftrace_lock);
2536         return ret;
2537 }
2538
2539 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2540
2541 static atomic_t ftrace_graph_active;
2542 static struct notifier_block ftrace_suspend_notifier;
2543
2544 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
2545 {
2546         return 0;
2547 }
2548
2549 /* The callbacks that hook a function */
2550 trace_func_graph_ret_t ftrace_graph_return =
2551                         (trace_func_graph_ret_t)ftrace_stub;
2552 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
2553
2554 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2555 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
2556 {
2557         int i;
2558         int ret = 0;
2559         unsigned long flags;
2560         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
2561         struct task_struct *g, *t;
2562
2563         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
2564                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
2565                                         * sizeof(struct ftrace_ret_stack),
2566                                         GFP_KERNEL);
2567                 if (!ret_stack_list[i]) {
2568                         start = 0;
2569                         end = i;
2570                         ret = -ENOMEM;
2571                         goto free;
2572                 }
2573         }
2574
2575         read_lock_irqsave(&tasklist_lock, flags);
2576         do_each_thread(g, t) {
2577                 if (start == end) {
2578                         ret = -EAGAIN;
2579                         goto unlock;
2580                 }
2581
2582                 if (t->ret_stack == NULL) {
2583                         t->curr_ret_stack = -1;
2584                         /* Make sure IRQs see the -1 first: */
2585                         barrier();
2586                         t->ret_stack = ret_stack_list[start++];
2587                         atomic_set(&t->tracing_graph_pause, 0);
2588                         atomic_set(&t->trace_overrun, 0);
2589                 }
2590         } while_each_thread(g, t);
2591
2592 unlock:
2593         read_unlock_irqrestore(&tasklist_lock, flags);
2594 free:
2595         for (i = start; i < end; i++)
2596                 kfree(ret_stack_list[i]);
2597         return ret;
2598 }
2599
2600 static void
2601 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
2602                                 struct task_struct *next)
2603 {
2604         unsigned long long timestamp;
2605         int index;
2606
2607         /*
2608          * Does the user want to count the time a function was asleep.
2609          * If so, do not update the time stamps.
2610          */
2611         if (trace_flags & TRACE_ITER_SLEEP_TIME)
2612                 return;
2613
2614         timestamp = trace_clock_local();
2615
2616         prev->ftrace_timestamp = timestamp;
2617
2618         /* only process tasks that we timestamped */
2619         if (!next->ftrace_timestamp)
2620                 return;
2621
2622         /*
2623          * Update all the counters in next to make up for the
2624          * time next was sleeping.
2625          */
2626         timestamp -= next->ftrace_timestamp;
2627
2628         for (index = next->curr_ret_stack; index >= 0; index--)
2629                 next->ret_stack[index].calltime += timestamp;
2630 }
2631
2632 /* Allocate a return stack for each task */
2633 static int start_graph_tracing(void)
2634 {
2635         struct ftrace_ret_stack **ret_stack_list;
2636         int ret, cpu;
2637
2638         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2639                                 sizeof(struct ftrace_ret_stack *),
2640                                 GFP_KERNEL);
2641
2642         if (!ret_stack_list)
2643                 return -ENOMEM;
2644
2645         /* The cpu_boot init_task->ret_stack will never be freed */
2646         for_each_online_cpu(cpu)
2647                 ftrace_graph_init_task(idle_task(cpu));
2648
2649         do {
2650                 ret = alloc_retstack_tasklist(ret_stack_list);
2651         } while (ret == -EAGAIN);
2652
2653         if (!ret) {
2654                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
2655                 if (ret)
2656                         pr_info("ftrace_graph: Couldn't activate tracepoint"
2657                                 " probe to kernel_sched_switch\n");
2658         }
2659
2660         kfree(ret_stack_list);
2661         return ret;
2662 }
2663
2664 /*
2665  * Hibernation protection.
2666  * The state of the current task is too much unstable during
2667  * suspend/restore to disk. We want to protect against that.
2668  */
2669 static int
2670 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2671                                                         void *unused)
2672 {
2673         switch (state) {
2674         case PM_HIBERNATION_PREPARE:
2675                 pause_graph_tracing();
2676                 break;
2677
2678         case PM_POST_HIBERNATION:
2679                 unpause_graph_tracing();
2680                 break;
2681         }
2682         return NOTIFY_DONE;
2683 }
2684
2685 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2686                         trace_func_graph_ent_t entryfunc)
2687 {
2688         int ret = 0;
2689
2690         mutex_lock(&ftrace_lock);
2691
2692         /* we currently allow only one tracer registered at a time */
2693         if (atomic_read(&ftrace_graph_active)) {
2694                 ret = -EBUSY;
2695                 goto out;
2696         }
2697
2698         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2699         register_pm_notifier(&ftrace_suspend_notifier);
2700
2701         atomic_inc(&ftrace_graph_active);
2702         ret = start_graph_tracing();
2703         if (ret) {
2704                 atomic_dec(&ftrace_graph_active);
2705                 goto out;
2706         }
2707
2708         ftrace_graph_return = retfunc;
2709         ftrace_graph_entry = entryfunc;
2710
2711         ftrace_startup(FTRACE_START_FUNC_RET);
2712
2713 out:
2714         mutex_unlock(&ftrace_lock);
2715         return ret;
2716 }
2717
2718 void unregister_ftrace_graph(void)
2719 {
2720         mutex_lock(&ftrace_lock);
2721
2722         if (!unlikely(atomic_read(&ftrace_graph_active)))
2723                 goto out;
2724
2725         atomic_dec(&ftrace_graph_active);
2726         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
2727         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2728         ftrace_graph_entry = ftrace_graph_entry_stub;
2729         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2730         unregister_pm_notifier(&ftrace_suspend_notifier);
2731
2732  out:
2733         mutex_unlock(&ftrace_lock);
2734 }
2735
2736 /* Allocate a return stack for newly created task */
2737 void ftrace_graph_init_task(struct task_struct *t)
2738 {
2739         if (atomic_read(&ftrace_graph_active)) {
2740                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2741                                 * sizeof(struct ftrace_ret_stack),
2742                                 GFP_KERNEL);
2743                 if (!t->ret_stack)
2744                         return;
2745                 t->curr_ret_stack = -1;
2746                 atomic_set(&t->tracing_graph_pause, 0);
2747                 atomic_set(&t->trace_overrun, 0);
2748                 t->ftrace_timestamp = 0;
2749         } else
2750                 t->ret_stack = NULL;
2751 }
2752
2753 void ftrace_graph_exit_task(struct task_struct *t)
2754 {
2755         struct ftrace_ret_stack *ret_stack = t->ret_stack;
2756
2757         t->ret_stack = NULL;
2758         /* NULL must become visible to IRQs before we free it: */
2759         barrier();
2760
2761         kfree(ret_stack);
2762 }
2763
2764 void ftrace_graph_stop(void)
2765 {
2766         ftrace_stop();
2767 }
2768 #endif
2769