Merge branch 'for-linus' of git://neil.brown.name/md
[linux-2.6] / kernel / trace / ftrace.c
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally ported from the -rt patch by:
8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 William Lee Irwin III
14  */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/debugfs.h>
21 #include <linux/hardirq.h>
22 #include <linux/kthread.h>
23 #include <linux/uaccess.h>
24 #include <linux/kprobes.h>
25 #include <linux/ftrace.h>
26 #include <linux/sysctl.h>
27 #include <linux/ctype.h>
28 #include <linux/hash.h>
29 #include <linux/list.h>
30
31 #include <asm/ftrace.h>
32
33 #include "trace.h"
34
35 /* ftrace_enabled is a method to turn ftrace on or off */
36 int ftrace_enabled __read_mostly;
37 static int last_ftrace_enabled;
38
39 /*
40  * ftrace_disabled is set when an anomaly is discovered.
41  * ftrace_disabled is much stronger than ftrace_enabled.
42  */
43 static int ftrace_disabled __read_mostly;
44
45 static DEFINE_SPINLOCK(ftrace_lock);
46 static DEFINE_MUTEX(ftrace_sysctl_lock);
47
48 static struct ftrace_ops ftrace_list_end __read_mostly =
49 {
50         .func = ftrace_stub,
51 };
52
53 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
54 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
55
56 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
57 {
58         struct ftrace_ops *op = ftrace_list;
59
60         /* in case someone actually ports this to alpha! */
61         read_barrier_depends();
62
63         while (op != &ftrace_list_end) {
64                 /* silly alpha */
65                 read_barrier_depends();
66                 op->func(ip, parent_ip);
67                 op = op->next;
68         };
69 }
70
71 /**
72  * clear_ftrace_function - reset the ftrace function
73  *
74  * This NULLs the ftrace function and in essence stops
75  * tracing.  There may be lag
76  */
77 void clear_ftrace_function(void)
78 {
79         ftrace_trace_function = ftrace_stub;
80 }
81
82 static int __register_ftrace_function(struct ftrace_ops *ops)
83 {
84         /* should not be called from interrupt context */
85         spin_lock(&ftrace_lock);
86
87         ops->next = ftrace_list;
88         /*
89          * We are entering ops into the ftrace_list but another
90          * CPU might be walking that list. We need to make sure
91          * the ops->next pointer is valid before another CPU sees
92          * the ops pointer included into the ftrace_list.
93          */
94         smp_wmb();
95         ftrace_list = ops;
96
97         if (ftrace_enabled) {
98                 /*
99                  * For one func, simply call it directly.
100                  * For more than one func, call the chain.
101                  */
102                 if (ops->next == &ftrace_list_end)
103                         ftrace_trace_function = ops->func;
104                 else
105                         ftrace_trace_function = ftrace_list_func;
106         }
107
108         spin_unlock(&ftrace_lock);
109
110         return 0;
111 }
112
113 static int __unregister_ftrace_function(struct ftrace_ops *ops)
114 {
115         struct ftrace_ops **p;
116         int ret = 0;
117
118         /* should not be called from interrupt context */
119         spin_lock(&ftrace_lock);
120
121         /*
122          * If we are removing the last function, then simply point
123          * to the ftrace_stub.
124          */
125         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
126                 ftrace_trace_function = ftrace_stub;
127                 ftrace_list = &ftrace_list_end;
128                 goto out;
129         }
130
131         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
132                 if (*p == ops)
133                         break;
134
135         if (*p != ops) {
136                 ret = -1;
137                 goto out;
138         }
139
140         *p = (*p)->next;
141
142         if (ftrace_enabled) {
143                 /* If we only have one func left, then call that directly */
144                 if (ftrace_list == &ftrace_list_end ||
145                     ftrace_list->next == &ftrace_list_end)
146                         ftrace_trace_function = ftrace_list->func;
147         }
148
149  out:
150         spin_unlock(&ftrace_lock);
151
152         return ret;
153 }
154
155 #ifdef CONFIG_DYNAMIC_FTRACE
156
157 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
158 /*
159  * The hash lock is only needed when the recording of the mcount
160  * callers are dynamic. That is, by the caller themselves and
161  * not recorded via the compilation.
162  */
163 static DEFINE_SPINLOCK(ftrace_hash_lock);
164 #define ftrace_hash_lock(flags)   spin_lock_irqsave(&ftrace_hash_lock, flags)
165 #define ftrace_hash_unlock(flags) \
166                         spin_unlock_irqrestore(&ftrace_hash_lock, flags)
167 #else
168 /* This is protected via the ftrace_lock with MCOUNT_RECORD. */
169 #define ftrace_hash_lock(flags)   do { (void)(flags); } while (0)
170 #define ftrace_hash_unlock(flags) do { } while(0)
171 #endif
172
173 /*
174  * Since MCOUNT_ADDR may point to mcount itself, we do not want
175  * to get it confused by reading a reference in the code as we
176  * are parsing on objcopy output of text. Use a variable for
177  * it instead.
178  */
179 static unsigned long mcount_addr = MCOUNT_ADDR;
180
181 static struct task_struct *ftraced_task;
182
183 enum {
184         FTRACE_ENABLE_CALLS             = (1 << 0),
185         FTRACE_DISABLE_CALLS            = (1 << 1),
186         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
187         FTRACE_ENABLE_MCOUNT            = (1 << 3),
188         FTRACE_DISABLE_MCOUNT           = (1 << 4),
189 };
190
191 static int ftrace_filtered;
192 static int tracing_on;
193 static int frozen_record_count;
194
195 static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
196
197 static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
198
199 static DEFINE_MUTEX(ftraced_lock);
200 static DEFINE_MUTEX(ftrace_regex_lock);
201
202 struct ftrace_page {
203         struct ftrace_page      *next;
204         unsigned long           index;
205         struct dyn_ftrace       records[];
206 };
207
208 #define ENTRIES_PER_PAGE \
209   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
210
211 /* estimate from running different kernels */
212 #define NR_TO_INIT              10000
213
214 static struct ftrace_page       *ftrace_pages_start;
215 static struct ftrace_page       *ftrace_pages;
216
217 static int ftraced_trigger;
218 static int ftraced_suspend;
219 static int ftraced_stop;
220
221 static int ftrace_record_suspend;
222
223 static struct dyn_ftrace *ftrace_free_records;
224
225
226 #ifdef CONFIG_KPROBES
227 static inline void freeze_record(struct dyn_ftrace *rec)
228 {
229         if (!(rec->flags & FTRACE_FL_FROZEN)) {
230                 rec->flags |= FTRACE_FL_FROZEN;
231                 frozen_record_count++;
232         }
233 }
234
235 static inline void unfreeze_record(struct dyn_ftrace *rec)
236 {
237         if (rec->flags & FTRACE_FL_FROZEN) {
238                 rec->flags &= ~FTRACE_FL_FROZEN;
239                 frozen_record_count--;
240         }
241 }
242
243 static inline int record_frozen(struct dyn_ftrace *rec)
244 {
245         return rec->flags & FTRACE_FL_FROZEN;
246 }
247 #else
248 # define freeze_record(rec)                     ({ 0; })
249 # define unfreeze_record(rec)                   ({ 0; })
250 # define record_frozen(rec)                     ({ 0; })
251 #endif /* CONFIG_KPROBES */
252
253 int skip_trace(unsigned long ip)
254 {
255         unsigned long fl;
256         struct dyn_ftrace *rec;
257         struct hlist_node *t;
258         struct hlist_head *head;
259
260         if (frozen_record_count == 0)
261                 return 0;
262
263         head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
264         hlist_for_each_entry_rcu(rec, t, head, node) {
265                 if (rec->ip == ip) {
266                         if (record_frozen(rec)) {
267                                 if (rec->flags & FTRACE_FL_FAILED)
268                                         return 1;
269
270                                 if (!(rec->flags & FTRACE_FL_CONVERTED))
271                                         return 1;
272
273                                 if (!tracing_on || !ftrace_enabled)
274                                         return 1;
275
276                                 if (ftrace_filtered) {
277                                         fl = rec->flags & (FTRACE_FL_FILTER |
278                                                            FTRACE_FL_NOTRACE);
279                                         if (!fl || (fl & FTRACE_FL_NOTRACE))
280                                                 return 1;
281                                 }
282                         }
283                         break;
284                 }
285         }
286
287         return 0;
288 }
289
290 static inline int
291 ftrace_ip_in_hash(unsigned long ip, unsigned long key)
292 {
293         struct dyn_ftrace *p;
294         struct hlist_node *t;
295         int found = 0;
296
297         hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
298                 if (p->ip == ip) {
299                         found = 1;
300                         break;
301                 }
302         }
303
304         return found;
305 }
306
307 static inline void
308 ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
309 {
310         hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
311 }
312
313 /* called from kstop_machine */
314 static inline void ftrace_del_hash(struct dyn_ftrace *node)
315 {
316         hlist_del(&node->node);
317 }
318
319 static void ftrace_free_rec(struct dyn_ftrace *rec)
320 {
321         rec->ip = (unsigned long)ftrace_free_records;
322         ftrace_free_records = rec;
323         rec->flags |= FTRACE_FL_FREE;
324 }
325
326 void ftrace_release(void *start, unsigned long size)
327 {
328         struct dyn_ftrace *rec;
329         struct ftrace_page *pg;
330         unsigned long s = (unsigned long)start;
331         unsigned long e = s + size;
332         int i;
333
334         if (ftrace_disabled || !start)
335                 return;
336
337         /* should not be called from interrupt context */
338         spin_lock(&ftrace_lock);
339
340         for (pg = ftrace_pages_start; pg; pg = pg->next) {
341                 for (i = 0; i < pg->index; i++) {
342                         rec = &pg->records[i];
343
344                         if ((rec->ip >= s) && (rec->ip < e))
345                                 ftrace_free_rec(rec);
346                 }
347         }
348         spin_unlock(&ftrace_lock);
349
350 }
351
352 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
353 {
354         struct dyn_ftrace *rec;
355
356         /* First check for freed records */
357         if (ftrace_free_records) {
358                 rec = ftrace_free_records;
359
360                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
361                         WARN_ON_ONCE(1);
362                         ftrace_free_records = NULL;
363                         ftrace_disabled = 1;
364                         ftrace_enabled = 0;
365                         return NULL;
366                 }
367
368                 ftrace_free_records = (void *)rec->ip;
369                 memset(rec, 0, sizeof(*rec));
370                 return rec;
371         }
372
373         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
374                 if (!ftrace_pages->next)
375                         return NULL;
376                 ftrace_pages = ftrace_pages->next;
377         }
378
379         return &ftrace_pages->records[ftrace_pages->index++];
380 }
381
382 static void
383 ftrace_record_ip(unsigned long ip)
384 {
385         struct dyn_ftrace *node;
386         unsigned long flags;
387         unsigned long key;
388         int resched;
389         int cpu;
390
391         if (!ftrace_enabled || ftrace_disabled)
392                 return;
393
394         resched = need_resched();
395         preempt_disable_notrace();
396
397         /*
398          * We simply need to protect against recursion.
399          * Use the the raw version of smp_processor_id and not
400          * __get_cpu_var which can call debug hooks that can
401          * cause a recursive crash here.
402          */
403         cpu = raw_smp_processor_id();
404         per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
405         if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
406                 goto out;
407
408         if (unlikely(ftrace_record_suspend))
409                 goto out;
410
411         key = hash_long(ip, FTRACE_HASHBITS);
412
413         WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
414
415         if (ftrace_ip_in_hash(ip, key))
416                 goto out;
417
418         ftrace_hash_lock(flags);
419
420         /* This ip may have hit the hash before the lock */
421         if (ftrace_ip_in_hash(ip, key))
422                 goto out_unlock;
423
424         node = ftrace_alloc_dyn_node(ip);
425         if (!node)
426                 goto out_unlock;
427
428         node->ip = ip;
429
430         ftrace_add_hash(node, key);
431
432         ftraced_trigger = 1;
433
434  out_unlock:
435         ftrace_hash_unlock(flags);
436  out:
437         per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
438
439         /* prevent recursion with scheduler */
440         if (resched)
441                 preempt_enable_no_resched_notrace();
442         else
443                 preempt_enable_notrace();
444 }
445
446 #define FTRACE_ADDR ((long)(ftrace_caller))
447
448 static int
449 __ftrace_replace_code(struct dyn_ftrace *rec,
450                       unsigned char *old, unsigned char *new, int enable)
451 {
452         unsigned long ip, fl;
453
454         ip = rec->ip;
455
456         if (ftrace_filtered && enable) {
457                 /*
458                  * If filtering is on:
459                  *
460                  * If this record is set to be filtered and
461                  * is enabled then do nothing.
462                  *
463                  * If this record is set to be filtered and
464                  * it is not enabled, enable it.
465                  *
466                  * If this record is not set to be filtered
467                  * and it is not enabled do nothing.
468                  *
469                  * If this record is set not to trace then
470                  * do nothing.
471                  *
472                  * If this record is set not to trace and
473                  * it is enabled then disable it.
474                  *
475                  * If this record is not set to be filtered and
476                  * it is enabled, disable it.
477                  */
478
479                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
480                                    FTRACE_FL_ENABLED);
481
482                 if ((fl ==  (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
483                     (fl ==  (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
484                     !fl || (fl == FTRACE_FL_NOTRACE))
485                         return 0;
486
487                 /*
488                  * If it is enabled disable it,
489                  * otherwise enable it!
490                  */
491                 if (fl & FTRACE_FL_ENABLED) {
492                         /* swap new and old */
493                         new = old;
494                         old = ftrace_call_replace(ip, FTRACE_ADDR);
495                         rec->flags &= ~FTRACE_FL_ENABLED;
496                 } else {
497                         new = ftrace_call_replace(ip, FTRACE_ADDR);
498                         rec->flags |= FTRACE_FL_ENABLED;
499                 }
500         } else {
501
502                 if (enable) {
503                         /*
504                          * If this record is set not to trace and is
505                          * not enabled, do nothing.
506                          */
507                         fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
508                         if (fl == FTRACE_FL_NOTRACE)
509                                 return 0;
510
511                         new = ftrace_call_replace(ip, FTRACE_ADDR);
512                 } else
513                         old = ftrace_call_replace(ip, FTRACE_ADDR);
514
515                 if (enable) {
516                         if (rec->flags & FTRACE_FL_ENABLED)
517                                 return 0;
518                         rec->flags |= FTRACE_FL_ENABLED;
519                 } else {
520                         if (!(rec->flags & FTRACE_FL_ENABLED))
521                                 return 0;
522                         rec->flags &= ~FTRACE_FL_ENABLED;
523                 }
524         }
525
526         return ftrace_modify_code(ip, old, new);
527 }
528
529 static void ftrace_replace_code(int enable)
530 {
531         int i, failed;
532         unsigned char *new = NULL, *old = NULL;
533         struct dyn_ftrace *rec;
534         struct ftrace_page *pg;
535
536         if (enable)
537                 old = ftrace_nop_replace();
538         else
539                 new = ftrace_nop_replace();
540
541         for (pg = ftrace_pages_start; pg; pg = pg->next) {
542                 for (i = 0; i < pg->index; i++) {
543                         rec = &pg->records[i];
544
545                         /* don't modify code that has already faulted */
546                         if (rec->flags & FTRACE_FL_FAILED)
547                                 continue;
548
549                         /* ignore updates to this record's mcount site */
550                         if (get_kprobe((void *)rec->ip)) {
551                                 freeze_record(rec);
552                                 continue;
553                         } else {
554                                 unfreeze_record(rec);
555                         }
556
557                         failed = __ftrace_replace_code(rec, old, new, enable);
558                         if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
559                                 rec->flags |= FTRACE_FL_FAILED;
560                                 if ((system_state == SYSTEM_BOOTING) ||
561                                     !core_kernel_text(rec->ip)) {
562                                         ftrace_del_hash(rec);
563                                         ftrace_free_rec(rec);
564                                 }
565                         }
566                 }
567         }
568 }
569
570 static void ftrace_shutdown_replenish(void)
571 {
572         if (ftrace_pages->next)
573                 return;
574
575         /* allocate another page */
576         ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
577 }
578
579 static void print_ip_ins(const char *fmt, unsigned char *p)
580 {
581         int i;
582
583         printk(KERN_CONT "%s", fmt);
584
585         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
586                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
587 }
588
589 static int
590 ftrace_code_disable(struct dyn_ftrace *rec)
591 {
592         unsigned long ip;
593         unsigned char *nop, *call;
594         int failed;
595
596         ip = rec->ip;
597
598         nop = ftrace_nop_replace();
599         call = ftrace_call_replace(ip, mcount_addr);
600
601         failed = ftrace_modify_code(ip, call, nop);
602         if (failed) {
603                 switch (failed) {
604                 case 1:
605                         WARN_ON_ONCE(1);
606                         pr_info("ftrace faulted on modifying ");
607                         print_ip_sym(ip);
608                         break;
609                 case 2:
610                         WARN_ON_ONCE(1);
611                         pr_info("ftrace failed to modify ");
612                         print_ip_sym(ip);
613                         print_ip_ins(" expected: ", call);
614                         print_ip_ins(" actual: ", (unsigned char *)ip);
615                         print_ip_ins(" replace: ", nop);
616                         printk(KERN_CONT "\n");
617                         break;
618                 }
619
620                 rec->flags |= FTRACE_FL_FAILED;
621                 return 0;
622         }
623         return 1;
624 }
625
626 static int __ftrace_update_code(void *ignore);
627
628 static int __ftrace_modify_code(void *data)
629 {
630         unsigned long addr;
631         int *command = data;
632
633         if (*command & FTRACE_ENABLE_CALLS) {
634                 /*
635                  * Update any recorded ips now that we have the
636                  * machine stopped
637                  */
638                 __ftrace_update_code(NULL);
639                 ftrace_replace_code(1);
640                 tracing_on = 1;
641         } else if (*command & FTRACE_DISABLE_CALLS) {
642                 ftrace_replace_code(0);
643                 tracing_on = 0;
644         }
645
646         if (*command & FTRACE_UPDATE_TRACE_FUNC)
647                 ftrace_update_ftrace_func(ftrace_trace_function);
648
649         if (*command & FTRACE_ENABLE_MCOUNT) {
650                 addr = (unsigned long)ftrace_record_ip;
651                 ftrace_mcount_set(&addr);
652         } else if (*command & FTRACE_DISABLE_MCOUNT) {
653                 addr = (unsigned long)ftrace_stub;
654                 ftrace_mcount_set(&addr);
655         }
656
657         return 0;
658 }
659
660 static void ftrace_run_update_code(int command)
661 {
662         stop_machine(__ftrace_modify_code, &command, NULL);
663 }
664
665 void ftrace_disable_daemon(void)
666 {
667         /* Stop the daemon from calling kstop_machine */
668         mutex_lock(&ftraced_lock);
669         ftraced_stop = 1;
670         mutex_unlock(&ftraced_lock);
671
672         ftrace_force_update();
673 }
674
675 void ftrace_enable_daemon(void)
676 {
677         mutex_lock(&ftraced_lock);
678         ftraced_stop = 0;
679         mutex_unlock(&ftraced_lock);
680
681         ftrace_force_update();
682 }
683
684 static ftrace_func_t saved_ftrace_func;
685
686 static void ftrace_startup(void)
687 {
688         int command = 0;
689
690         if (unlikely(ftrace_disabled))
691                 return;
692
693         mutex_lock(&ftraced_lock);
694         ftraced_suspend++;
695         if (ftraced_suspend == 1)
696                 command |= FTRACE_ENABLE_CALLS;
697
698         if (saved_ftrace_func != ftrace_trace_function) {
699                 saved_ftrace_func = ftrace_trace_function;
700                 command |= FTRACE_UPDATE_TRACE_FUNC;
701         }
702
703         if (!command || !ftrace_enabled)
704                 goto out;
705
706         ftrace_run_update_code(command);
707  out:
708         mutex_unlock(&ftraced_lock);
709 }
710
711 static void ftrace_shutdown(void)
712 {
713         int command = 0;
714
715         if (unlikely(ftrace_disabled))
716                 return;
717
718         mutex_lock(&ftraced_lock);
719         ftraced_suspend--;
720         if (!ftraced_suspend)
721                 command |= FTRACE_DISABLE_CALLS;
722
723         if (saved_ftrace_func != ftrace_trace_function) {
724                 saved_ftrace_func = ftrace_trace_function;
725                 command |= FTRACE_UPDATE_TRACE_FUNC;
726         }
727
728         if (!command || !ftrace_enabled)
729                 goto out;
730
731         ftrace_run_update_code(command);
732  out:
733         mutex_unlock(&ftraced_lock);
734 }
735
736 static void ftrace_startup_sysctl(void)
737 {
738         int command = FTRACE_ENABLE_MCOUNT;
739
740         if (unlikely(ftrace_disabled))
741                 return;
742
743         mutex_lock(&ftraced_lock);
744         /* Force update next time */
745         saved_ftrace_func = NULL;
746         /* ftraced_suspend is true if we want ftrace running */
747         if (ftraced_suspend)
748                 command |= FTRACE_ENABLE_CALLS;
749
750         ftrace_run_update_code(command);
751         mutex_unlock(&ftraced_lock);
752 }
753
754 static void ftrace_shutdown_sysctl(void)
755 {
756         int command = FTRACE_DISABLE_MCOUNT;
757
758         if (unlikely(ftrace_disabled))
759                 return;
760
761         mutex_lock(&ftraced_lock);
762         /* ftraced_suspend is true if ftrace is running */
763         if (ftraced_suspend)
764                 command |= FTRACE_DISABLE_CALLS;
765
766         ftrace_run_update_code(command);
767         mutex_unlock(&ftraced_lock);
768 }
769
770 static cycle_t          ftrace_update_time;
771 static unsigned long    ftrace_update_cnt;
772 unsigned long           ftrace_update_tot_cnt;
773
774 static int __ftrace_update_code(void *ignore)
775 {
776         int i, save_ftrace_enabled;
777         cycle_t start, stop;
778         struct dyn_ftrace *p;
779         struct hlist_node *t, *n;
780         struct hlist_head *head, temp_list;
781
782         /* Don't be recording funcs now */
783         ftrace_record_suspend++;
784         save_ftrace_enabled = ftrace_enabled;
785         ftrace_enabled = 0;
786
787         start = ftrace_now(raw_smp_processor_id());
788         ftrace_update_cnt = 0;
789
790         /* No locks needed, the machine is stopped! */
791         for (i = 0; i < FTRACE_HASHSIZE; i++) {
792                 INIT_HLIST_HEAD(&temp_list);
793                 head = &ftrace_hash[i];
794
795                 /* all CPUS are stopped, we are safe to modify code */
796                 hlist_for_each_entry_safe(p, t, n, head, node) {
797                         /* Skip over failed records which have not been
798                          * freed. */
799                         if (p->flags & FTRACE_FL_FAILED)
800                                 continue;
801
802                         /* Unconverted records are always at the head of the
803                          * hash bucket. Once we encounter a converted record,
804                          * simply skip over to the next bucket. Saves ftraced
805                          * some processor cycles (ftrace does its bid for
806                          * global warming :-p ). */
807                         if (p->flags & (FTRACE_FL_CONVERTED))
808                                 break;
809
810                         /* Ignore updates to this record's mcount site.
811                          * Reintroduce this record at the head of this
812                          * bucket to attempt to "convert" it again if
813                          * the kprobe on it is unregistered before the
814                          * next run. */
815                         if (get_kprobe((void *)p->ip)) {
816                                 ftrace_del_hash(p);
817                                 INIT_HLIST_NODE(&p->node);
818                                 hlist_add_head(&p->node, &temp_list);
819                                 freeze_record(p);
820                                 continue;
821                         } else {
822                                 unfreeze_record(p);
823                         }
824
825                         /* convert record (i.e, patch mcount-call with NOP) */
826                         if (ftrace_code_disable(p)) {
827                                 p->flags |= FTRACE_FL_CONVERTED;
828                                 ftrace_update_cnt++;
829                         } else {
830                                 if ((system_state == SYSTEM_BOOTING) ||
831                                     !core_kernel_text(p->ip)) {
832                                         ftrace_del_hash(p);
833                                         ftrace_free_rec(p);
834                                 }
835                         }
836                 }
837
838                 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
839                         hlist_del(&p->node);
840                         INIT_HLIST_NODE(&p->node);
841                         hlist_add_head(&p->node, head);
842                 }
843         }
844
845         stop = ftrace_now(raw_smp_processor_id());
846         ftrace_update_time = stop - start;
847         ftrace_update_tot_cnt += ftrace_update_cnt;
848         ftraced_trigger = 0;
849
850         ftrace_enabled = save_ftrace_enabled;
851         ftrace_record_suspend--;
852
853         return 0;
854 }
855
856 static int ftrace_update_code(void)
857 {
858         if (unlikely(ftrace_disabled) ||
859             !ftrace_enabled || !ftraced_trigger)
860                 return 0;
861
862         stop_machine(__ftrace_update_code, NULL, NULL);
863
864         return 1;
865 }
866
867 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
868 {
869         struct ftrace_page *pg;
870         int cnt;
871         int i;
872
873         /* allocate a few pages */
874         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
875         if (!ftrace_pages_start)
876                 return -1;
877
878         /*
879          * Allocate a few more pages.
880          *
881          * TODO: have some parser search vmlinux before
882          *   final linking to find all calls to ftrace.
883          *   Then we can:
884          *    a) know how many pages to allocate.
885          *     and/or
886          *    b) set up the table then.
887          *
888          *  The dynamic code is still necessary for
889          *  modules.
890          */
891
892         pg = ftrace_pages = ftrace_pages_start;
893
894         cnt = num_to_init / ENTRIES_PER_PAGE;
895         pr_info("ftrace: allocating %ld hash entries in %d pages\n",
896                 num_to_init, cnt);
897
898         for (i = 0; i < cnt; i++) {
899                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
900
901                 /* If we fail, we'll try later anyway */
902                 if (!pg->next)
903                         break;
904
905                 pg = pg->next;
906         }
907
908         return 0;
909 }
910
911 enum {
912         FTRACE_ITER_FILTER      = (1 << 0),
913         FTRACE_ITER_CONT        = (1 << 1),
914         FTRACE_ITER_NOTRACE     = (1 << 2),
915         FTRACE_ITER_FAILURES    = (1 << 3),
916 };
917
918 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
919
920 struct ftrace_iterator {
921         loff_t                  pos;
922         struct ftrace_page      *pg;
923         unsigned                idx;
924         unsigned                flags;
925         unsigned char           buffer[FTRACE_BUFF_MAX+1];
926         unsigned                buffer_idx;
927         unsigned                filtered;
928 };
929
930 static void *
931 t_next(struct seq_file *m, void *v, loff_t *pos)
932 {
933         struct ftrace_iterator *iter = m->private;
934         struct dyn_ftrace *rec = NULL;
935
936         (*pos)++;
937
938         /* should not be called from interrupt context */
939         spin_lock(&ftrace_lock);
940  retry:
941         if (iter->idx >= iter->pg->index) {
942                 if (iter->pg->next) {
943                         iter->pg = iter->pg->next;
944                         iter->idx = 0;
945                         goto retry;
946                 }
947         } else {
948                 rec = &iter->pg->records[iter->idx++];
949                 if ((rec->flags & FTRACE_FL_FREE) ||
950
951                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
952                      (rec->flags & FTRACE_FL_FAILED)) ||
953
954                     ((iter->flags & FTRACE_ITER_FAILURES) &&
955                      !(rec->flags & FTRACE_FL_FAILED)) ||
956
957                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
958                      !(rec->flags & FTRACE_FL_NOTRACE))) {
959                         rec = NULL;
960                         goto retry;
961                 }
962         }
963         spin_unlock(&ftrace_lock);
964
965         iter->pos = *pos;
966
967         return rec;
968 }
969
970 static void *t_start(struct seq_file *m, loff_t *pos)
971 {
972         struct ftrace_iterator *iter = m->private;
973         void *p = NULL;
974         loff_t l = -1;
975
976         if (*pos != iter->pos) {
977                 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
978                         ;
979         } else {
980                 l = *pos;
981                 p = t_next(m, p, &l);
982         }
983
984         return p;
985 }
986
987 static void t_stop(struct seq_file *m, void *p)
988 {
989 }
990
991 static int t_show(struct seq_file *m, void *v)
992 {
993         struct dyn_ftrace *rec = v;
994         char str[KSYM_SYMBOL_LEN];
995
996         if (!rec)
997                 return 0;
998
999         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1000
1001         seq_printf(m, "%s\n", str);
1002
1003         return 0;
1004 }
1005
1006 static struct seq_operations show_ftrace_seq_ops = {
1007         .start = t_start,
1008         .next = t_next,
1009         .stop = t_stop,
1010         .show = t_show,
1011 };
1012
1013 static int
1014 ftrace_avail_open(struct inode *inode, struct file *file)
1015 {
1016         struct ftrace_iterator *iter;
1017         int ret;
1018
1019         if (unlikely(ftrace_disabled))
1020                 return -ENODEV;
1021
1022         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1023         if (!iter)
1024                 return -ENOMEM;
1025
1026         iter->pg = ftrace_pages_start;
1027         iter->pos = -1;
1028
1029         ret = seq_open(file, &show_ftrace_seq_ops);
1030         if (!ret) {
1031                 struct seq_file *m = file->private_data;
1032
1033                 m->private = iter;
1034         } else {
1035                 kfree(iter);
1036         }
1037
1038         return ret;
1039 }
1040
1041 int ftrace_avail_release(struct inode *inode, struct file *file)
1042 {
1043         struct seq_file *m = (struct seq_file *)file->private_data;
1044         struct ftrace_iterator *iter = m->private;
1045
1046         seq_release(inode, file);
1047         kfree(iter);
1048
1049         return 0;
1050 }
1051
1052 static int
1053 ftrace_failures_open(struct inode *inode, struct file *file)
1054 {
1055         int ret;
1056         struct seq_file *m;
1057         struct ftrace_iterator *iter;
1058
1059         ret = ftrace_avail_open(inode, file);
1060         if (!ret) {
1061                 m = (struct seq_file *)file->private_data;
1062                 iter = (struct ftrace_iterator *)m->private;
1063                 iter->flags = FTRACE_ITER_FAILURES;
1064         }
1065
1066         return ret;
1067 }
1068
1069
1070 static void ftrace_filter_reset(int enable)
1071 {
1072         struct ftrace_page *pg;
1073         struct dyn_ftrace *rec;
1074         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1075         unsigned i;
1076
1077         /* should not be called from interrupt context */
1078         spin_lock(&ftrace_lock);
1079         if (enable)
1080                 ftrace_filtered = 0;
1081         pg = ftrace_pages_start;
1082         while (pg) {
1083                 for (i = 0; i < pg->index; i++) {
1084                         rec = &pg->records[i];
1085                         if (rec->flags & FTRACE_FL_FAILED)
1086                                 continue;
1087                         rec->flags &= ~type;
1088                 }
1089                 pg = pg->next;
1090         }
1091         spin_unlock(&ftrace_lock);
1092 }
1093
1094 static int
1095 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1096 {
1097         struct ftrace_iterator *iter;
1098         int ret = 0;
1099
1100         if (unlikely(ftrace_disabled))
1101                 return -ENODEV;
1102
1103         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1104         if (!iter)
1105                 return -ENOMEM;
1106
1107         mutex_lock(&ftrace_regex_lock);
1108         if ((file->f_mode & FMODE_WRITE) &&
1109             !(file->f_flags & O_APPEND))
1110                 ftrace_filter_reset(enable);
1111
1112         if (file->f_mode & FMODE_READ) {
1113                 iter->pg = ftrace_pages_start;
1114                 iter->pos = -1;
1115                 iter->flags = enable ? FTRACE_ITER_FILTER :
1116                         FTRACE_ITER_NOTRACE;
1117
1118                 ret = seq_open(file, &show_ftrace_seq_ops);
1119                 if (!ret) {
1120                         struct seq_file *m = file->private_data;
1121                         m->private = iter;
1122                 } else
1123                         kfree(iter);
1124         } else
1125                 file->private_data = iter;
1126         mutex_unlock(&ftrace_regex_lock);
1127
1128         return ret;
1129 }
1130
1131 static int
1132 ftrace_filter_open(struct inode *inode, struct file *file)
1133 {
1134         return ftrace_regex_open(inode, file, 1);
1135 }
1136
1137 static int
1138 ftrace_notrace_open(struct inode *inode, struct file *file)
1139 {
1140         return ftrace_regex_open(inode, file, 0);
1141 }
1142
1143 static ssize_t
1144 ftrace_regex_read(struct file *file, char __user *ubuf,
1145                        size_t cnt, loff_t *ppos)
1146 {
1147         if (file->f_mode & FMODE_READ)
1148                 return seq_read(file, ubuf, cnt, ppos);
1149         else
1150                 return -EPERM;
1151 }
1152
1153 static loff_t
1154 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1155 {
1156         loff_t ret;
1157
1158         if (file->f_mode & FMODE_READ)
1159                 ret = seq_lseek(file, offset, origin);
1160         else
1161                 file->f_pos = ret = 1;
1162
1163         return ret;
1164 }
1165
1166 enum {
1167         MATCH_FULL,
1168         MATCH_FRONT_ONLY,
1169         MATCH_MIDDLE_ONLY,
1170         MATCH_END_ONLY,
1171 };
1172
1173 static void
1174 ftrace_match(unsigned char *buff, int len, int enable)
1175 {
1176         char str[KSYM_SYMBOL_LEN];
1177         char *search = NULL;
1178         struct ftrace_page *pg;
1179         struct dyn_ftrace *rec;
1180         int type = MATCH_FULL;
1181         unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1182         unsigned i, match = 0, search_len = 0;
1183
1184         for (i = 0; i < len; i++) {
1185                 if (buff[i] == '*') {
1186                         if (!i) {
1187                                 search = buff + i + 1;
1188                                 type = MATCH_END_ONLY;
1189                                 search_len = len - (i + 1);
1190                         } else {
1191                                 if (type == MATCH_END_ONLY) {
1192                                         type = MATCH_MIDDLE_ONLY;
1193                                 } else {
1194                                         match = i;
1195                                         type = MATCH_FRONT_ONLY;
1196                                 }
1197                                 buff[i] = 0;
1198                                 break;
1199                         }
1200                 }
1201         }
1202
1203         /* should not be called from interrupt context */
1204         spin_lock(&ftrace_lock);
1205         if (enable)
1206                 ftrace_filtered = 1;
1207         pg = ftrace_pages_start;
1208         while (pg) {
1209                 for (i = 0; i < pg->index; i++) {
1210                         int matched = 0;
1211                         char *ptr;
1212
1213                         rec = &pg->records[i];
1214                         if (rec->flags & FTRACE_FL_FAILED)
1215                                 continue;
1216                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1217                         switch (type) {
1218                         case MATCH_FULL:
1219                                 if (strcmp(str, buff) == 0)
1220                                         matched = 1;
1221                                 break;
1222                         case MATCH_FRONT_ONLY:
1223                                 if (memcmp(str, buff, match) == 0)
1224                                         matched = 1;
1225                                 break;
1226                         case MATCH_MIDDLE_ONLY:
1227                                 if (strstr(str, search))
1228                                         matched = 1;
1229                                 break;
1230                         case MATCH_END_ONLY:
1231                                 ptr = strstr(str, search);
1232                                 if (ptr && (ptr[search_len] == 0))
1233                                         matched = 1;
1234                                 break;
1235                         }
1236                         if (matched)
1237                                 rec->flags |= flag;
1238                 }
1239                 pg = pg->next;
1240         }
1241         spin_unlock(&ftrace_lock);
1242 }
1243
1244 static ssize_t
1245 ftrace_regex_write(struct file *file, const char __user *ubuf,
1246                    size_t cnt, loff_t *ppos, int enable)
1247 {
1248         struct ftrace_iterator *iter;
1249         char ch;
1250         size_t read = 0;
1251         ssize_t ret;
1252
1253         if (!cnt || cnt < 0)
1254                 return 0;
1255
1256         mutex_lock(&ftrace_regex_lock);
1257
1258         if (file->f_mode & FMODE_READ) {
1259                 struct seq_file *m = file->private_data;
1260                 iter = m->private;
1261         } else
1262                 iter = file->private_data;
1263
1264         if (!*ppos) {
1265                 iter->flags &= ~FTRACE_ITER_CONT;
1266                 iter->buffer_idx = 0;
1267         }
1268
1269         ret = get_user(ch, ubuf++);
1270         if (ret)
1271                 goto out;
1272         read++;
1273         cnt--;
1274
1275         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1276                 /* skip white space */
1277                 while (cnt && isspace(ch)) {
1278                         ret = get_user(ch, ubuf++);
1279                         if (ret)
1280                                 goto out;
1281                         read++;
1282                         cnt--;
1283                 }
1284
1285                 if (isspace(ch)) {
1286                         file->f_pos += read;
1287                         ret = read;
1288                         goto out;
1289                 }
1290
1291                 iter->buffer_idx = 0;
1292         }
1293
1294         while (cnt && !isspace(ch)) {
1295                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1296                         iter->buffer[iter->buffer_idx++] = ch;
1297                 else {
1298                         ret = -EINVAL;
1299                         goto out;
1300                 }
1301                 ret = get_user(ch, ubuf++);
1302                 if (ret)
1303                         goto out;
1304                 read++;
1305                 cnt--;
1306         }
1307
1308         if (isspace(ch)) {
1309                 iter->filtered++;
1310                 iter->buffer[iter->buffer_idx] = 0;
1311                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1312                 iter->buffer_idx = 0;
1313         } else
1314                 iter->flags |= FTRACE_ITER_CONT;
1315
1316
1317         file->f_pos += read;
1318
1319         ret = read;
1320  out:
1321         mutex_unlock(&ftrace_regex_lock);
1322
1323         return ret;
1324 }
1325
1326 static ssize_t
1327 ftrace_filter_write(struct file *file, const char __user *ubuf,
1328                     size_t cnt, loff_t *ppos)
1329 {
1330         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1331 }
1332
1333 static ssize_t
1334 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1335                      size_t cnt, loff_t *ppos)
1336 {
1337         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1338 }
1339
1340 static void
1341 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1342 {
1343         if (unlikely(ftrace_disabled))
1344                 return;
1345
1346         mutex_lock(&ftrace_regex_lock);
1347         if (reset)
1348                 ftrace_filter_reset(enable);
1349         if (buf)
1350                 ftrace_match(buf, len, enable);
1351         mutex_unlock(&ftrace_regex_lock);
1352 }
1353
1354 /**
1355  * ftrace_set_filter - set a function to filter on in ftrace
1356  * @buf - the string that holds the function filter text.
1357  * @len - the length of the string.
1358  * @reset - non zero to reset all filters before applying this filter.
1359  *
1360  * Filters denote which functions should be enabled when tracing is enabled.
1361  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1362  */
1363 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1364 {
1365         ftrace_set_regex(buf, len, reset, 1);
1366 }
1367
1368 /**
1369  * ftrace_set_notrace - set a function to not trace in ftrace
1370  * @buf - the string that holds the function notrace text.
1371  * @len - the length of the string.
1372  * @reset - non zero to reset all filters before applying this filter.
1373  *
1374  * Notrace Filters denote which functions should not be enabled when tracing
1375  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1376  * for tracing.
1377  */
1378 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1379 {
1380         ftrace_set_regex(buf, len, reset, 0);
1381 }
1382
1383 static int
1384 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1385 {
1386         struct seq_file *m = (struct seq_file *)file->private_data;
1387         struct ftrace_iterator *iter;
1388
1389         mutex_lock(&ftrace_regex_lock);
1390         if (file->f_mode & FMODE_READ) {
1391                 iter = m->private;
1392
1393                 seq_release(inode, file);
1394         } else
1395                 iter = file->private_data;
1396
1397         if (iter->buffer_idx) {
1398                 iter->filtered++;
1399                 iter->buffer[iter->buffer_idx] = 0;
1400                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1401         }
1402
1403         mutex_lock(&ftrace_sysctl_lock);
1404         mutex_lock(&ftraced_lock);
1405         if (iter->filtered && ftraced_suspend && ftrace_enabled)
1406                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1407         mutex_unlock(&ftraced_lock);
1408         mutex_unlock(&ftrace_sysctl_lock);
1409
1410         kfree(iter);
1411         mutex_unlock(&ftrace_regex_lock);
1412         return 0;
1413 }
1414
1415 static int
1416 ftrace_filter_release(struct inode *inode, struct file *file)
1417 {
1418         return ftrace_regex_release(inode, file, 1);
1419 }
1420
1421 static int
1422 ftrace_notrace_release(struct inode *inode, struct file *file)
1423 {
1424         return ftrace_regex_release(inode, file, 0);
1425 }
1426
1427 static ssize_t
1428 ftraced_read(struct file *filp, char __user *ubuf,
1429                      size_t cnt, loff_t *ppos)
1430 {
1431         /* don't worry about races */
1432         char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1433         int r = strlen(buf);
1434
1435         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1436 }
1437
1438 static ssize_t
1439 ftraced_write(struct file *filp, const char __user *ubuf,
1440                       size_t cnt, loff_t *ppos)
1441 {
1442         char buf[64];
1443         long val;
1444         int ret;
1445
1446         if (cnt >= sizeof(buf))
1447                 return -EINVAL;
1448
1449         if (copy_from_user(&buf, ubuf, cnt))
1450                 return -EFAULT;
1451
1452         if (strncmp(buf, "enable", 6) == 0)
1453                 val = 1;
1454         else if (strncmp(buf, "disable", 7) == 0)
1455                 val = 0;
1456         else {
1457                 buf[cnt] = 0;
1458
1459                 ret = strict_strtoul(buf, 10, &val);
1460                 if (ret < 0)
1461                         return ret;
1462
1463                 val = !!val;
1464         }
1465
1466         if (val)
1467                 ftrace_enable_daemon();
1468         else
1469                 ftrace_disable_daemon();
1470
1471         filp->f_pos += cnt;
1472
1473         return cnt;
1474 }
1475
1476 static struct file_operations ftrace_avail_fops = {
1477         .open = ftrace_avail_open,
1478         .read = seq_read,
1479         .llseek = seq_lseek,
1480         .release = ftrace_avail_release,
1481 };
1482
1483 static struct file_operations ftrace_failures_fops = {
1484         .open = ftrace_failures_open,
1485         .read = seq_read,
1486         .llseek = seq_lseek,
1487         .release = ftrace_avail_release,
1488 };
1489
1490 static struct file_operations ftrace_filter_fops = {
1491         .open = ftrace_filter_open,
1492         .read = ftrace_regex_read,
1493         .write = ftrace_filter_write,
1494         .llseek = ftrace_regex_lseek,
1495         .release = ftrace_filter_release,
1496 };
1497
1498 static struct file_operations ftrace_notrace_fops = {
1499         .open = ftrace_notrace_open,
1500         .read = ftrace_regex_read,
1501         .write = ftrace_notrace_write,
1502         .llseek = ftrace_regex_lseek,
1503         .release = ftrace_notrace_release,
1504 };
1505
1506 static struct file_operations ftraced_fops = {
1507         .open = tracing_open_generic,
1508         .read = ftraced_read,
1509         .write = ftraced_write,
1510 };
1511
1512 /**
1513  * ftrace_force_update - force an update to all recording ftrace functions
1514  */
1515 int ftrace_force_update(void)
1516 {
1517         int ret = 0;
1518
1519         if (unlikely(ftrace_disabled))
1520                 return -ENODEV;
1521
1522         mutex_lock(&ftrace_sysctl_lock);
1523         mutex_lock(&ftraced_lock);
1524
1525         /*
1526          * If ftraced_trigger is not set, then there is nothing
1527          * to update.
1528          */
1529         if (ftraced_trigger && !ftrace_update_code())
1530                 ret = -EBUSY;
1531
1532         mutex_unlock(&ftraced_lock);
1533         mutex_unlock(&ftrace_sysctl_lock);
1534
1535         return ret;
1536 }
1537
1538 static void ftrace_force_shutdown(void)
1539 {
1540         struct task_struct *task;
1541         int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1542
1543         mutex_lock(&ftraced_lock);
1544         task = ftraced_task;
1545         ftraced_task = NULL;
1546         ftraced_suspend = -1;
1547         ftrace_run_update_code(command);
1548         mutex_unlock(&ftraced_lock);
1549
1550         if (task)
1551                 kthread_stop(task);
1552 }
1553
1554 static __init int ftrace_init_debugfs(void)
1555 {
1556         struct dentry *d_tracer;
1557         struct dentry *entry;
1558
1559         d_tracer = tracing_init_dentry();
1560
1561         entry = debugfs_create_file("available_filter_functions", 0444,
1562                                     d_tracer, NULL, &ftrace_avail_fops);
1563         if (!entry)
1564                 pr_warning("Could not create debugfs "
1565                            "'available_filter_functions' entry\n");
1566
1567         entry = debugfs_create_file("failures", 0444,
1568                                     d_tracer, NULL, &ftrace_failures_fops);
1569         if (!entry)
1570                 pr_warning("Could not create debugfs 'failures' entry\n");
1571
1572         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1573                                     NULL, &ftrace_filter_fops);
1574         if (!entry)
1575                 pr_warning("Could not create debugfs "
1576                            "'set_ftrace_filter' entry\n");
1577
1578         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1579                                     NULL, &ftrace_notrace_fops);
1580         if (!entry)
1581                 pr_warning("Could not create debugfs "
1582                            "'set_ftrace_notrace' entry\n");
1583
1584         entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1585                                     NULL, &ftraced_fops);
1586         if (!entry)
1587                 pr_warning("Could not create debugfs "
1588                            "'ftraced_enabled' entry\n");
1589         return 0;
1590 }
1591
1592 fs_initcall(ftrace_init_debugfs);
1593
1594 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
1595 static int ftrace_convert_nops(unsigned long *start,
1596                                unsigned long *end)
1597 {
1598         unsigned long *p;
1599         unsigned long addr;
1600         unsigned long flags;
1601
1602         p = start;
1603         while (p < end) {
1604                 addr = ftrace_call_adjust(*p++);
1605                 /* should not be called from interrupt context */
1606                 spin_lock(&ftrace_lock);
1607                 ftrace_record_ip(addr);
1608                 spin_unlock(&ftrace_lock);
1609                 ftrace_shutdown_replenish();
1610         }
1611
1612         /* p is ignored */
1613         local_irq_save(flags);
1614         __ftrace_update_code(p);
1615         local_irq_restore(flags);
1616
1617         return 0;
1618 }
1619
1620 void ftrace_init_module(unsigned long *start, unsigned long *end)
1621 {
1622         if (ftrace_disabled || start == end)
1623                 return;
1624         ftrace_convert_nops(start, end);
1625 }
1626
1627 extern unsigned long __start_mcount_loc[];
1628 extern unsigned long __stop_mcount_loc[];
1629
1630 void __init ftrace_init(void)
1631 {
1632         unsigned long count, addr, flags;
1633         int ret;
1634
1635         /* Keep the ftrace pointer to the stub */
1636         addr = (unsigned long)ftrace_stub;
1637
1638         local_irq_save(flags);
1639         ftrace_dyn_arch_init(&addr);
1640         local_irq_restore(flags);
1641
1642         /* ftrace_dyn_arch_init places the return code in addr */
1643         if (addr)
1644                 goto failed;
1645
1646         count = __stop_mcount_loc - __start_mcount_loc;
1647
1648         ret = ftrace_dyn_table_alloc(count);
1649         if (ret)
1650                 goto failed;
1651
1652         last_ftrace_enabled = ftrace_enabled = 1;
1653
1654         ret = ftrace_convert_nops(__start_mcount_loc,
1655                                   __stop_mcount_loc);
1656
1657         return;
1658  failed:
1659         ftrace_disabled = 1;
1660 }
1661 #else /* CONFIG_FTRACE_MCOUNT_RECORD */
1662 static int ftraced(void *ignore)
1663 {
1664         unsigned long usecs;
1665
1666         while (!kthread_should_stop()) {
1667
1668                 set_current_state(TASK_INTERRUPTIBLE);
1669
1670                 /* check once a second */
1671                 schedule_timeout(HZ);
1672
1673                 if (unlikely(ftrace_disabled))
1674                         continue;
1675
1676                 mutex_lock(&ftrace_sysctl_lock);
1677                 mutex_lock(&ftraced_lock);
1678                 if (!ftraced_suspend && !ftraced_stop &&
1679                     ftrace_update_code()) {
1680                         usecs = nsecs_to_usecs(ftrace_update_time);
1681                         if (ftrace_update_tot_cnt > 100000) {
1682                                 ftrace_update_tot_cnt = 0;
1683                                 pr_info("hm, dftrace overflow: %lu change%s"
1684                                         " (%lu total) in %lu usec%s\n",
1685                                         ftrace_update_cnt,
1686                                         ftrace_update_cnt != 1 ? "s" : "",
1687                                         ftrace_update_tot_cnt,
1688                                         usecs, usecs != 1 ? "s" : "");
1689                                 ftrace_disabled = 1;
1690                                 WARN_ON_ONCE(1);
1691                         }
1692                 }
1693                 mutex_unlock(&ftraced_lock);
1694                 mutex_unlock(&ftrace_sysctl_lock);
1695
1696                 ftrace_shutdown_replenish();
1697         }
1698         __set_current_state(TASK_RUNNING);
1699         return 0;
1700 }
1701
1702 static int __init ftrace_dynamic_init(void)
1703 {
1704         struct task_struct *p;
1705         unsigned long addr;
1706         int ret;
1707
1708         addr = (unsigned long)ftrace_record_ip;
1709
1710         stop_machine(ftrace_dyn_arch_init, &addr, NULL);
1711
1712         /* ftrace_dyn_arch_init places the return code in addr */
1713         if (addr) {
1714                 ret = (int)addr;
1715                 goto failed;
1716         }
1717
1718         ret = ftrace_dyn_table_alloc(NR_TO_INIT);
1719         if (ret)
1720                 goto failed;
1721
1722         p = kthread_run(ftraced, NULL, "ftraced");
1723         if (IS_ERR(p)) {
1724                 ret = -1;
1725                 goto failed;
1726         }
1727
1728         last_ftrace_enabled = ftrace_enabled = 1;
1729         ftraced_task = p;
1730
1731         return 0;
1732
1733  failed:
1734         ftrace_disabled = 1;
1735         return ret;
1736 }
1737
1738 core_initcall(ftrace_dynamic_init);
1739 #endif /* CONFIG_FTRACE_MCOUNT_RECORD */
1740
1741 #else
1742 # define ftrace_startup()               do { } while (0)
1743 # define ftrace_shutdown()              do { } while (0)
1744 # define ftrace_startup_sysctl()        do { } while (0)
1745 # define ftrace_shutdown_sysctl()       do { } while (0)
1746 # define ftrace_force_shutdown()        do { } while (0)
1747 #endif /* CONFIG_DYNAMIC_FTRACE */
1748
1749 /**
1750  * ftrace_kill_atomic - kill ftrace from critical sections
1751  *
1752  * This function should be used by panic code. It stops ftrace
1753  * but in a not so nice way. If you need to simply kill ftrace
1754  * from a non-atomic section, use ftrace_kill.
1755  */
1756 void ftrace_kill_atomic(void)
1757 {
1758         ftrace_disabled = 1;
1759         ftrace_enabled = 0;
1760 #ifdef CONFIG_DYNAMIC_FTRACE
1761         ftraced_suspend = -1;
1762 #endif
1763         clear_ftrace_function();
1764 }
1765
1766 /**
1767  * ftrace_kill - totally shutdown ftrace
1768  *
1769  * This is a safety measure. If something was detected that seems
1770  * wrong, calling this function will keep ftrace from doing
1771  * any more modifications, and updates.
1772  * used when something went wrong.
1773  */
1774 void ftrace_kill(void)
1775 {
1776         mutex_lock(&ftrace_sysctl_lock);
1777         ftrace_disabled = 1;
1778         ftrace_enabled = 0;
1779
1780         clear_ftrace_function();
1781         mutex_unlock(&ftrace_sysctl_lock);
1782
1783         /* Try to totally disable ftrace */
1784         ftrace_force_shutdown();
1785 }
1786
1787 /**
1788  * register_ftrace_function - register a function for profiling
1789  * @ops - ops structure that holds the function for profiling.
1790  *
1791  * Register a function to be called by all functions in the
1792  * kernel.
1793  *
1794  * Note: @ops->func and all the functions it calls must be labeled
1795  *       with "notrace", otherwise it will go into a
1796  *       recursive loop.
1797  */
1798 int register_ftrace_function(struct ftrace_ops *ops)
1799 {
1800         int ret;
1801
1802         if (unlikely(ftrace_disabled))
1803                 return -1;
1804
1805         mutex_lock(&ftrace_sysctl_lock);
1806         ret = __register_ftrace_function(ops);
1807         ftrace_startup();
1808         mutex_unlock(&ftrace_sysctl_lock);
1809
1810         return ret;
1811 }
1812
1813 /**
1814  * unregister_ftrace_function - unresgister a function for profiling.
1815  * @ops - ops structure that holds the function to unregister
1816  *
1817  * Unregister a function that was added to be called by ftrace profiling.
1818  */
1819 int unregister_ftrace_function(struct ftrace_ops *ops)
1820 {
1821         int ret;
1822
1823         mutex_lock(&ftrace_sysctl_lock);
1824         ret = __unregister_ftrace_function(ops);
1825         ftrace_shutdown();
1826         mutex_unlock(&ftrace_sysctl_lock);
1827
1828         return ret;
1829 }
1830
1831 int
1832 ftrace_enable_sysctl(struct ctl_table *table, int write,
1833                      struct file *file, void __user *buffer, size_t *lenp,
1834                      loff_t *ppos)
1835 {
1836         int ret;
1837
1838         if (unlikely(ftrace_disabled))
1839                 return -ENODEV;
1840
1841         mutex_lock(&ftrace_sysctl_lock);
1842
1843         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
1844
1845         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1846                 goto out;
1847
1848         last_ftrace_enabled = ftrace_enabled;
1849
1850         if (ftrace_enabled) {
1851
1852                 ftrace_startup_sysctl();
1853
1854                 /* we are starting ftrace again */
1855                 if (ftrace_list != &ftrace_list_end) {
1856                         if (ftrace_list->next == &ftrace_list_end)
1857                                 ftrace_trace_function = ftrace_list->func;
1858                         else
1859                                 ftrace_trace_function = ftrace_list_func;
1860                 }
1861
1862         } else {
1863                 /* stopping ftrace calls (just send to ftrace_stub) */
1864                 ftrace_trace_function = ftrace_stub;
1865
1866                 ftrace_shutdown_sysctl();
1867         }
1868
1869  out:
1870         mutex_unlock(&ftrace_sysctl_lock);
1871         return ret;
1872 }