Merge branch 'link_removal' of git://www.jni.nu/cris
[linux-2.6] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/utsrelease.h>
15 #include <linux/kallsyms.h>
16 #include <linux/seq_file.h>
17 #include <linux/notifier.h>
18 #include <linux/debugfs.h>
19 #include <linux/pagemap.h>
20 #include <linux/hardirq.h>
21 #include <linux/linkage.h>
22 #include <linux/uaccess.h>
23 #include <linux/ftrace.h>
24 #include <linux/module.h>
25 #include <linux/percpu.h>
26 #include <linux/kdebug.h>
27 #include <linux/ctype.h>
28 #include <linux/init.h>
29 #include <linux/poll.h>
30 #include <linux/gfp.h>
31 #include <linux/fs.h>
32 #include <linux/kprobes.h>
33 #include <linux/writeback.h>
34
35 #include <linux/stacktrace.h>
36 #include <linux/ring_buffer.h>
37 #include <linux/irqflags.h>
38
39 #include "trace.h"
40
41 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
42
43 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
44 unsigned long __read_mostly     tracing_thresh;
45
46 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
47
48 static inline void ftrace_disable_cpu(void)
49 {
50         preempt_disable();
51         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
52 }
53
54 static inline void ftrace_enable_cpu(void)
55 {
56         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
57         preempt_enable();
58 }
59
60 static cpumask_t __read_mostly          tracing_buffer_mask;
61
62 #define for_each_tracing_cpu(cpu)       \
63         for_each_cpu_mask(cpu, tracing_buffer_mask)
64
65 static int tracing_disabled = 1;
66
67 long
68 ns2usecs(cycle_t nsec)
69 {
70         nsec += 500;
71         do_div(nsec, 1000);
72         return nsec;
73 }
74
75 cycle_t ftrace_now(int cpu)
76 {
77         u64 ts = ring_buffer_time_stamp(cpu);
78         ring_buffer_normalize_time_stamp(cpu, &ts);
79         return ts;
80 }
81
82 /*
83  * The global_trace is the descriptor that holds the tracing
84  * buffers for the live tracing. For each CPU, it contains
85  * a link list of pages that will store trace entries. The
86  * page descriptor of the pages in the memory is used to hold
87  * the link list by linking the lru item in the page descriptor
88  * to each of the pages in the buffer per CPU.
89  *
90  * For each active CPU there is a data field that holds the
91  * pages for the buffer for that CPU. Each CPU has the same number
92  * of pages allocated for its buffer.
93  */
94 static struct trace_array       global_trace;
95
96 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
97
98 /*
99  * The max_tr is used to snapshot the global_trace when a maximum
100  * latency is reached. Some tracers will use this to store a maximum
101  * trace while it continues examining live traces.
102  *
103  * The buffers for the max_tr are set up the same as the global_trace.
104  * When a snapshot is taken, the link list of the max_tr is swapped
105  * with the link list of the global_trace and the buffers are reset for
106  * the global_trace so the tracing can continue.
107  */
108 static struct trace_array       max_tr;
109
110 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
111
112 /* tracer_enabled is used to toggle activation of a tracer */
113 static int                      tracer_enabled = 1;
114
115 /* function tracing enabled */
116 int                             ftrace_function_enabled;
117
118 /*
119  * trace_buf_size is the size in bytes that is allocated
120  * for a buffer. Note, the number of bytes is always rounded
121  * to page size.
122  *
123  * This number is purposely set to a low number of 16384.
124  * If the dump on oops happens, it will be much appreciated
125  * to not have to wait for all that output. Anyway this can be
126  * boot time and run time configurable.
127  */
128 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
129
130 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
131
132 /* trace_types holds a link list of available tracers. */
133 static struct tracer            *trace_types __read_mostly;
134
135 /* current_trace points to the tracer that is currently active */
136 static struct tracer            *current_trace __read_mostly;
137
138 /*
139  * max_tracer_type_len is used to simplify the allocating of
140  * buffers to read userspace tracer names. We keep track of
141  * the longest tracer name registered.
142  */
143 static int                      max_tracer_type_len;
144
145 /*
146  * trace_types_lock is used to protect the trace_types list.
147  * This lock is also used to keep user access serialized.
148  * Accesses from userspace will grab this lock while userspace
149  * activities happen inside the kernel.
150  */
151 static DEFINE_MUTEX(trace_types_lock);
152
153 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
154 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
155
156 /* trace_flags holds iter_ctrl options */
157 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
158
159 /**
160  * trace_wake_up - wake up tasks waiting for trace input
161  *
162  * Simply wakes up any task that is blocked on the trace_wait
163  * queue. These is used with trace_poll for tasks polling the trace.
164  */
165 void trace_wake_up(void)
166 {
167         /*
168          * The runqueue_is_locked() can fail, but this is the best we
169          * have for now:
170          */
171         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
172                 wake_up(&trace_wait);
173 }
174
175 static int __init set_buf_size(char *str)
176 {
177         unsigned long buf_size;
178         int ret;
179
180         if (!str)
181                 return 0;
182         ret = strict_strtoul(str, 0, &buf_size);
183         /* nr_entries can not be zero */
184         if (ret < 0 || buf_size == 0)
185                 return 0;
186         trace_buf_size = buf_size;
187         return 1;
188 }
189 __setup("trace_buf_size=", set_buf_size);
190
191 unsigned long nsecs_to_usecs(unsigned long nsecs)
192 {
193         return nsecs / 1000;
194 }
195
196 /*
197  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
198  * control the output of kernel symbols.
199  */
200 #define TRACE_ITER_SYM_MASK \
201         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
202
203 /* These must match the bit postions in trace_iterator_flags */
204 static const char *trace_options[] = {
205         "print-parent",
206         "sym-offset",
207         "sym-addr",
208         "verbose",
209         "raw",
210         "hex",
211         "bin",
212         "block",
213         "stacktrace",
214         "sched-tree",
215         "ftrace_printk",
216         NULL
217 };
218
219 /*
220  * ftrace_max_lock is used to protect the swapping of buffers
221  * when taking a max snapshot. The buffers themselves are
222  * protected by per_cpu spinlocks. But the action of the swap
223  * needs its own lock.
224  *
225  * This is defined as a raw_spinlock_t in order to help
226  * with performance when lockdep debugging is enabled.
227  */
228 static raw_spinlock_t ftrace_max_lock =
229         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
230
231 /*
232  * Copy the new maximum trace into the separate maximum-trace
233  * structure. (this way the maximum trace is permanently saved,
234  * for later retrieval via /debugfs/tracing/latency_trace)
235  */
236 static void
237 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
238 {
239         struct trace_array_cpu *data = tr->data[cpu];
240
241         max_tr.cpu = cpu;
242         max_tr.time_start = data->preempt_timestamp;
243
244         data = max_tr.data[cpu];
245         data->saved_latency = tracing_max_latency;
246
247         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
248         data->pid = tsk->pid;
249         data->uid = tsk->uid;
250         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
251         data->policy = tsk->policy;
252         data->rt_priority = tsk->rt_priority;
253
254         /* record this tasks comm */
255         tracing_record_cmdline(current);
256 }
257
258 /**
259  * trace_seq_printf - sequence printing of trace information
260  * @s: trace sequence descriptor
261  * @fmt: printf format string
262  *
263  * The tracer may use either sequence operations or its own
264  * copy to user routines. To simplify formating of a trace
265  * trace_seq_printf is used to store strings into a special
266  * buffer (@s). Then the output may be either used by
267  * the sequencer or pulled into another buffer.
268  */
269 int
270 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
271 {
272         int len = (PAGE_SIZE - 1) - s->len;
273         va_list ap;
274         int ret;
275
276         if (!len)
277                 return 0;
278
279         va_start(ap, fmt);
280         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
281         va_end(ap);
282
283         /* If we can't write it all, don't bother writing anything */
284         if (ret >= len)
285                 return 0;
286
287         s->len += ret;
288
289         return len;
290 }
291
292 /**
293  * trace_seq_puts - trace sequence printing of simple string
294  * @s: trace sequence descriptor
295  * @str: simple string to record
296  *
297  * The tracer may use either the sequence operations or its own
298  * copy to user routines. This function records a simple string
299  * into a special buffer (@s) for later retrieval by a sequencer
300  * or other mechanism.
301  */
302 static int
303 trace_seq_puts(struct trace_seq *s, const char *str)
304 {
305         int len = strlen(str);
306
307         if (len > ((PAGE_SIZE - 1) - s->len))
308                 return 0;
309
310         memcpy(s->buffer + s->len, str, len);
311         s->len += len;
312
313         return len;
314 }
315
316 static int
317 trace_seq_putc(struct trace_seq *s, unsigned char c)
318 {
319         if (s->len >= (PAGE_SIZE - 1))
320                 return 0;
321
322         s->buffer[s->len++] = c;
323
324         return 1;
325 }
326
327 static int
328 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
329 {
330         if (len > ((PAGE_SIZE - 1) - s->len))
331                 return 0;
332
333         memcpy(s->buffer + s->len, mem, len);
334         s->len += len;
335
336         return len;
337 }
338
339 #define MAX_MEMHEX_BYTES        8
340 #define HEX_CHARS               (MAX_MEMHEX_BYTES*2 + 1)
341
342 static int
343 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
344 {
345         unsigned char hex[HEX_CHARS];
346         unsigned char *data = mem;
347         int i, j;
348
349 #ifdef __BIG_ENDIAN
350         for (i = 0, j = 0; i < len; i++) {
351 #else
352         for (i = len-1, j = 0; i >= 0; i--) {
353 #endif
354                 hex[j++] = hex_asc_hi(data[i]);
355                 hex[j++] = hex_asc_lo(data[i]);
356         }
357         hex[j++] = ' ';
358
359         return trace_seq_putmem(s, hex, j);
360 }
361
362 static void
363 trace_seq_reset(struct trace_seq *s)
364 {
365         s->len = 0;
366         s->readpos = 0;
367 }
368
369 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
370 {
371         int len;
372         int ret;
373
374         if (s->len <= s->readpos)
375                 return -EBUSY;
376
377         len = s->len - s->readpos;
378         if (cnt > len)
379                 cnt = len;
380         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
381         if (ret)
382                 return -EFAULT;
383
384         s->readpos += len;
385         return cnt;
386 }
387
388 static void
389 trace_print_seq(struct seq_file *m, struct trace_seq *s)
390 {
391         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
392
393         s->buffer[len] = 0;
394         seq_puts(m, s->buffer);
395
396         trace_seq_reset(s);
397 }
398
399 /**
400  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
401  * @tr: tracer
402  * @tsk: the task with the latency
403  * @cpu: The cpu that initiated the trace.
404  *
405  * Flip the buffers between the @tr and the max_tr and record information
406  * about which task was the cause of this latency.
407  */
408 void
409 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
410 {
411         struct ring_buffer *buf = tr->buffer;
412
413         WARN_ON_ONCE(!irqs_disabled());
414         __raw_spin_lock(&ftrace_max_lock);
415
416         tr->buffer = max_tr.buffer;
417         max_tr.buffer = buf;
418
419         ftrace_disable_cpu();
420         ring_buffer_reset(tr->buffer);
421         ftrace_enable_cpu();
422
423         __update_max_tr(tr, tsk, cpu);
424         __raw_spin_unlock(&ftrace_max_lock);
425 }
426
427 /**
428  * update_max_tr_single - only copy one trace over, and reset the rest
429  * @tr - tracer
430  * @tsk - task with the latency
431  * @cpu - the cpu of the buffer to copy.
432  *
433  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
434  */
435 void
436 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
437 {
438         int ret;
439
440         WARN_ON_ONCE(!irqs_disabled());
441         __raw_spin_lock(&ftrace_max_lock);
442
443         ftrace_disable_cpu();
444
445         ring_buffer_reset(max_tr.buffer);
446         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
447
448         ftrace_enable_cpu();
449
450         WARN_ON_ONCE(ret);
451
452         __update_max_tr(tr, tsk, cpu);
453         __raw_spin_unlock(&ftrace_max_lock);
454 }
455
456 /**
457  * register_tracer - register a tracer with the ftrace system.
458  * @type - the plugin for the tracer
459  *
460  * Register a new plugin tracer.
461  */
462 int register_tracer(struct tracer *type)
463 {
464         struct tracer *t;
465         int len;
466         int ret = 0;
467
468         if (!type->name) {
469                 pr_info("Tracer must have a name\n");
470                 return -1;
471         }
472
473         mutex_lock(&trace_types_lock);
474         for (t = trace_types; t; t = t->next) {
475                 if (strcmp(type->name, t->name) == 0) {
476                         /* already found */
477                         pr_info("Trace %s already registered\n",
478                                 type->name);
479                         ret = -1;
480                         goto out;
481                 }
482         }
483
484 #ifdef CONFIG_FTRACE_STARTUP_TEST
485         if (type->selftest) {
486                 struct tracer *saved_tracer = current_trace;
487                 struct trace_array *tr = &global_trace;
488                 int saved_ctrl = tr->ctrl;
489                 int i;
490                 /*
491                  * Run a selftest on this tracer.
492                  * Here we reset the trace buffer, and set the current
493                  * tracer to be this tracer. The tracer can then run some
494                  * internal tracing to verify that everything is in order.
495                  * If we fail, we do not register this tracer.
496                  */
497                 for_each_tracing_cpu(i) {
498                         tracing_reset(tr, i);
499                 }
500                 current_trace = type;
501                 tr->ctrl = 0;
502                 /* the test is responsible for initializing and enabling */
503                 pr_info("Testing tracer %s: ", type->name);
504                 ret = type->selftest(type, tr);
505                 /* the test is responsible for resetting too */
506                 current_trace = saved_tracer;
507                 tr->ctrl = saved_ctrl;
508                 if (ret) {
509                         printk(KERN_CONT "FAILED!\n");
510                         goto out;
511                 }
512                 /* Only reset on passing, to avoid touching corrupted buffers */
513                 for_each_tracing_cpu(i) {
514                         tracing_reset(tr, i);
515                 }
516                 printk(KERN_CONT "PASSED\n");
517         }
518 #endif
519
520         type->next = trace_types;
521         trace_types = type;
522         len = strlen(type->name);
523         if (len > max_tracer_type_len)
524                 max_tracer_type_len = len;
525
526  out:
527         mutex_unlock(&trace_types_lock);
528
529         return ret;
530 }
531
532 void unregister_tracer(struct tracer *type)
533 {
534         struct tracer **t;
535         int len;
536
537         mutex_lock(&trace_types_lock);
538         for (t = &trace_types; *t; t = &(*t)->next) {
539                 if (*t == type)
540                         goto found;
541         }
542         pr_info("Trace %s not registered\n", type->name);
543         goto out;
544
545  found:
546         *t = (*t)->next;
547         if (strlen(type->name) != max_tracer_type_len)
548                 goto out;
549
550         max_tracer_type_len = 0;
551         for (t = &trace_types; *t; t = &(*t)->next) {
552                 len = strlen((*t)->name);
553                 if (len > max_tracer_type_len)
554                         max_tracer_type_len = len;
555         }
556  out:
557         mutex_unlock(&trace_types_lock);
558 }
559
560 void tracing_reset(struct trace_array *tr, int cpu)
561 {
562         ftrace_disable_cpu();
563         ring_buffer_reset_cpu(tr->buffer, cpu);
564         ftrace_enable_cpu();
565 }
566
567 #define SAVED_CMDLINES 128
568 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
569 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
570 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
571 static int cmdline_idx;
572 static DEFINE_SPINLOCK(trace_cmdline_lock);
573
574 /* temporary disable recording */
575 atomic_t trace_record_cmdline_disabled __read_mostly;
576
577 static void trace_init_cmdlines(void)
578 {
579         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
580         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
581         cmdline_idx = 0;
582 }
583
584 void trace_stop_cmdline_recording(void);
585
586 static void trace_save_cmdline(struct task_struct *tsk)
587 {
588         unsigned map;
589         unsigned idx;
590
591         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
592                 return;
593
594         /*
595          * It's not the end of the world if we don't get
596          * the lock, but we also don't want to spin
597          * nor do we want to disable interrupts,
598          * so if we miss here, then better luck next time.
599          */
600         if (!spin_trylock(&trace_cmdline_lock))
601                 return;
602
603         idx = map_pid_to_cmdline[tsk->pid];
604         if (idx >= SAVED_CMDLINES) {
605                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
606
607                 map = map_cmdline_to_pid[idx];
608                 if (map <= PID_MAX_DEFAULT)
609                         map_pid_to_cmdline[map] = (unsigned)-1;
610
611                 map_pid_to_cmdline[tsk->pid] = idx;
612
613                 cmdline_idx = idx;
614         }
615
616         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
617
618         spin_unlock(&trace_cmdline_lock);
619 }
620
621 static char *trace_find_cmdline(int pid)
622 {
623         char *cmdline = "<...>";
624         unsigned map;
625
626         if (!pid)
627                 return "<idle>";
628
629         if (pid > PID_MAX_DEFAULT)
630                 goto out;
631
632         map = map_pid_to_cmdline[pid];
633         if (map >= SAVED_CMDLINES)
634                 goto out;
635
636         cmdline = saved_cmdlines[map];
637
638  out:
639         return cmdline;
640 }
641
642 void tracing_record_cmdline(struct task_struct *tsk)
643 {
644         if (atomic_read(&trace_record_cmdline_disabled))
645                 return;
646
647         trace_save_cmdline(tsk);
648 }
649
650 void
651 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
652                              int pc)
653 {
654         struct task_struct *tsk = current;
655
656         entry->preempt_count            = pc & 0xff;
657         entry->pid                      = (tsk) ? tsk->pid : 0;
658         entry->flags =
659 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
660                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
661 #else
662                 TRACE_FLAG_IRQS_NOSUPPORT |
663 #endif
664                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
665                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
666                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
667 }
668
669 void
670 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
671                unsigned long ip, unsigned long parent_ip, unsigned long flags,
672                int pc)
673 {
674         struct ring_buffer_event *event;
675         struct ftrace_entry *entry;
676         unsigned long irq_flags;
677
678         /* If we are reading the ring buffer, don't trace */
679         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
680                 return;
681
682         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
683                                          &irq_flags);
684         if (!event)
685                 return;
686         entry   = ring_buffer_event_data(event);
687         tracing_generic_entry_update(&entry->ent, flags, pc);
688         entry->ent.type                 = TRACE_FN;
689         entry->ip                       = ip;
690         entry->parent_ip                = parent_ip;
691         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
692 }
693
694 void
695 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
696        unsigned long ip, unsigned long parent_ip, unsigned long flags,
697        int pc)
698 {
699         if (likely(!atomic_read(&data->disabled)))
700                 trace_function(tr, data, ip, parent_ip, flags, pc);
701 }
702
703 static void ftrace_trace_stack(struct trace_array *tr,
704                                struct trace_array_cpu *data,
705                                unsigned long flags,
706                                int skip, int pc)
707 {
708         struct ring_buffer_event *event;
709         struct stack_entry *entry;
710         struct stack_trace trace;
711         unsigned long irq_flags;
712
713         if (!(trace_flags & TRACE_ITER_STACKTRACE))
714                 return;
715
716         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
717                                          &irq_flags);
718         if (!event)
719                 return;
720         entry   = ring_buffer_event_data(event);
721         tracing_generic_entry_update(&entry->ent, flags, pc);
722         entry->ent.type         = TRACE_STACK;
723
724         memset(&entry->caller, 0, sizeof(entry->caller));
725
726         trace.nr_entries        = 0;
727         trace.max_entries       = FTRACE_STACK_ENTRIES;
728         trace.skip              = skip;
729         trace.entries           = entry->caller;
730
731         save_stack_trace(&trace);
732         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
733 }
734
735 void __trace_stack(struct trace_array *tr,
736                    struct trace_array_cpu *data,
737                    unsigned long flags,
738                    int skip)
739 {
740         ftrace_trace_stack(tr, data, flags, skip, preempt_count());
741 }
742
743 static void
744 ftrace_trace_special(void *__tr, void *__data,
745                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
746                      int pc)
747 {
748         struct ring_buffer_event *event;
749         struct trace_array_cpu *data = __data;
750         struct trace_array *tr = __tr;
751         struct special_entry *entry;
752         unsigned long irq_flags;
753
754         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
755                                          &irq_flags);
756         if (!event)
757                 return;
758         entry   = ring_buffer_event_data(event);
759         tracing_generic_entry_update(&entry->ent, 0, pc);
760         entry->ent.type                 = TRACE_SPECIAL;
761         entry->arg1                     = arg1;
762         entry->arg2                     = arg2;
763         entry->arg3                     = arg3;
764         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
765         ftrace_trace_stack(tr, data, irq_flags, 4, pc);
766
767         trace_wake_up();
768 }
769
770 void
771 __trace_special(void *__tr, void *__data,
772                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
773 {
774         ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
775 }
776
777 void
778 tracing_sched_switch_trace(struct trace_array *tr,
779                            struct trace_array_cpu *data,
780                            struct task_struct *prev,
781                            struct task_struct *next,
782                            unsigned long flags, int pc)
783 {
784         struct ring_buffer_event *event;
785         struct ctx_switch_entry *entry;
786         unsigned long irq_flags;
787
788         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
789                                            &irq_flags);
790         if (!event)
791                 return;
792         entry   = ring_buffer_event_data(event);
793         tracing_generic_entry_update(&entry->ent, flags, pc);
794         entry->ent.type                 = TRACE_CTX;
795         entry->prev_pid                 = prev->pid;
796         entry->prev_prio                = prev->prio;
797         entry->prev_state               = prev->state;
798         entry->next_pid                 = next->pid;
799         entry->next_prio                = next->prio;
800         entry->next_state               = next->state;
801         entry->next_cpu = task_cpu(next);
802         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
803         ftrace_trace_stack(tr, data, flags, 5, pc);
804 }
805
806 void
807 tracing_sched_wakeup_trace(struct trace_array *tr,
808                            struct trace_array_cpu *data,
809                            struct task_struct *wakee,
810                            struct task_struct *curr,
811                            unsigned long flags, int pc)
812 {
813         struct ring_buffer_event *event;
814         struct ctx_switch_entry *entry;
815         unsigned long irq_flags;
816
817         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
818                                            &irq_flags);
819         if (!event)
820                 return;
821         entry   = ring_buffer_event_data(event);
822         tracing_generic_entry_update(&entry->ent, flags, pc);
823         entry->ent.type                 = TRACE_WAKE;
824         entry->prev_pid                 = curr->pid;
825         entry->prev_prio                = curr->prio;
826         entry->prev_state               = curr->state;
827         entry->next_pid                 = wakee->pid;
828         entry->next_prio                = wakee->prio;
829         entry->next_state               = wakee->state;
830         entry->next_cpu                 = task_cpu(wakee);
831         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
832         ftrace_trace_stack(tr, data, flags, 6, pc);
833
834         trace_wake_up();
835 }
836
837 void
838 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
839 {
840         struct trace_array *tr = &global_trace;
841         struct trace_array_cpu *data;
842         int cpu;
843         int pc;
844
845         if (tracing_disabled || !tr->ctrl)
846                 return;
847
848         pc = preempt_count();
849         preempt_disable_notrace();
850         cpu = raw_smp_processor_id();
851         data = tr->data[cpu];
852
853         if (likely(!atomic_read(&data->disabled)))
854                 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
855
856         preempt_enable_notrace();
857 }
858
859 #ifdef CONFIG_FUNCTION_TRACER
860 static void
861 function_trace_call(unsigned long ip, unsigned long parent_ip)
862 {
863         struct trace_array *tr = &global_trace;
864         struct trace_array_cpu *data;
865         unsigned long flags;
866         long disabled;
867         int cpu, resched;
868         int pc;
869
870         if (unlikely(!ftrace_function_enabled))
871                 return;
872
873         pc = preempt_count();
874         resched = need_resched();
875         preempt_disable_notrace();
876         local_save_flags(flags);
877         cpu = raw_smp_processor_id();
878         data = tr->data[cpu];
879         disabled = atomic_inc_return(&data->disabled);
880
881         if (likely(disabled == 1))
882                 trace_function(tr, data, ip, parent_ip, flags, pc);
883
884         atomic_dec(&data->disabled);
885         if (resched)
886                 preempt_enable_no_resched_notrace();
887         else
888                 preempt_enable_notrace();
889 }
890
891 static struct ftrace_ops trace_ops __read_mostly =
892 {
893         .func = function_trace_call,
894 };
895
896 void tracing_start_function_trace(void)
897 {
898         ftrace_function_enabled = 0;
899         register_ftrace_function(&trace_ops);
900         if (tracer_enabled)
901                 ftrace_function_enabled = 1;
902 }
903
904 void tracing_stop_function_trace(void)
905 {
906         ftrace_function_enabled = 0;
907         unregister_ftrace_function(&trace_ops);
908 }
909 #endif
910
911 enum trace_file_type {
912         TRACE_FILE_LAT_FMT      = 1,
913 };
914
915 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
916 {
917         /* Don't allow ftrace to trace into the ring buffers */
918         ftrace_disable_cpu();
919
920         iter->idx++;
921         if (iter->buffer_iter[iter->cpu])
922                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
923
924         ftrace_enable_cpu();
925 }
926
927 static struct trace_entry *
928 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
929 {
930         struct ring_buffer_event *event;
931         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
932
933         /* Don't allow ftrace to trace into the ring buffers */
934         ftrace_disable_cpu();
935
936         if (buf_iter)
937                 event = ring_buffer_iter_peek(buf_iter, ts);
938         else
939                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
940
941         ftrace_enable_cpu();
942
943         return event ? ring_buffer_event_data(event) : NULL;
944 }
945
946 static struct trace_entry *
947 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
948 {
949         struct ring_buffer *buffer = iter->tr->buffer;
950         struct trace_entry *ent, *next = NULL;
951         u64 next_ts = 0, ts;
952         int next_cpu = -1;
953         int cpu;
954
955         for_each_tracing_cpu(cpu) {
956
957                 if (ring_buffer_empty_cpu(buffer, cpu))
958                         continue;
959
960                 ent = peek_next_entry(iter, cpu, &ts);
961
962                 /*
963                  * Pick the entry with the smallest timestamp:
964                  */
965                 if (ent && (!next || ts < next_ts)) {
966                         next = ent;
967                         next_cpu = cpu;
968                         next_ts = ts;
969                 }
970         }
971
972         if (ent_cpu)
973                 *ent_cpu = next_cpu;
974
975         if (ent_ts)
976                 *ent_ts = next_ts;
977
978         return next;
979 }
980
981 /* Find the next real entry, without updating the iterator itself */
982 static struct trace_entry *
983 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
984 {
985         return __find_next_entry(iter, ent_cpu, ent_ts);
986 }
987
988 /* Find the next real entry, and increment the iterator to the next entry */
989 static void *find_next_entry_inc(struct trace_iterator *iter)
990 {
991         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
992
993         if (iter->ent)
994                 trace_iterator_increment(iter, iter->cpu);
995
996         return iter->ent ? iter : NULL;
997 }
998
999 static void trace_consume(struct trace_iterator *iter)
1000 {
1001         /* Don't allow ftrace to trace into the ring buffers */
1002         ftrace_disable_cpu();
1003         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1004         ftrace_enable_cpu();
1005 }
1006
1007 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1008 {
1009         struct trace_iterator *iter = m->private;
1010         int i = (int)*pos;
1011         void *ent;
1012
1013         (*pos)++;
1014
1015         /* can't go backwards */
1016         if (iter->idx > i)
1017                 return NULL;
1018
1019         if (iter->idx < 0)
1020                 ent = find_next_entry_inc(iter);
1021         else
1022                 ent = iter;
1023
1024         while (ent && iter->idx < i)
1025                 ent = find_next_entry_inc(iter);
1026
1027         iter->pos = *pos;
1028
1029         return ent;
1030 }
1031
1032 static void *s_start(struct seq_file *m, loff_t *pos)
1033 {
1034         struct trace_iterator *iter = m->private;
1035         void *p = NULL;
1036         loff_t l = 0;
1037         int cpu;
1038
1039         mutex_lock(&trace_types_lock);
1040
1041         if (!current_trace || current_trace != iter->trace) {
1042                 mutex_unlock(&trace_types_lock);
1043                 return NULL;
1044         }
1045
1046         atomic_inc(&trace_record_cmdline_disabled);
1047
1048         /* let the tracer grab locks here if needed */
1049         if (current_trace->start)
1050                 current_trace->start(iter);
1051
1052         if (*pos != iter->pos) {
1053                 iter->ent = NULL;
1054                 iter->cpu = 0;
1055                 iter->idx = -1;
1056
1057                 ftrace_disable_cpu();
1058
1059                 for_each_tracing_cpu(cpu) {
1060                         ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1061                 }
1062
1063                 ftrace_enable_cpu();
1064
1065                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1066                         ;
1067
1068         } else {
1069                 l = *pos - 1;
1070                 p = s_next(m, p, &l);
1071         }
1072
1073         return p;
1074 }
1075
1076 static void s_stop(struct seq_file *m, void *p)
1077 {
1078         struct trace_iterator *iter = m->private;
1079
1080         atomic_dec(&trace_record_cmdline_disabled);
1081
1082         /* let the tracer release locks here if needed */
1083         if (current_trace && current_trace == iter->trace && iter->trace->stop)
1084                 iter->trace->stop(iter);
1085
1086         mutex_unlock(&trace_types_lock);
1087 }
1088
1089 #define KRETPROBE_MSG "[unknown/kretprobe'd]"
1090
1091 #ifdef CONFIG_KRETPROBES
1092 static inline int kretprobed(unsigned long addr)
1093 {
1094         return addr == (unsigned long)kretprobe_trampoline;
1095 }
1096 #else
1097 static inline int kretprobed(unsigned long addr)
1098 {
1099         return 0;
1100 }
1101 #endif /* CONFIG_KRETPROBES */
1102
1103 static int
1104 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1105 {
1106 #ifdef CONFIG_KALLSYMS
1107         char str[KSYM_SYMBOL_LEN];
1108
1109         kallsyms_lookup(address, NULL, NULL, NULL, str);
1110
1111         return trace_seq_printf(s, fmt, str);
1112 #endif
1113         return 1;
1114 }
1115
1116 static int
1117 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1118                      unsigned long address)
1119 {
1120 #ifdef CONFIG_KALLSYMS
1121         char str[KSYM_SYMBOL_LEN];
1122
1123         sprint_symbol(str, address);
1124         return trace_seq_printf(s, fmt, str);
1125 #endif
1126         return 1;
1127 }
1128
1129 #ifndef CONFIG_64BIT
1130 # define IP_FMT "%08lx"
1131 #else
1132 # define IP_FMT "%016lx"
1133 #endif
1134
1135 static int
1136 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1137 {
1138         int ret;
1139
1140         if (!ip)
1141                 return trace_seq_printf(s, "0");
1142
1143         if (sym_flags & TRACE_ITER_SYM_OFFSET)
1144                 ret = seq_print_sym_offset(s, "%s", ip);
1145         else
1146                 ret = seq_print_sym_short(s, "%s", ip);
1147
1148         if (!ret)
1149                 return 0;
1150
1151         if (sym_flags & TRACE_ITER_SYM_ADDR)
1152                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1153         return ret;
1154 }
1155
1156 static void print_lat_help_header(struct seq_file *m)
1157 {
1158         seq_puts(m, "#                  _------=> CPU#            \n");
1159         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1160         seq_puts(m, "#                | / _----=> need-resched    \n");
1161         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1162         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1163         seq_puts(m, "#                |||| /                      \n");
1164         seq_puts(m, "#                |||||     delay             \n");
1165         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1166         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1167 }
1168
1169 static void print_func_help_header(struct seq_file *m)
1170 {
1171         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1172         seq_puts(m, "#              | |       |          |         |\n");
1173 }
1174
1175
1176 static void
1177 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1178 {
1179         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1180         struct trace_array *tr = iter->tr;
1181         struct trace_array_cpu *data = tr->data[tr->cpu];
1182         struct tracer *type = current_trace;
1183         unsigned long total;
1184         unsigned long entries;
1185         const char *name = "preemption";
1186
1187         if (type)
1188                 name = type->name;
1189
1190         entries = ring_buffer_entries(iter->tr->buffer);
1191         total = entries +
1192                 ring_buffer_overruns(iter->tr->buffer);
1193
1194         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1195                    name, UTS_RELEASE);
1196         seq_puts(m, "-----------------------------------"
1197                  "---------------------------------\n");
1198         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1199                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1200                    nsecs_to_usecs(data->saved_latency),
1201                    entries,
1202                    total,
1203                    tr->cpu,
1204 #if defined(CONFIG_PREEMPT_NONE)
1205                    "server",
1206 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1207                    "desktop",
1208 #elif defined(CONFIG_PREEMPT)
1209                    "preempt",
1210 #else
1211                    "unknown",
1212 #endif
1213                    /* These are reserved for later use */
1214                    0, 0, 0, 0);
1215 #ifdef CONFIG_SMP
1216         seq_printf(m, " #P:%d)\n", num_online_cpus());
1217 #else
1218         seq_puts(m, ")\n");
1219 #endif
1220         seq_puts(m, "    -----------------\n");
1221         seq_printf(m, "    | task: %.16s-%d "
1222                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1223                    data->comm, data->pid, data->uid, data->nice,
1224                    data->policy, data->rt_priority);
1225         seq_puts(m, "    -----------------\n");
1226
1227         if (data->critical_start) {
1228                 seq_puts(m, " => started at: ");
1229                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1230                 trace_print_seq(m, &iter->seq);
1231                 seq_puts(m, "\n => ended at:   ");
1232                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1233                 trace_print_seq(m, &iter->seq);
1234                 seq_puts(m, "\n");
1235         }
1236
1237         seq_puts(m, "\n");
1238 }
1239
1240 static void
1241 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1242 {
1243         int hardirq, softirq;
1244         char *comm;
1245
1246         comm = trace_find_cmdline(entry->pid);
1247
1248         trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1249         trace_seq_printf(s, "%3d", cpu);
1250         trace_seq_printf(s, "%c%c",
1251                         (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1252                          (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1253                         ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1254
1255         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1256         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1257         if (hardirq && softirq) {
1258                 trace_seq_putc(s, 'H');
1259         } else {
1260                 if (hardirq) {
1261                         trace_seq_putc(s, 'h');
1262                 } else {
1263                         if (softirq)
1264                                 trace_seq_putc(s, 's');
1265                         else
1266                                 trace_seq_putc(s, '.');
1267                 }
1268         }
1269
1270         if (entry->preempt_count)
1271                 trace_seq_printf(s, "%x", entry->preempt_count);
1272         else
1273                 trace_seq_puts(s, ".");
1274 }
1275
1276 unsigned long preempt_mark_thresh = 100;
1277
1278 static void
1279 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1280                     unsigned long rel_usecs)
1281 {
1282         trace_seq_printf(s, " %4lldus", abs_usecs);
1283         if (rel_usecs > preempt_mark_thresh)
1284                 trace_seq_puts(s, "!: ");
1285         else if (rel_usecs > 1)
1286                 trace_seq_puts(s, "+: ");
1287         else
1288                 trace_seq_puts(s, " : ");
1289 }
1290
1291 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1292
1293 /*
1294  * The message is supposed to contain an ending newline.
1295  * If the printing stops prematurely, try to add a newline of our own.
1296  */
1297 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1298 {
1299         struct trace_entry *ent;
1300         struct trace_field_cont *cont;
1301         bool ok = true;
1302
1303         ent = peek_next_entry(iter, iter->cpu, NULL);
1304         if (!ent || ent->type != TRACE_CONT) {
1305                 trace_seq_putc(s, '\n');
1306                 return;
1307         }
1308
1309         do {
1310                 cont = (struct trace_field_cont *)ent;
1311                 if (ok)
1312                         ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1313
1314                 ftrace_disable_cpu();
1315
1316                 if (iter->buffer_iter[iter->cpu])
1317                         ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1318                 else
1319                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1320
1321                 ftrace_enable_cpu();
1322
1323                 ent = peek_next_entry(iter, iter->cpu, NULL);
1324         } while (ent && ent->type == TRACE_CONT);
1325
1326         if (!ok)
1327                 trace_seq_putc(s, '\n');
1328 }
1329
1330 static enum print_line_t
1331 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1332 {
1333         struct trace_seq *s = &iter->seq;
1334         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1335         struct trace_entry *next_entry;
1336         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1337         struct trace_entry *entry = iter->ent;
1338         unsigned long abs_usecs;
1339         unsigned long rel_usecs;
1340         u64 next_ts;
1341         char *comm;
1342         int S, T;
1343         int i;
1344         unsigned state;
1345
1346         if (entry->type == TRACE_CONT)
1347                 return TRACE_TYPE_HANDLED;
1348
1349         next_entry = find_next_entry(iter, NULL, &next_ts);
1350         if (!next_entry)
1351                 next_ts = iter->ts;
1352         rel_usecs = ns2usecs(next_ts - iter->ts);
1353         abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1354
1355         if (verbose) {
1356                 comm = trace_find_cmdline(entry->pid);
1357                 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1358                                  " %ld.%03ldms (+%ld.%03ldms): ",
1359                                  comm,
1360                                  entry->pid, cpu, entry->flags,
1361                                  entry->preempt_count, trace_idx,
1362                                  ns2usecs(iter->ts),
1363                                  abs_usecs/1000,
1364                                  abs_usecs % 1000, rel_usecs/1000,
1365                                  rel_usecs % 1000);
1366         } else {
1367                 lat_print_generic(s, entry, cpu);
1368                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1369         }
1370         switch (entry->type) {
1371         case TRACE_FN: {
1372                 struct ftrace_entry *field;
1373
1374                 trace_assign_type(field, entry);
1375
1376                 seq_print_ip_sym(s, field->ip, sym_flags);
1377                 trace_seq_puts(s, " (");
1378                 if (kretprobed(field->parent_ip))
1379                         trace_seq_puts(s, KRETPROBE_MSG);
1380                 else
1381                         seq_print_ip_sym(s, field->parent_ip, sym_flags);
1382                 trace_seq_puts(s, ")\n");
1383                 break;
1384         }
1385         case TRACE_CTX:
1386         case TRACE_WAKE: {
1387                 struct ctx_switch_entry *field;
1388
1389                 trace_assign_type(field, entry);
1390
1391                 T = field->next_state < sizeof(state_to_char) ?
1392                         state_to_char[field->next_state] : 'X';
1393
1394                 state = field->prev_state ?
1395                         __ffs(field->prev_state) + 1 : 0;
1396                 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1397                 comm = trace_find_cmdline(field->next_pid);
1398                 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1399                                  field->prev_pid,
1400                                  field->prev_prio,
1401                                  S, entry->type == TRACE_CTX ? "==>" : "  +",
1402                                  field->next_cpu,
1403                                  field->next_pid,
1404                                  field->next_prio,
1405                                  T, comm);
1406                 break;
1407         }
1408         case TRACE_SPECIAL: {
1409                 struct special_entry *field;
1410
1411                 trace_assign_type(field, entry);
1412
1413                 trace_seq_printf(s, "# %ld %ld %ld\n",
1414                                  field->arg1,
1415                                  field->arg2,
1416                                  field->arg3);
1417                 break;
1418         }
1419         case TRACE_STACK: {
1420                 struct stack_entry *field;
1421
1422                 trace_assign_type(field, entry);
1423
1424                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1425                         if (i)
1426                                 trace_seq_puts(s, " <= ");
1427                         seq_print_ip_sym(s, field->caller[i], sym_flags);
1428                 }
1429                 trace_seq_puts(s, "\n");
1430                 break;
1431         }
1432         case TRACE_PRINT: {
1433                 struct print_entry *field;
1434
1435                 trace_assign_type(field, entry);
1436
1437                 seq_print_ip_sym(s, field->ip, sym_flags);
1438                 trace_seq_printf(s, ": %s", field->buf);
1439                 if (entry->flags & TRACE_FLAG_CONT)
1440                         trace_seq_print_cont(s, iter);
1441                 break;
1442         }
1443         default:
1444                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1445         }
1446         return TRACE_TYPE_HANDLED;
1447 }
1448
1449 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1450 {
1451         struct trace_seq *s = &iter->seq;
1452         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1453         struct trace_entry *entry;
1454         unsigned long usec_rem;
1455         unsigned long long t;
1456         unsigned long secs;
1457         char *comm;
1458         int ret;
1459         int S, T;
1460         int i;
1461
1462         entry = iter->ent;
1463
1464         if (entry->type == TRACE_CONT)
1465                 return TRACE_TYPE_HANDLED;
1466
1467         comm = trace_find_cmdline(iter->ent->pid);
1468
1469         t = ns2usecs(iter->ts);
1470         usec_rem = do_div(t, 1000000ULL);
1471         secs = (unsigned long)t;
1472
1473         ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1474         if (!ret)
1475                 return TRACE_TYPE_PARTIAL_LINE;
1476         ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1477         if (!ret)
1478                 return TRACE_TYPE_PARTIAL_LINE;
1479         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1480         if (!ret)
1481                 return TRACE_TYPE_PARTIAL_LINE;
1482
1483         switch (entry->type) {
1484         case TRACE_FN: {
1485                 struct ftrace_entry *field;
1486
1487                 trace_assign_type(field, entry);
1488
1489                 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1490                 if (!ret)
1491                         return TRACE_TYPE_PARTIAL_LINE;
1492                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1493                                                 field->parent_ip) {
1494                         ret = trace_seq_printf(s, " <-");
1495                         if (!ret)
1496                                 return TRACE_TYPE_PARTIAL_LINE;
1497                         if (kretprobed(field->parent_ip))
1498                                 ret = trace_seq_puts(s, KRETPROBE_MSG);
1499                         else
1500                                 ret = seq_print_ip_sym(s,
1501                                                        field->parent_ip,
1502                                                        sym_flags);
1503                         if (!ret)
1504                                 return TRACE_TYPE_PARTIAL_LINE;
1505                 }
1506                 ret = trace_seq_printf(s, "\n");
1507                 if (!ret)
1508                         return TRACE_TYPE_PARTIAL_LINE;
1509                 break;
1510         }
1511         case TRACE_CTX:
1512         case TRACE_WAKE: {
1513                 struct ctx_switch_entry *field;
1514
1515                 trace_assign_type(field, entry);
1516
1517                 S = field->prev_state < sizeof(state_to_char) ?
1518                         state_to_char[field->prev_state] : 'X';
1519                 T = field->next_state < sizeof(state_to_char) ?
1520                         state_to_char[field->next_state] : 'X';
1521                 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1522                                        field->prev_pid,
1523                                        field->prev_prio,
1524                                        S,
1525                                        entry->type == TRACE_CTX ? "==>" : "  +",
1526                                        field->next_cpu,
1527                                        field->next_pid,
1528                                        field->next_prio,
1529                                        T);
1530                 if (!ret)
1531                         return TRACE_TYPE_PARTIAL_LINE;
1532                 break;
1533         }
1534         case TRACE_SPECIAL: {
1535                 struct special_entry *field;
1536
1537                 trace_assign_type(field, entry);
1538
1539                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1540                                  field->arg1,
1541                                  field->arg2,
1542                                  field->arg3);
1543                 if (!ret)
1544                         return TRACE_TYPE_PARTIAL_LINE;
1545                 break;
1546         }
1547         case TRACE_STACK: {
1548                 struct stack_entry *field;
1549
1550                 trace_assign_type(field, entry);
1551
1552                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1553                         if (i) {
1554                                 ret = trace_seq_puts(s, " <= ");
1555                                 if (!ret)
1556                                         return TRACE_TYPE_PARTIAL_LINE;
1557                         }
1558                         ret = seq_print_ip_sym(s, field->caller[i],
1559                                                sym_flags);
1560                         if (!ret)
1561                                 return TRACE_TYPE_PARTIAL_LINE;
1562                 }
1563                 ret = trace_seq_puts(s, "\n");
1564                 if (!ret)
1565                         return TRACE_TYPE_PARTIAL_LINE;
1566                 break;
1567         }
1568         case TRACE_PRINT: {
1569                 struct print_entry *field;
1570
1571                 trace_assign_type(field, entry);
1572
1573                 seq_print_ip_sym(s, field->ip, sym_flags);
1574                 trace_seq_printf(s, ": %s", field->buf);
1575                 if (entry->flags & TRACE_FLAG_CONT)
1576                         trace_seq_print_cont(s, iter);
1577                 break;
1578         }
1579         }
1580         return TRACE_TYPE_HANDLED;
1581 }
1582
1583 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1584 {
1585         struct trace_seq *s = &iter->seq;
1586         struct trace_entry *entry;
1587         int ret;
1588         int S, T;
1589
1590         entry = iter->ent;
1591
1592         if (entry->type == TRACE_CONT)
1593                 return TRACE_TYPE_HANDLED;
1594
1595         ret = trace_seq_printf(s, "%d %d %llu ",
1596                 entry->pid, iter->cpu, iter->ts);
1597         if (!ret)
1598                 return TRACE_TYPE_PARTIAL_LINE;
1599
1600         switch (entry->type) {
1601         case TRACE_FN: {
1602                 struct ftrace_entry *field;
1603
1604                 trace_assign_type(field, entry);
1605
1606                 ret = trace_seq_printf(s, "%x %x\n",
1607                                         field->ip,
1608                                         field->parent_ip);
1609                 if (!ret)
1610                         return TRACE_TYPE_PARTIAL_LINE;
1611                 break;
1612         }
1613         case TRACE_CTX:
1614         case TRACE_WAKE: {
1615                 struct ctx_switch_entry *field;
1616
1617                 trace_assign_type(field, entry);
1618
1619                 S = field->prev_state < sizeof(state_to_char) ?
1620                         state_to_char[field->prev_state] : 'X';
1621                 T = field->next_state < sizeof(state_to_char) ?
1622                         state_to_char[field->next_state] : 'X';
1623                 if (entry->type == TRACE_WAKE)
1624                         S = '+';
1625                 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1626                                        field->prev_pid,
1627                                        field->prev_prio,
1628                                        S,
1629                                        field->next_cpu,
1630                                        field->next_pid,
1631                                        field->next_prio,
1632                                        T);
1633                 if (!ret)
1634                         return TRACE_TYPE_PARTIAL_LINE;
1635                 break;
1636         }
1637         case TRACE_SPECIAL:
1638         case TRACE_STACK: {
1639                 struct special_entry *field;
1640
1641                 trace_assign_type(field, entry);
1642
1643                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1644                                  field->arg1,
1645                                  field->arg2,
1646                                  field->arg3);
1647                 if (!ret)
1648                         return TRACE_TYPE_PARTIAL_LINE;
1649                 break;
1650         }
1651         case TRACE_PRINT: {
1652                 struct print_entry *field;
1653
1654                 trace_assign_type(field, entry);
1655
1656                 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1657                 if (entry->flags & TRACE_FLAG_CONT)
1658                         trace_seq_print_cont(s, iter);
1659                 break;
1660         }
1661         }
1662         return TRACE_TYPE_HANDLED;
1663 }
1664
1665 #define SEQ_PUT_FIELD_RET(s, x)                         \
1666 do {                                                    \
1667         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1668                 return 0;                               \
1669 } while (0)
1670
1671 #define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
1672 do {                                                    \
1673         BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES);     \
1674         if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
1675                 return 0;                               \
1676 } while (0)
1677
1678 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1679 {
1680         struct trace_seq *s = &iter->seq;
1681         unsigned char newline = '\n';
1682         struct trace_entry *entry;
1683         int S, T;
1684
1685         entry = iter->ent;
1686
1687         if (entry->type == TRACE_CONT)
1688                 return TRACE_TYPE_HANDLED;
1689
1690         SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1691         SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1692         SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1693
1694         switch (entry->type) {
1695         case TRACE_FN: {
1696                 struct ftrace_entry *field;
1697
1698                 trace_assign_type(field, entry);
1699
1700                 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1701                 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1702                 break;
1703         }
1704         case TRACE_CTX:
1705         case TRACE_WAKE: {
1706                 struct ctx_switch_entry *field;
1707
1708                 trace_assign_type(field, entry);
1709
1710                 S = field->prev_state < sizeof(state_to_char) ?
1711                         state_to_char[field->prev_state] : 'X';
1712                 T = field->next_state < sizeof(state_to_char) ?
1713                         state_to_char[field->next_state] : 'X';
1714                 if (entry->type == TRACE_WAKE)
1715                         S = '+';
1716                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1717                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1718                 SEQ_PUT_HEX_FIELD_RET(s, S);
1719                 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1720                 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1721                 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1722                 SEQ_PUT_HEX_FIELD_RET(s, T);
1723                 break;
1724         }
1725         case TRACE_SPECIAL:
1726         case TRACE_STACK: {
1727                 struct special_entry *field;
1728
1729                 trace_assign_type(field, entry);
1730
1731                 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1732                 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1733                 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1734                 break;
1735         }
1736         }
1737         SEQ_PUT_FIELD_RET(s, newline);
1738
1739         return TRACE_TYPE_HANDLED;
1740 }
1741
1742 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1743 {
1744         struct trace_seq *s = &iter->seq;
1745         struct trace_entry *entry;
1746
1747         entry = iter->ent;
1748
1749         if (entry->type == TRACE_CONT)
1750                 return TRACE_TYPE_HANDLED;
1751
1752         SEQ_PUT_FIELD_RET(s, entry->pid);
1753         SEQ_PUT_FIELD_RET(s, iter->cpu);
1754         SEQ_PUT_FIELD_RET(s, iter->ts);
1755
1756         switch (entry->type) {
1757         case TRACE_FN: {
1758                 struct ftrace_entry *field;
1759
1760                 trace_assign_type(field, entry);
1761
1762                 SEQ_PUT_FIELD_RET(s, field->ip);
1763                 SEQ_PUT_FIELD_RET(s, field->parent_ip);
1764                 break;
1765         }
1766         case TRACE_CTX: {
1767                 struct ctx_switch_entry *field;
1768
1769                 trace_assign_type(field, entry);
1770
1771                 SEQ_PUT_FIELD_RET(s, field->prev_pid);
1772                 SEQ_PUT_FIELD_RET(s, field->prev_prio);
1773                 SEQ_PUT_FIELD_RET(s, field->prev_state);
1774                 SEQ_PUT_FIELD_RET(s, field->next_pid);
1775                 SEQ_PUT_FIELD_RET(s, field->next_prio);
1776                 SEQ_PUT_FIELD_RET(s, field->next_state);
1777                 break;
1778         }
1779         case TRACE_SPECIAL:
1780         case TRACE_STACK: {
1781                 struct special_entry *field;
1782
1783                 trace_assign_type(field, entry);
1784
1785                 SEQ_PUT_FIELD_RET(s, field->arg1);
1786                 SEQ_PUT_FIELD_RET(s, field->arg2);
1787                 SEQ_PUT_FIELD_RET(s, field->arg3);
1788                 break;
1789         }
1790         }
1791         return 1;
1792 }
1793
1794 static int trace_empty(struct trace_iterator *iter)
1795 {
1796         int cpu;
1797
1798         for_each_tracing_cpu(cpu) {
1799                 if (iter->buffer_iter[cpu]) {
1800                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1801                                 return 0;
1802                 } else {
1803                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1804                                 return 0;
1805                 }
1806         }
1807
1808         return 1;
1809 }
1810
1811 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1812 {
1813         enum print_line_t ret;
1814
1815         if (iter->trace && iter->trace->print_line) {
1816                 ret = iter->trace->print_line(iter);
1817                 if (ret != TRACE_TYPE_UNHANDLED)
1818                         return ret;
1819         }
1820
1821         if (trace_flags & TRACE_ITER_BIN)
1822                 return print_bin_fmt(iter);
1823
1824         if (trace_flags & TRACE_ITER_HEX)
1825                 return print_hex_fmt(iter);
1826
1827         if (trace_flags & TRACE_ITER_RAW)
1828                 return print_raw_fmt(iter);
1829
1830         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1831                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1832
1833         return print_trace_fmt(iter);
1834 }
1835
1836 static int s_show(struct seq_file *m, void *v)
1837 {
1838         struct trace_iterator *iter = v;
1839
1840         if (iter->ent == NULL) {
1841                 if (iter->tr) {
1842                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1843                         seq_puts(m, "#\n");
1844                 }
1845                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1846                         /* print nothing if the buffers are empty */
1847                         if (trace_empty(iter))
1848                                 return 0;
1849                         print_trace_header(m, iter);
1850                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1851                                 print_lat_help_header(m);
1852                 } else {
1853                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1854                                 print_func_help_header(m);
1855                 }
1856         } else {
1857                 print_trace_line(iter);
1858                 trace_print_seq(m, &iter->seq);
1859         }
1860
1861         return 0;
1862 }
1863
1864 static struct seq_operations tracer_seq_ops = {
1865         .start          = s_start,
1866         .next           = s_next,
1867         .stop           = s_stop,
1868         .show           = s_show,
1869 };
1870
1871 static struct trace_iterator *
1872 __tracing_open(struct inode *inode, struct file *file, int *ret)
1873 {
1874         struct trace_iterator *iter;
1875         struct seq_file *m;
1876         int cpu;
1877
1878         if (tracing_disabled) {
1879                 *ret = -ENODEV;
1880                 return NULL;
1881         }
1882
1883         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1884         if (!iter) {
1885                 *ret = -ENOMEM;
1886                 goto out;
1887         }
1888
1889         mutex_lock(&trace_types_lock);
1890         if (current_trace && current_trace->print_max)
1891                 iter->tr = &max_tr;
1892         else
1893                 iter->tr = inode->i_private;
1894         iter->trace = current_trace;
1895         iter->pos = -1;
1896
1897         for_each_tracing_cpu(cpu) {
1898
1899                 iter->buffer_iter[cpu] =
1900                         ring_buffer_read_start(iter->tr->buffer, cpu);
1901
1902                 if (!iter->buffer_iter[cpu])
1903                         goto fail_buffer;
1904         }
1905
1906         /* TODO stop tracer */
1907         *ret = seq_open(file, &tracer_seq_ops);
1908         if (*ret)
1909                 goto fail_buffer;
1910
1911         m = file->private_data;
1912         m->private = iter;
1913
1914         /* stop the trace while dumping */
1915         if (iter->tr->ctrl) {
1916                 tracer_enabled = 0;
1917                 ftrace_function_enabled = 0;
1918         }
1919
1920         if (iter->trace && iter->trace->open)
1921                         iter->trace->open(iter);
1922
1923         mutex_unlock(&trace_types_lock);
1924
1925  out:
1926         return iter;
1927
1928  fail_buffer:
1929         for_each_tracing_cpu(cpu) {
1930                 if (iter->buffer_iter[cpu])
1931                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1932         }
1933         mutex_unlock(&trace_types_lock);
1934
1935         return ERR_PTR(-ENOMEM);
1936 }
1937
1938 int tracing_open_generic(struct inode *inode, struct file *filp)
1939 {
1940         if (tracing_disabled)
1941                 return -ENODEV;
1942
1943         filp->private_data = inode->i_private;
1944         return 0;
1945 }
1946
1947 int tracing_release(struct inode *inode, struct file *file)
1948 {
1949         struct seq_file *m = (struct seq_file *)file->private_data;
1950         struct trace_iterator *iter = m->private;
1951         int cpu;
1952
1953         mutex_lock(&trace_types_lock);
1954         for_each_tracing_cpu(cpu) {
1955                 if (iter->buffer_iter[cpu])
1956                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1957         }
1958
1959         if (iter->trace && iter->trace->close)
1960                 iter->trace->close(iter);
1961
1962         /* reenable tracing if it was previously enabled */
1963         if (iter->tr->ctrl) {
1964                 tracer_enabled = 1;
1965                 /*
1966                  * It is safe to enable function tracing even if it
1967                  * isn't used
1968                  */
1969                 ftrace_function_enabled = 1;
1970         }
1971         mutex_unlock(&trace_types_lock);
1972
1973         seq_release(inode, file);
1974         kfree(iter);
1975         return 0;
1976 }
1977
1978 static int tracing_open(struct inode *inode, struct file *file)
1979 {
1980         int ret;
1981
1982         __tracing_open(inode, file, &ret);
1983
1984         return ret;
1985 }
1986
1987 static int tracing_lt_open(struct inode *inode, struct file *file)
1988 {
1989         struct trace_iterator *iter;
1990         int ret;
1991
1992         iter = __tracing_open(inode, file, &ret);
1993
1994         if (!ret)
1995                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1996
1997         return ret;
1998 }
1999
2000
2001 static void *
2002 t_next(struct seq_file *m, void *v, loff_t *pos)
2003 {
2004         struct tracer *t = m->private;
2005
2006         (*pos)++;
2007
2008         if (t)
2009                 t = t->next;
2010
2011         m->private = t;
2012
2013         return t;
2014 }
2015
2016 static void *t_start(struct seq_file *m, loff_t *pos)
2017 {
2018         struct tracer *t = m->private;
2019         loff_t l = 0;
2020
2021         mutex_lock(&trace_types_lock);
2022         for (; t && l < *pos; t = t_next(m, t, &l))
2023                 ;
2024
2025         return t;
2026 }
2027
2028 static void t_stop(struct seq_file *m, void *p)
2029 {
2030         mutex_unlock(&trace_types_lock);
2031 }
2032
2033 static int t_show(struct seq_file *m, void *v)
2034 {
2035         struct tracer *t = v;
2036
2037         if (!t)
2038                 return 0;
2039
2040         seq_printf(m, "%s", t->name);
2041         if (t->next)
2042                 seq_putc(m, ' ');
2043         else
2044                 seq_putc(m, '\n');
2045
2046         return 0;
2047 }
2048
2049 static struct seq_operations show_traces_seq_ops = {
2050         .start          = t_start,
2051         .next           = t_next,
2052         .stop           = t_stop,
2053         .show           = t_show,
2054 };
2055
2056 static int show_traces_open(struct inode *inode, struct file *file)
2057 {
2058         int ret;
2059
2060         if (tracing_disabled)
2061                 return -ENODEV;
2062
2063         ret = seq_open(file, &show_traces_seq_ops);
2064         if (!ret) {
2065                 struct seq_file *m = file->private_data;
2066                 m->private = trace_types;
2067         }
2068
2069         return ret;
2070 }
2071
2072 static struct file_operations tracing_fops = {
2073         .open           = tracing_open,
2074         .read           = seq_read,
2075         .llseek         = seq_lseek,
2076         .release        = tracing_release,
2077 };
2078
2079 static struct file_operations tracing_lt_fops = {
2080         .open           = tracing_lt_open,
2081         .read           = seq_read,
2082         .llseek         = seq_lseek,
2083         .release        = tracing_release,
2084 };
2085
2086 static struct file_operations show_traces_fops = {
2087         .open           = show_traces_open,
2088         .read           = seq_read,
2089         .release        = seq_release,
2090 };
2091
2092 /*
2093  * Only trace on a CPU if the bitmask is set:
2094  */
2095 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2096
2097 /*
2098  * When tracing/tracing_cpu_mask is modified then this holds
2099  * the new bitmask we are about to install:
2100  */
2101 static cpumask_t tracing_cpumask_new;
2102
2103 /*
2104  * The tracer itself will not take this lock, but still we want
2105  * to provide a consistent cpumask to user-space:
2106  */
2107 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2108
2109 /*
2110  * Temporary storage for the character representation of the
2111  * CPU bitmask (and one more byte for the newline):
2112  */
2113 static char mask_str[NR_CPUS + 1];
2114
2115 static ssize_t
2116 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2117                      size_t count, loff_t *ppos)
2118 {
2119         int len;
2120
2121         mutex_lock(&tracing_cpumask_update_lock);
2122
2123         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2124         if (count - len < 2) {
2125                 count = -EINVAL;
2126                 goto out_err;
2127         }
2128         len += sprintf(mask_str + len, "\n");
2129         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2130
2131 out_err:
2132         mutex_unlock(&tracing_cpumask_update_lock);
2133
2134         return count;
2135 }
2136
2137 static ssize_t
2138 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2139                       size_t count, loff_t *ppos)
2140 {
2141         int err, cpu;
2142
2143         mutex_lock(&tracing_cpumask_update_lock);
2144         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2145         if (err)
2146                 goto err_unlock;
2147
2148         raw_local_irq_disable();
2149         __raw_spin_lock(&ftrace_max_lock);
2150         for_each_tracing_cpu(cpu) {
2151                 /*
2152                  * Increase/decrease the disabled counter if we are
2153                  * about to flip a bit in the cpumask:
2154                  */
2155                 if (cpu_isset(cpu, tracing_cpumask) &&
2156                                 !cpu_isset(cpu, tracing_cpumask_new)) {
2157                         atomic_inc(&global_trace.data[cpu]->disabled);
2158                 }
2159                 if (!cpu_isset(cpu, tracing_cpumask) &&
2160                                 cpu_isset(cpu, tracing_cpumask_new)) {
2161                         atomic_dec(&global_trace.data[cpu]->disabled);
2162                 }
2163         }
2164         __raw_spin_unlock(&ftrace_max_lock);
2165         raw_local_irq_enable();
2166
2167         tracing_cpumask = tracing_cpumask_new;
2168
2169         mutex_unlock(&tracing_cpumask_update_lock);
2170
2171         return count;
2172
2173 err_unlock:
2174         mutex_unlock(&tracing_cpumask_update_lock);
2175
2176         return err;
2177 }
2178
2179 static struct file_operations tracing_cpumask_fops = {
2180         .open           = tracing_open_generic,
2181         .read           = tracing_cpumask_read,
2182         .write          = tracing_cpumask_write,
2183 };
2184
2185 static ssize_t
2186 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2187                        size_t cnt, loff_t *ppos)
2188 {
2189         char *buf;
2190         int r = 0;
2191         int len = 0;
2192         int i;
2193
2194         /* calulate max size */
2195         for (i = 0; trace_options[i]; i++) {
2196                 len += strlen(trace_options[i]);
2197                 len += 3; /* "no" and space */
2198         }
2199
2200         /* +2 for \n and \0 */
2201         buf = kmalloc(len + 2, GFP_KERNEL);
2202         if (!buf)
2203                 return -ENOMEM;
2204
2205         for (i = 0; trace_options[i]; i++) {
2206                 if (trace_flags & (1 << i))
2207                         r += sprintf(buf + r, "%s ", trace_options[i]);
2208                 else
2209                         r += sprintf(buf + r, "no%s ", trace_options[i]);
2210         }
2211
2212         r += sprintf(buf + r, "\n");
2213         WARN_ON(r >= len + 2);
2214
2215         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2216
2217         kfree(buf);
2218
2219         return r;
2220 }
2221
2222 static ssize_t
2223 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2224                         size_t cnt, loff_t *ppos)
2225 {
2226         char buf[64];
2227         char *cmp = buf;
2228         int neg = 0;
2229         int i;
2230
2231         if (cnt >= sizeof(buf))
2232                 return -EINVAL;
2233
2234         if (copy_from_user(&buf, ubuf, cnt))
2235                 return -EFAULT;
2236
2237         buf[cnt] = 0;
2238
2239         if (strncmp(buf, "no", 2) == 0) {
2240                 neg = 1;
2241                 cmp += 2;
2242         }
2243
2244         for (i = 0; trace_options[i]; i++) {
2245                 int len = strlen(trace_options[i]);
2246
2247                 if (strncmp(cmp, trace_options[i], len) == 0) {
2248                         if (neg)
2249                                 trace_flags &= ~(1 << i);
2250                         else
2251                                 trace_flags |= (1 << i);
2252                         break;
2253                 }
2254         }
2255         /*
2256          * If no option could be set, return an error:
2257          */
2258         if (!trace_options[i])
2259                 return -EINVAL;
2260
2261         filp->f_pos += cnt;
2262
2263         return cnt;
2264 }
2265
2266 static struct file_operations tracing_iter_fops = {
2267         .open           = tracing_open_generic,
2268         .read           = tracing_iter_ctrl_read,
2269         .write          = tracing_iter_ctrl_write,
2270 };
2271
2272 static const char readme_msg[] =
2273         "tracing mini-HOWTO:\n\n"
2274         "# mkdir /debug\n"
2275         "# mount -t debugfs nodev /debug\n\n"
2276         "# cat /debug/tracing/available_tracers\n"
2277         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2278         "# cat /debug/tracing/current_tracer\n"
2279         "none\n"
2280         "# echo sched_switch > /debug/tracing/current_tracer\n"
2281         "# cat /debug/tracing/current_tracer\n"
2282         "sched_switch\n"
2283         "# cat /debug/tracing/iter_ctrl\n"
2284         "noprint-parent nosym-offset nosym-addr noverbose\n"
2285         "# echo print-parent > /debug/tracing/iter_ctrl\n"
2286         "# echo 1 > /debug/tracing/tracing_enabled\n"
2287         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2288         "echo 0 > /debug/tracing/tracing_enabled\n"
2289 ;
2290
2291 static ssize_t
2292 tracing_readme_read(struct file *filp, char __user *ubuf,
2293                        size_t cnt, loff_t *ppos)
2294 {
2295         return simple_read_from_buffer(ubuf, cnt, ppos,
2296                                         readme_msg, strlen(readme_msg));
2297 }
2298
2299 static struct file_operations tracing_readme_fops = {
2300         .open           = tracing_open_generic,
2301         .read           = tracing_readme_read,
2302 };
2303
2304 static ssize_t
2305 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2306                   size_t cnt, loff_t *ppos)
2307 {
2308         struct trace_array *tr = filp->private_data;
2309         char buf[64];
2310         int r;
2311
2312         r = sprintf(buf, "%ld\n", tr->ctrl);
2313         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2314 }
2315
2316 static ssize_t
2317 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2318                    size_t cnt, loff_t *ppos)
2319 {
2320         struct trace_array *tr = filp->private_data;
2321         char buf[64];
2322         long val;
2323         int ret;
2324
2325         if (cnt >= sizeof(buf))
2326                 return -EINVAL;
2327
2328         if (copy_from_user(&buf, ubuf, cnt))
2329                 return -EFAULT;
2330
2331         buf[cnt] = 0;
2332
2333         ret = strict_strtoul(buf, 10, &val);
2334         if (ret < 0)
2335                 return ret;
2336
2337         val = !!val;
2338
2339         mutex_lock(&trace_types_lock);
2340         if (tr->ctrl ^ val) {
2341                 if (val)
2342                         tracer_enabled = 1;
2343                 else
2344                         tracer_enabled = 0;
2345
2346                 tr->ctrl = val;
2347
2348                 if (current_trace && current_trace->ctrl_update)
2349                         current_trace->ctrl_update(tr);
2350         }
2351         mutex_unlock(&trace_types_lock);
2352
2353         filp->f_pos += cnt;
2354
2355         return cnt;
2356 }
2357
2358 static ssize_t
2359 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2360                        size_t cnt, loff_t *ppos)
2361 {
2362         char buf[max_tracer_type_len+2];
2363         int r;
2364
2365         mutex_lock(&trace_types_lock);
2366         if (current_trace)
2367                 r = sprintf(buf, "%s\n", current_trace->name);
2368         else
2369                 r = sprintf(buf, "\n");
2370         mutex_unlock(&trace_types_lock);
2371
2372         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2373 }
2374
2375 static ssize_t
2376 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2377                         size_t cnt, loff_t *ppos)
2378 {
2379         struct trace_array *tr = &global_trace;
2380         struct tracer *t;
2381         char buf[max_tracer_type_len+1];
2382         int i;
2383         size_t ret;
2384
2385         ret = cnt;
2386
2387         if (cnt > max_tracer_type_len)
2388                 cnt = max_tracer_type_len;
2389
2390         if (copy_from_user(&buf, ubuf, cnt))
2391                 return -EFAULT;
2392
2393         buf[cnt] = 0;
2394
2395         /* strip ending whitespace. */
2396         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2397                 buf[i] = 0;
2398
2399         mutex_lock(&trace_types_lock);
2400         for (t = trace_types; t; t = t->next) {
2401                 if (strcmp(t->name, buf) == 0)
2402                         break;
2403         }
2404         if (!t) {
2405                 ret = -EINVAL;
2406                 goto out;
2407         }
2408         if (t == current_trace)
2409                 goto out;
2410
2411         if (current_trace && current_trace->reset)
2412                 current_trace->reset(tr);
2413
2414         current_trace = t;
2415         if (t->init)
2416                 t->init(tr);
2417
2418  out:
2419         mutex_unlock(&trace_types_lock);
2420
2421         if (ret > 0)
2422                 filp->f_pos += ret;
2423
2424         return ret;
2425 }
2426
2427 static ssize_t
2428 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2429                      size_t cnt, loff_t *ppos)
2430 {
2431         unsigned long *ptr = filp->private_data;
2432         char buf[64];
2433         int r;
2434
2435         r = snprintf(buf, sizeof(buf), "%ld\n",
2436                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2437         if (r > sizeof(buf))
2438                 r = sizeof(buf);
2439         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2440 }
2441
2442 static ssize_t
2443 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2444                       size_t cnt, loff_t *ppos)
2445 {
2446         long *ptr = filp->private_data;
2447         char buf[64];
2448         long val;
2449         int ret;
2450
2451         if (cnt >= sizeof(buf))
2452                 return -EINVAL;
2453
2454         if (copy_from_user(&buf, ubuf, cnt))
2455                 return -EFAULT;
2456
2457         buf[cnt] = 0;
2458
2459         ret = strict_strtoul(buf, 10, &val);
2460         if (ret < 0)
2461                 return ret;
2462
2463         *ptr = val * 1000;
2464
2465         return cnt;
2466 }
2467
2468 static atomic_t tracing_reader;
2469
2470 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2471 {
2472         struct trace_iterator *iter;
2473
2474         if (tracing_disabled)
2475                 return -ENODEV;
2476
2477         /* We only allow for reader of the pipe */
2478         if (atomic_inc_return(&tracing_reader) != 1) {
2479                 atomic_dec(&tracing_reader);
2480                 return -EBUSY;
2481         }
2482
2483         /* create a buffer to store the information to pass to userspace */
2484         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2485         if (!iter)
2486                 return -ENOMEM;
2487
2488         mutex_lock(&trace_types_lock);
2489         iter->tr = &global_trace;
2490         iter->trace = current_trace;
2491         filp->private_data = iter;
2492
2493         if (iter->trace->pipe_open)
2494                 iter->trace->pipe_open(iter);
2495         mutex_unlock(&trace_types_lock);
2496
2497         return 0;
2498 }
2499
2500 static int tracing_release_pipe(struct inode *inode, struct file *file)
2501 {
2502         struct trace_iterator *iter = file->private_data;
2503
2504         kfree(iter);
2505         atomic_dec(&tracing_reader);
2506
2507         return 0;
2508 }
2509
2510 static unsigned int
2511 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2512 {
2513         struct trace_iterator *iter = filp->private_data;
2514
2515         if (trace_flags & TRACE_ITER_BLOCK) {
2516                 /*
2517                  * Always select as readable when in blocking mode
2518                  */
2519                 return POLLIN | POLLRDNORM;
2520         } else {
2521                 if (!trace_empty(iter))
2522                         return POLLIN | POLLRDNORM;
2523                 poll_wait(filp, &trace_wait, poll_table);
2524                 if (!trace_empty(iter))
2525                         return POLLIN | POLLRDNORM;
2526
2527                 return 0;
2528         }
2529 }
2530
2531 /*
2532  * Consumer reader.
2533  */
2534 static ssize_t
2535 tracing_read_pipe(struct file *filp, char __user *ubuf,
2536                   size_t cnt, loff_t *ppos)
2537 {
2538         struct trace_iterator *iter = filp->private_data;
2539         ssize_t sret;
2540
2541         /* return any leftover data */
2542         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2543         if (sret != -EBUSY)
2544                 return sret;
2545
2546         trace_seq_reset(&iter->seq);
2547
2548         mutex_lock(&trace_types_lock);
2549         if (iter->trace->read) {
2550                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2551                 if (sret)
2552                         goto out;
2553         }
2554
2555 waitagain:
2556         sret = 0;
2557         while (trace_empty(iter)) {
2558
2559                 if ((filp->f_flags & O_NONBLOCK)) {
2560                         sret = -EAGAIN;
2561                         goto out;
2562                 }
2563
2564                 /*
2565                  * This is a make-shift waitqueue. The reason we don't use
2566                  * an actual wait queue is because:
2567                  *  1) we only ever have one waiter
2568                  *  2) the tracing, traces all functions, we don't want
2569                  *     the overhead of calling wake_up and friends
2570                  *     (and tracing them too)
2571                  *     Anyway, this is really very primitive wakeup.
2572                  */
2573                 set_current_state(TASK_INTERRUPTIBLE);
2574                 iter->tr->waiter = current;
2575
2576                 mutex_unlock(&trace_types_lock);
2577
2578                 /* sleep for 100 msecs, and try again. */
2579                 schedule_timeout(HZ/10);
2580
2581                 mutex_lock(&trace_types_lock);
2582
2583                 iter->tr->waiter = NULL;
2584
2585                 if (signal_pending(current)) {
2586                         sret = -EINTR;
2587                         goto out;
2588                 }
2589
2590                 if (iter->trace != current_trace)
2591                         goto out;
2592
2593                 /*
2594                  * We block until we read something and tracing is disabled.
2595                  * We still block if tracing is disabled, but we have never
2596                  * read anything. This allows a user to cat this file, and
2597                  * then enable tracing. But after we have read something,
2598                  * we give an EOF when tracing is again disabled.
2599                  *
2600                  * iter->pos will be 0 if we haven't read anything.
2601                  */
2602                 if (!tracer_enabled && iter->pos)
2603                         break;
2604
2605                 continue;
2606         }
2607
2608         /* stop when tracing is finished */
2609         if (trace_empty(iter))
2610                 goto out;
2611
2612         if (cnt >= PAGE_SIZE)
2613                 cnt = PAGE_SIZE - 1;
2614
2615         /* reset all but tr, trace, and overruns */
2616         memset(&iter->seq, 0,
2617                sizeof(struct trace_iterator) -
2618                offsetof(struct trace_iterator, seq));
2619         iter->pos = -1;
2620
2621         while (find_next_entry_inc(iter) != NULL) {
2622                 enum print_line_t ret;
2623                 int len = iter->seq.len;
2624
2625                 ret = print_trace_line(iter);
2626                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2627                         /* don't print partial lines */
2628                         iter->seq.len = len;
2629                         break;
2630                 }
2631
2632                 trace_consume(iter);
2633
2634                 if (iter->seq.len >= cnt)
2635                         break;
2636         }
2637
2638         /* Now copy what we have to the user */
2639         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2640         if (iter->seq.readpos >= iter->seq.len)
2641                 trace_seq_reset(&iter->seq);
2642
2643         /*
2644          * If there was nothing to send to user, inspite of consuming trace
2645          * entries, go back to wait for more entries.
2646          */
2647         if (sret == -EBUSY)
2648                 goto waitagain;
2649
2650 out:
2651         mutex_unlock(&trace_types_lock);
2652
2653         return sret;
2654 }
2655
2656 static ssize_t
2657 tracing_entries_read(struct file *filp, char __user *ubuf,
2658                      size_t cnt, loff_t *ppos)
2659 {
2660         struct trace_array *tr = filp->private_data;
2661         char buf[64];
2662         int r;
2663
2664         r = sprintf(buf, "%lu\n", tr->entries);
2665         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2666 }
2667
2668 static ssize_t
2669 tracing_entries_write(struct file *filp, const char __user *ubuf,
2670                       size_t cnt, loff_t *ppos)
2671 {
2672         unsigned long val;
2673         char buf[64];
2674         int ret;
2675         struct trace_array *tr = filp->private_data;
2676
2677         if (cnt >= sizeof(buf))
2678                 return -EINVAL;
2679
2680         if (copy_from_user(&buf, ubuf, cnt))
2681                 return -EFAULT;
2682
2683         buf[cnt] = 0;
2684
2685         ret = strict_strtoul(buf, 10, &val);
2686         if (ret < 0)
2687                 return ret;
2688
2689         /* must have at least 1 entry */
2690         if (!val)
2691                 return -EINVAL;
2692
2693         mutex_lock(&trace_types_lock);
2694
2695         if (tr->ctrl) {
2696                 cnt = -EBUSY;
2697                 pr_info("ftrace: please disable tracing"
2698                         " before modifying buffer size\n");
2699                 goto out;
2700         }
2701
2702         if (val != global_trace.entries) {
2703                 ret = ring_buffer_resize(global_trace.buffer, val);
2704                 if (ret < 0) {
2705                         cnt = ret;
2706                         goto out;
2707                 }
2708
2709                 ret = ring_buffer_resize(max_tr.buffer, val);
2710                 if (ret < 0) {
2711                         int r;
2712                         cnt = ret;
2713                         r = ring_buffer_resize(global_trace.buffer,
2714                                                global_trace.entries);
2715                         if (r < 0) {
2716                                 /* AARGH! We are left with different
2717                                  * size max buffer!!!! */
2718                                 WARN_ON(1);
2719                                 tracing_disabled = 1;
2720                         }
2721                         goto out;
2722                 }
2723
2724                 global_trace.entries = val;
2725         }
2726
2727         filp->f_pos += cnt;
2728
2729         /* If check pages failed, return ENOMEM */
2730         if (tracing_disabled)
2731                 cnt = -ENOMEM;
2732  out:
2733         max_tr.entries = global_trace.entries;
2734         mutex_unlock(&trace_types_lock);
2735
2736         return cnt;
2737 }
2738
2739 static int mark_printk(const char *fmt, ...)
2740 {
2741         int ret;
2742         va_list args;
2743         va_start(args, fmt);
2744         ret = trace_vprintk(0, fmt, args);
2745         va_end(args);
2746         return ret;
2747 }
2748
2749 static ssize_t
2750 tracing_mark_write(struct file *filp, const char __user *ubuf,
2751                                         size_t cnt, loff_t *fpos)
2752 {
2753         char *buf;
2754         char *end;
2755         struct trace_array *tr = &global_trace;
2756
2757         if (!tr->ctrl || tracing_disabled)
2758                 return -EINVAL;
2759
2760         if (cnt > TRACE_BUF_SIZE)
2761                 cnt = TRACE_BUF_SIZE;
2762
2763         buf = kmalloc(cnt + 1, GFP_KERNEL);
2764         if (buf == NULL)
2765                 return -ENOMEM;
2766
2767         if (copy_from_user(buf, ubuf, cnt)) {
2768                 kfree(buf);
2769                 return -EFAULT;
2770         }
2771
2772         /* Cut from the first nil or newline. */
2773         buf[cnt] = '\0';
2774         end = strchr(buf, '\n');
2775         if (end)
2776                 *end = '\0';
2777
2778         cnt = mark_printk("%s\n", buf);
2779         kfree(buf);
2780         *fpos += cnt;
2781
2782         return cnt;
2783 }
2784
2785 static struct file_operations tracing_max_lat_fops = {
2786         .open           = tracing_open_generic,
2787         .read           = tracing_max_lat_read,
2788         .write          = tracing_max_lat_write,
2789 };
2790
2791 static struct file_operations tracing_ctrl_fops = {
2792         .open           = tracing_open_generic,
2793         .read           = tracing_ctrl_read,
2794         .write          = tracing_ctrl_write,
2795 };
2796
2797 static struct file_operations set_tracer_fops = {
2798         .open           = tracing_open_generic,
2799         .read           = tracing_set_trace_read,
2800         .write          = tracing_set_trace_write,
2801 };
2802
2803 static struct file_operations tracing_pipe_fops = {
2804         .open           = tracing_open_pipe,
2805         .poll           = tracing_poll_pipe,
2806         .read           = tracing_read_pipe,
2807         .release        = tracing_release_pipe,
2808 };
2809
2810 static struct file_operations tracing_entries_fops = {
2811         .open           = tracing_open_generic,
2812         .read           = tracing_entries_read,
2813         .write          = tracing_entries_write,
2814 };
2815
2816 static struct file_operations tracing_mark_fops = {
2817         .open           = tracing_open_generic,
2818         .write          = tracing_mark_write,
2819 };
2820
2821 #ifdef CONFIG_DYNAMIC_FTRACE
2822
2823 static ssize_t
2824 tracing_read_long(struct file *filp, char __user *ubuf,
2825                   size_t cnt, loff_t *ppos)
2826 {
2827         unsigned long *p = filp->private_data;
2828         char buf[64];
2829         int r;
2830
2831         r = sprintf(buf, "%ld\n", *p);
2832
2833         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2834 }
2835
2836 static struct file_operations tracing_read_long_fops = {
2837         .open           = tracing_open_generic,
2838         .read           = tracing_read_long,
2839 };
2840 #endif
2841
2842 static struct dentry *d_tracer;
2843
2844 struct dentry *tracing_init_dentry(void)
2845 {
2846         static int once;
2847
2848         if (d_tracer)
2849                 return d_tracer;
2850
2851         d_tracer = debugfs_create_dir("tracing", NULL);
2852
2853         if (!d_tracer && !once) {
2854                 once = 1;
2855                 pr_warning("Could not create debugfs directory 'tracing'\n");
2856                 return NULL;
2857         }
2858
2859         return d_tracer;
2860 }
2861
2862 #ifdef CONFIG_FTRACE_SELFTEST
2863 /* Let selftest have access to static functions in this file */
2864 #include "trace_selftest.c"
2865 #endif
2866
2867 static __init int tracer_init_debugfs(void)
2868 {
2869         struct dentry *d_tracer;
2870         struct dentry *entry;
2871
2872         d_tracer = tracing_init_dentry();
2873
2874         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2875                                     &global_trace, &tracing_ctrl_fops);
2876         if (!entry)
2877                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2878
2879         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2880                                     NULL, &tracing_iter_fops);
2881         if (!entry)
2882                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2883
2884         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2885                                     NULL, &tracing_cpumask_fops);
2886         if (!entry)
2887                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2888
2889         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2890                                     &global_trace, &tracing_lt_fops);
2891         if (!entry)
2892                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2893
2894         entry = debugfs_create_file("trace", 0444, d_tracer,
2895                                     &global_trace, &tracing_fops);
2896         if (!entry)
2897                 pr_warning("Could not create debugfs 'trace' entry\n");
2898
2899         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2900                                     &global_trace, &show_traces_fops);
2901         if (!entry)
2902                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
2903
2904         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2905                                     &global_trace, &set_tracer_fops);
2906         if (!entry)
2907                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
2908
2909         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2910                                     &tracing_max_latency,
2911                                     &tracing_max_lat_fops);
2912         if (!entry)
2913                 pr_warning("Could not create debugfs "
2914                            "'tracing_max_latency' entry\n");
2915
2916         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2917                                     &tracing_thresh, &tracing_max_lat_fops);
2918         if (!entry)
2919                 pr_warning("Could not create debugfs "
2920                            "'tracing_thresh' entry\n");
2921         entry = debugfs_create_file("README", 0644, d_tracer,
2922                                     NULL, &tracing_readme_fops);
2923         if (!entry)
2924                 pr_warning("Could not create debugfs 'README' entry\n");
2925
2926         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2927                                     NULL, &tracing_pipe_fops);
2928         if (!entry)
2929                 pr_warning("Could not create debugfs "
2930                            "'trace_pipe' entry\n");
2931
2932         entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2933                                     &global_trace, &tracing_entries_fops);
2934         if (!entry)
2935                 pr_warning("Could not create debugfs "
2936                            "'trace_entries' entry\n");
2937
2938         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
2939                                     NULL, &tracing_mark_fops);
2940         if (!entry)
2941                 pr_warning("Could not create debugfs "
2942                            "'trace_marker' entry\n");
2943
2944 #ifdef CONFIG_DYNAMIC_FTRACE
2945         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2946                                     &ftrace_update_tot_cnt,
2947                                     &tracing_read_long_fops);
2948         if (!entry)
2949                 pr_warning("Could not create debugfs "
2950                            "'dyn_ftrace_total_info' entry\n");
2951 #endif
2952 #ifdef CONFIG_SYSPROF_TRACER
2953         init_tracer_sysprof_debugfs(d_tracer);
2954 #endif
2955         return 0;
2956 }
2957
2958 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2959 {
2960         static DEFINE_SPINLOCK(trace_buf_lock);
2961         static char trace_buf[TRACE_BUF_SIZE];
2962
2963         struct ring_buffer_event *event;
2964         struct trace_array *tr = &global_trace;
2965         struct trace_array_cpu *data;
2966         struct print_entry *entry;
2967         unsigned long flags, irq_flags;
2968         int cpu, len = 0, size, pc;
2969
2970         if (!tr->ctrl || tracing_disabled)
2971                 return 0;
2972
2973         pc = preempt_count();
2974         preempt_disable_notrace();
2975         cpu = raw_smp_processor_id();
2976         data = tr->data[cpu];
2977
2978         if (unlikely(atomic_read(&data->disabled)))
2979                 goto out;
2980
2981         spin_lock_irqsave(&trace_buf_lock, flags);
2982         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
2983
2984         len = min(len, TRACE_BUF_SIZE-1);
2985         trace_buf[len] = 0;
2986
2987         size = sizeof(*entry) + len + 1;
2988         event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
2989         if (!event)
2990                 goto out_unlock;
2991         entry = ring_buffer_event_data(event);
2992         tracing_generic_entry_update(&entry->ent, flags, pc);
2993         entry->ent.type                 = TRACE_PRINT;
2994         entry->ip                       = ip;
2995
2996         memcpy(&entry->buf, trace_buf, len);
2997         entry->buf[len] = 0;
2998         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
2999
3000  out_unlock:
3001         spin_unlock_irqrestore(&trace_buf_lock, flags);
3002
3003  out:
3004         preempt_enable_notrace();
3005
3006         return len;
3007 }
3008 EXPORT_SYMBOL_GPL(trace_vprintk);
3009
3010 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3011 {
3012         int ret;
3013         va_list ap;
3014
3015         if (!(trace_flags & TRACE_ITER_PRINTK))
3016                 return 0;
3017
3018         va_start(ap, fmt);
3019         ret = trace_vprintk(ip, fmt, ap);
3020         va_end(ap);
3021         return ret;
3022 }
3023 EXPORT_SYMBOL_GPL(__ftrace_printk);
3024
3025 static int trace_panic_handler(struct notifier_block *this,
3026                                unsigned long event, void *unused)
3027 {
3028         ftrace_dump();
3029         return NOTIFY_OK;
3030 }
3031
3032 static struct notifier_block trace_panic_notifier = {
3033         .notifier_call  = trace_panic_handler,
3034         .next           = NULL,
3035         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
3036 };
3037
3038 static int trace_die_handler(struct notifier_block *self,
3039                              unsigned long val,
3040                              void *data)
3041 {
3042         switch (val) {
3043         case DIE_OOPS:
3044                 ftrace_dump();
3045                 break;
3046         default:
3047                 break;
3048         }
3049         return NOTIFY_OK;
3050 }
3051
3052 static struct notifier_block trace_die_notifier = {
3053         .notifier_call = trace_die_handler,
3054         .priority = 200
3055 };
3056
3057 /*
3058  * printk is set to max of 1024, we really don't need it that big.
3059  * Nothing should be printing 1000 characters anyway.
3060  */
3061 #define TRACE_MAX_PRINT         1000
3062
3063 /*
3064  * Define here KERN_TRACE so that we have one place to modify
3065  * it if we decide to change what log level the ftrace dump
3066  * should be at.
3067  */
3068 #define KERN_TRACE              KERN_INFO
3069
3070 static void
3071 trace_printk_seq(struct trace_seq *s)
3072 {
3073         /* Probably should print a warning here. */
3074         if (s->len >= 1000)
3075                 s->len = 1000;
3076
3077         /* should be zero ended, but we are paranoid. */
3078         s->buffer[s->len] = 0;
3079
3080         printk(KERN_TRACE "%s", s->buffer);
3081
3082         trace_seq_reset(s);
3083 }
3084
3085
3086 void ftrace_dump(void)
3087 {
3088         static DEFINE_SPINLOCK(ftrace_dump_lock);
3089         /* use static because iter can be a bit big for the stack */
3090         static struct trace_iterator iter;
3091         static cpumask_t mask;
3092         static int dump_ran;
3093         unsigned long flags;
3094         int cnt = 0, cpu;
3095
3096         /* only one dump */
3097         spin_lock_irqsave(&ftrace_dump_lock, flags);
3098         if (dump_ran)
3099                 goto out;
3100
3101         dump_ran = 1;
3102
3103         /* No turning back! */
3104         ftrace_kill();
3105
3106         for_each_tracing_cpu(cpu) {
3107                 atomic_inc(&global_trace.data[cpu]->disabled);
3108         }
3109
3110         printk(KERN_TRACE "Dumping ftrace buffer:\n");
3111
3112         iter.tr = &global_trace;
3113         iter.trace = current_trace;
3114
3115         /*
3116          * We need to stop all tracing on all CPUS to read the
3117          * the next buffer. This is a bit expensive, but is
3118          * not done often. We fill all what we can read,
3119          * and then release the locks again.
3120          */
3121
3122         cpus_clear(mask);
3123
3124         while (!trace_empty(&iter)) {
3125
3126                 if (!cnt)
3127                         printk(KERN_TRACE "---------------------------------\n");
3128
3129                 cnt++;
3130
3131                 /* reset all but tr, trace, and overruns */
3132                 memset(&iter.seq, 0,
3133                        sizeof(struct trace_iterator) -
3134                        offsetof(struct trace_iterator, seq));
3135                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3136                 iter.pos = -1;
3137
3138                 if (find_next_entry_inc(&iter) != NULL) {
3139                         print_trace_line(&iter);
3140                         trace_consume(&iter);
3141                 }
3142
3143                 trace_printk_seq(&iter.seq);
3144         }
3145
3146         if (!cnt)
3147                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3148         else
3149                 printk(KERN_TRACE "---------------------------------\n");
3150
3151  out:
3152         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3153 }
3154
3155 __init static int tracer_alloc_buffers(void)
3156 {
3157         struct trace_array_cpu *data;
3158         int i;
3159
3160         /* TODO: make the number of buffers hot pluggable with CPUS */
3161         tracing_buffer_mask = cpu_possible_map;
3162
3163         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3164                                                    TRACE_BUFFER_FLAGS);
3165         if (!global_trace.buffer) {
3166                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3167                 WARN_ON(1);
3168                 return 0;
3169         }
3170         global_trace.entries = ring_buffer_size(global_trace.buffer);
3171
3172 #ifdef CONFIG_TRACER_MAX_TRACE
3173         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3174                                              TRACE_BUFFER_FLAGS);
3175         if (!max_tr.buffer) {
3176                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3177                 WARN_ON(1);
3178                 ring_buffer_free(global_trace.buffer);
3179                 return 0;
3180         }
3181         max_tr.entries = ring_buffer_size(max_tr.buffer);
3182         WARN_ON(max_tr.entries != global_trace.entries);
3183 #endif
3184
3185         /* Allocate the first page for all buffers */
3186         for_each_tracing_cpu(i) {
3187                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3188                 max_tr.data[i] = &per_cpu(max_data, i);
3189         }
3190
3191         trace_init_cmdlines();
3192
3193         register_tracer(&nop_trace);
3194 #ifdef CONFIG_BOOT_TRACER
3195         register_tracer(&boot_tracer);
3196         current_trace = &boot_tracer;
3197         current_trace->init(&global_trace);
3198 #else
3199         current_trace = &nop_trace;
3200 #endif
3201
3202         /* All seems OK, enable tracing */
3203         global_trace.ctrl = tracer_enabled;
3204         tracing_disabled = 0;
3205
3206         atomic_notifier_chain_register(&panic_notifier_list,
3207                                        &trace_panic_notifier);
3208
3209         register_die_notifier(&trace_die_notifier);
3210
3211         return 0;
3212 }
3213 early_initcall(tracer_alloc_buffers);
3214 fs_initcall(tracer_init_debugfs);