ftrace: preempt disable over interrupt disable
[linux-2.6] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/utsrelease.h>
15 #include <linux/kallsyms.h>
16 #include <linux/seq_file.h>
17 #include <linux/notifier.h>
18 #include <linux/debugfs.h>
19 #include <linux/pagemap.h>
20 #include <linux/hardirq.h>
21 #include <linux/linkage.h>
22 #include <linux/uaccess.h>
23 #include <linux/ftrace.h>
24 #include <linux/module.h>
25 #include <linux/percpu.h>
26 #include <linux/kdebug.h>
27 #include <linux/ctype.h>
28 #include <linux/init.h>
29 #include <linux/poll.h>
30 #include <linux/gfp.h>
31 #include <linux/fs.h>
32 #include <linux/kprobes.h>
33 #include <linux/writeback.h>
34
35 #include <linux/stacktrace.h>
36 #include <linux/ring_buffer.h>
37
38 #include "trace.h"
39
40 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
41
42 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
43 unsigned long __read_mostly     tracing_thresh;
44
45 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
46
47 static inline void ftrace_disable_cpu(void)
48 {
49         preempt_disable();
50         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
51 }
52
53 static inline void ftrace_enable_cpu(void)
54 {
55         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
56         preempt_enable();
57 }
58
59 static cpumask_t __read_mostly          tracing_buffer_mask;
60
61 #define for_each_tracing_cpu(cpu)       \
62         for_each_cpu_mask(cpu, tracing_buffer_mask)
63
64 static int tracing_disabled = 1;
65
66 long
67 ns2usecs(cycle_t nsec)
68 {
69         nsec += 500;
70         do_div(nsec, 1000);
71         return nsec;
72 }
73
74 cycle_t ftrace_now(int cpu)
75 {
76         u64 ts = ring_buffer_time_stamp(cpu);
77         ring_buffer_normalize_time_stamp(cpu, &ts);
78         return ts;
79 }
80
81 /*
82  * The global_trace is the descriptor that holds the tracing
83  * buffers for the live tracing. For each CPU, it contains
84  * a link list of pages that will store trace entries. The
85  * page descriptor of the pages in the memory is used to hold
86  * the link list by linking the lru item in the page descriptor
87  * to each of the pages in the buffer per CPU.
88  *
89  * For each active CPU there is a data field that holds the
90  * pages for the buffer for that CPU. Each CPU has the same number
91  * of pages allocated for its buffer.
92  */
93 static struct trace_array       global_trace;
94
95 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
96
97 /*
98  * The max_tr is used to snapshot the global_trace when a maximum
99  * latency is reached. Some tracers will use this to store a maximum
100  * trace while it continues examining live traces.
101  *
102  * The buffers for the max_tr are set up the same as the global_trace.
103  * When a snapshot is taken, the link list of the max_tr is swapped
104  * with the link list of the global_trace and the buffers are reset for
105  * the global_trace so the tracing can continue.
106  */
107 static struct trace_array       max_tr;
108
109 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
110
111 /* tracer_enabled is used to toggle activation of a tracer */
112 static int                      tracer_enabled = 1;
113
114 /* function tracing enabled */
115 int                             ftrace_function_enabled;
116
117 /*
118  * trace_buf_size is the size in bytes that is allocated
119  * for a buffer. Note, the number of bytes is always rounded
120  * to page size.
121  *
122  * This number is purposely set to a low number of 16384.
123  * If the dump on oops happens, it will be much appreciated
124  * to not have to wait for all that output. Anyway this can be
125  * boot time and run time configurable.
126  */
127 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
128
129 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
130
131 /* trace_types holds a link list of available tracers. */
132 static struct tracer            *trace_types __read_mostly;
133
134 /* current_trace points to the tracer that is currently active */
135 static struct tracer            *current_trace __read_mostly;
136
137 /*
138  * max_tracer_type_len is used to simplify the allocating of
139  * buffers to read userspace tracer names. We keep track of
140  * the longest tracer name registered.
141  */
142 static int                      max_tracer_type_len;
143
144 /*
145  * trace_types_lock is used to protect the trace_types list.
146  * This lock is also used to keep user access serialized.
147  * Accesses from userspace will grab this lock while userspace
148  * activities happen inside the kernel.
149  */
150 static DEFINE_MUTEX(trace_types_lock);
151
152 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
153 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
154
155 /* trace_flags holds iter_ctrl options */
156 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
157
158 /**
159  * trace_wake_up - wake up tasks waiting for trace input
160  *
161  * Simply wakes up any task that is blocked on the trace_wait
162  * queue. These is used with trace_poll for tasks polling the trace.
163  */
164 void trace_wake_up(void)
165 {
166         /*
167          * The runqueue_is_locked() can fail, but this is the best we
168          * have for now:
169          */
170         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
171                 wake_up(&trace_wait);
172 }
173
174 static int __init set_buf_size(char *str)
175 {
176         unsigned long buf_size;
177         int ret;
178
179         if (!str)
180                 return 0;
181         ret = strict_strtoul(str, 0, &buf_size);
182         /* nr_entries can not be zero */
183         if (ret < 0 || buf_size == 0)
184                 return 0;
185         trace_buf_size = buf_size;
186         return 1;
187 }
188 __setup("trace_buf_size=", set_buf_size);
189
190 unsigned long nsecs_to_usecs(unsigned long nsecs)
191 {
192         return nsecs / 1000;
193 }
194
195 /*
196  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
197  * control the output of kernel symbols.
198  */
199 #define TRACE_ITER_SYM_MASK \
200         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
201
202 /* These must match the bit postions in trace_iterator_flags */
203 static const char *trace_options[] = {
204         "print-parent",
205         "sym-offset",
206         "sym-addr",
207         "verbose",
208         "raw",
209         "hex",
210         "bin",
211         "block",
212         "stacktrace",
213         "sched-tree",
214         "ftrace_printk",
215         NULL
216 };
217
218 /*
219  * ftrace_max_lock is used to protect the swapping of buffers
220  * when taking a max snapshot. The buffers themselves are
221  * protected by per_cpu spinlocks. But the action of the swap
222  * needs its own lock.
223  *
224  * This is defined as a raw_spinlock_t in order to help
225  * with performance when lockdep debugging is enabled.
226  */
227 static raw_spinlock_t ftrace_max_lock =
228         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
229
230 /*
231  * Copy the new maximum trace into the separate maximum-trace
232  * structure. (this way the maximum trace is permanently saved,
233  * for later retrieval via /debugfs/tracing/latency_trace)
234  */
235 static void
236 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
237 {
238         struct trace_array_cpu *data = tr->data[cpu];
239
240         max_tr.cpu = cpu;
241         max_tr.time_start = data->preempt_timestamp;
242
243         data = max_tr.data[cpu];
244         data->saved_latency = tracing_max_latency;
245
246         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
247         data->pid = tsk->pid;
248         data->uid = tsk->uid;
249         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
250         data->policy = tsk->policy;
251         data->rt_priority = tsk->rt_priority;
252
253         /* record this tasks comm */
254         tracing_record_cmdline(current);
255 }
256
257 /**
258  * trace_seq_printf - sequence printing of trace information
259  * @s: trace sequence descriptor
260  * @fmt: printf format string
261  *
262  * The tracer may use either sequence operations or its own
263  * copy to user routines. To simplify formating of a trace
264  * trace_seq_printf is used to store strings into a special
265  * buffer (@s). Then the output may be either used by
266  * the sequencer or pulled into another buffer.
267  */
268 int
269 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
270 {
271         int len = (PAGE_SIZE - 1) - s->len;
272         va_list ap;
273         int ret;
274
275         if (!len)
276                 return 0;
277
278         va_start(ap, fmt);
279         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
280         va_end(ap);
281
282         /* If we can't write it all, don't bother writing anything */
283         if (ret >= len)
284                 return 0;
285
286         s->len += ret;
287
288         return len;
289 }
290
291 /**
292  * trace_seq_puts - trace sequence printing of simple string
293  * @s: trace sequence descriptor
294  * @str: simple string to record
295  *
296  * The tracer may use either the sequence operations or its own
297  * copy to user routines. This function records a simple string
298  * into a special buffer (@s) for later retrieval by a sequencer
299  * or other mechanism.
300  */
301 static int
302 trace_seq_puts(struct trace_seq *s, const char *str)
303 {
304         int len = strlen(str);
305
306         if (len > ((PAGE_SIZE - 1) - s->len))
307                 return 0;
308
309         memcpy(s->buffer + s->len, str, len);
310         s->len += len;
311
312         return len;
313 }
314
315 static int
316 trace_seq_putc(struct trace_seq *s, unsigned char c)
317 {
318         if (s->len >= (PAGE_SIZE - 1))
319                 return 0;
320
321         s->buffer[s->len++] = c;
322
323         return 1;
324 }
325
326 static int
327 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
328 {
329         if (len > ((PAGE_SIZE - 1) - s->len))
330                 return 0;
331
332         memcpy(s->buffer + s->len, mem, len);
333         s->len += len;
334
335         return len;
336 }
337
338 #define HEX_CHARS 17
339 static const char hex2asc[] = "0123456789abcdef";
340
341 static int
342 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
343 {
344         unsigned char hex[HEX_CHARS];
345         unsigned char *data = mem;
346         unsigned char byte;
347         int i, j;
348
349         BUG_ON(len >= HEX_CHARS);
350
351 #ifdef __BIG_ENDIAN
352         for (i = 0, j = 0; i < len; i++) {
353 #else
354         for (i = len-1, j = 0; i >= 0; i--) {
355 #endif
356                 byte = data[i];
357
358                 hex[j++] = hex2asc[byte & 0x0f];
359                 hex[j++] = hex2asc[byte >> 4];
360         }
361         hex[j++] = ' ';
362
363         return trace_seq_putmem(s, hex, j);
364 }
365
366 static void
367 trace_seq_reset(struct trace_seq *s)
368 {
369         s->len = 0;
370         s->readpos = 0;
371 }
372
373 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
374 {
375         int len;
376         int ret;
377
378         if (s->len <= s->readpos)
379                 return -EBUSY;
380
381         len = s->len - s->readpos;
382         if (cnt > len)
383                 cnt = len;
384         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
385         if (ret)
386                 return -EFAULT;
387
388         s->readpos += len;
389         return cnt;
390 }
391
392 static void
393 trace_print_seq(struct seq_file *m, struct trace_seq *s)
394 {
395         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
396
397         s->buffer[len] = 0;
398         seq_puts(m, s->buffer);
399
400         trace_seq_reset(s);
401 }
402
403 /**
404  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
405  * @tr: tracer
406  * @tsk: the task with the latency
407  * @cpu: The cpu that initiated the trace.
408  *
409  * Flip the buffers between the @tr and the max_tr and record information
410  * about which task was the cause of this latency.
411  */
412 void
413 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
414 {
415         struct ring_buffer *buf = tr->buffer;
416
417         WARN_ON_ONCE(!irqs_disabled());
418         __raw_spin_lock(&ftrace_max_lock);
419
420         tr->buffer = max_tr.buffer;
421         max_tr.buffer = buf;
422
423         ftrace_disable_cpu();
424         ring_buffer_reset(tr->buffer);
425         ftrace_enable_cpu();
426
427         __update_max_tr(tr, tsk, cpu);
428         __raw_spin_unlock(&ftrace_max_lock);
429 }
430
431 /**
432  * update_max_tr_single - only copy one trace over, and reset the rest
433  * @tr - tracer
434  * @tsk - task with the latency
435  * @cpu - the cpu of the buffer to copy.
436  *
437  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
438  */
439 void
440 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
441 {
442         int ret;
443
444         WARN_ON_ONCE(!irqs_disabled());
445         __raw_spin_lock(&ftrace_max_lock);
446
447         ftrace_disable_cpu();
448
449         ring_buffer_reset(max_tr.buffer);
450         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
451
452         ftrace_enable_cpu();
453
454         WARN_ON_ONCE(ret);
455
456         __update_max_tr(tr, tsk, cpu);
457         __raw_spin_unlock(&ftrace_max_lock);
458 }
459
460 /**
461  * register_tracer - register a tracer with the ftrace system.
462  * @type - the plugin for the tracer
463  *
464  * Register a new plugin tracer.
465  */
466 int register_tracer(struct tracer *type)
467 {
468         struct tracer *t;
469         int len;
470         int ret = 0;
471
472         if (!type->name) {
473                 pr_info("Tracer must have a name\n");
474                 return -1;
475         }
476
477         mutex_lock(&trace_types_lock);
478         for (t = trace_types; t; t = t->next) {
479                 if (strcmp(type->name, t->name) == 0) {
480                         /* already found */
481                         pr_info("Trace %s already registered\n",
482                                 type->name);
483                         ret = -1;
484                         goto out;
485                 }
486         }
487
488 #ifdef CONFIG_FTRACE_STARTUP_TEST
489         if (type->selftest) {
490                 struct tracer *saved_tracer = current_trace;
491                 struct trace_array *tr = &global_trace;
492                 int saved_ctrl = tr->ctrl;
493                 int i;
494                 /*
495                  * Run a selftest on this tracer.
496                  * Here we reset the trace buffer, and set the current
497                  * tracer to be this tracer. The tracer can then run some
498                  * internal tracing to verify that everything is in order.
499                  * If we fail, we do not register this tracer.
500                  */
501                 for_each_tracing_cpu(i) {
502                         tracing_reset(tr, i);
503                 }
504                 current_trace = type;
505                 tr->ctrl = 0;
506                 /* the test is responsible for initializing and enabling */
507                 pr_info("Testing tracer %s: ", type->name);
508                 ret = type->selftest(type, tr);
509                 /* the test is responsible for resetting too */
510                 current_trace = saved_tracer;
511                 tr->ctrl = saved_ctrl;
512                 if (ret) {
513                         printk(KERN_CONT "FAILED!\n");
514                         goto out;
515                 }
516                 /* Only reset on passing, to avoid touching corrupted buffers */
517                 for_each_tracing_cpu(i) {
518                         tracing_reset(tr, i);
519                 }
520                 printk(KERN_CONT "PASSED\n");
521         }
522 #endif
523
524         type->next = trace_types;
525         trace_types = type;
526         len = strlen(type->name);
527         if (len > max_tracer_type_len)
528                 max_tracer_type_len = len;
529
530  out:
531         mutex_unlock(&trace_types_lock);
532
533         return ret;
534 }
535
536 void unregister_tracer(struct tracer *type)
537 {
538         struct tracer **t;
539         int len;
540
541         mutex_lock(&trace_types_lock);
542         for (t = &trace_types; *t; t = &(*t)->next) {
543                 if (*t == type)
544                         goto found;
545         }
546         pr_info("Trace %s not registered\n", type->name);
547         goto out;
548
549  found:
550         *t = (*t)->next;
551         if (strlen(type->name) != max_tracer_type_len)
552                 goto out;
553
554         max_tracer_type_len = 0;
555         for (t = &trace_types; *t; t = &(*t)->next) {
556                 len = strlen((*t)->name);
557                 if (len > max_tracer_type_len)
558                         max_tracer_type_len = len;
559         }
560  out:
561         mutex_unlock(&trace_types_lock);
562 }
563
564 void tracing_reset(struct trace_array *tr, int cpu)
565 {
566         ftrace_disable_cpu();
567         ring_buffer_reset_cpu(tr->buffer, cpu);
568         ftrace_enable_cpu();
569 }
570
571 #define SAVED_CMDLINES 128
572 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
573 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
574 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
575 static int cmdline_idx;
576 static DEFINE_SPINLOCK(trace_cmdline_lock);
577
578 /* temporary disable recording */
579 atomic_t trace_record_cmdline_disabled __read_mostly;
580
581 static void trace_init_cmdlines(void)
582 {
583         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
584         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
585         cmdline_idx = 0;
586 }
587
588 void trace_stop_cmdline_recording(void);
589
590 static void trace_save_cmdline(struct task_struct *tsk)
591 {
592         unsigned map;
593         unsigned idx;
594
595         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
596                 return;
597
598         /*
599          * It's not the end of the world if we don't get
600          * the lock, but we also don't want to spin
601          * nor do we want to disable interrupts,
602          * so if we miss here, then better luck next time.
603          */
604         if (!spin_trylock(&trace_cmdline_lock))
605                 return;
606
607         idx = map_pid_to_cmdline[tsk->pid];
608         if (idx >= SAVED_CMDLINES) {
609                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
610
611                 map = map_cmdline_to_pid[idx];
612                 if (map <= PID_MAX_DEFAULT)
613                         map_pid_to_cmdline[map] = (unsigned)-1;
614
615                 map_pid_to_cmdline[tsk->pid] = idx;
616
617                 cmdline_idx = idx;
618         }
619
620         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
621
622         spin_unlock(&trace_cmdline_lock);
623 }
624
625 static char *trace_find_cmdline(int pid)
626 {
627         char *cmdline = "<...>";
628         unsigned map;
629
630         if (!pid)
631                 return "<idle>";
632
633         if (pid > PID_MAX_DEFAULT)
634                 goto out;
635
636         map = map_pid_to_cmdline[pid];
637         if (map >= SAVED_CMDLINES)
638                 goto out;
639
640         cmdline = saved_cmdlines[map];
641
642  out:
643         return cmdline;
644 }
645
646 void tracing_record_cmdline(struct task_struct *tsk)
647 {
648         if (atomic_read(&trace_record_cmdline_disabled))
649                 return;
650
651         trace_save_cmdline(tsk);
652 }
653
654 void
655 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
656                              int pc)
657 {
658         struct task_struct *tsk = current;
659
660         entry->preempt_count            = pc & 0xff;
661         entry->pid                      = (tsk) ? tsk->pid : 0;
662         entry->flags =
663                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
664                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
665                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
666                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
667 }
668
669 void
670 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
671                unsigned long ip, unsigned long parent_ip, unsigned long flags,
672                int pc)
673 {
674         struct ring_buffer_event *event;
675         struct ftrace_entry *entry;
676         unsigned long irq_flags;
677
678         /* If we are reading the ring buffer, don't trace */
679         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
680                 return;
681
682         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
683                                          &irq_flags);
684         if (!event)
685                 return;
686         entry   = ring_buffer_event_data(event);
687         tracing_generic_entry_update(&entry->ent, flags, pc);
688         entry->ent.type                 = TRACE_FN;
689         entry->ip                       = ip;
690         entry->parent_ip                = parent_ip;
691         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
692 }
693
694 void
695 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
696        unsigned long ip, unsigned long parent_ip, unsigned long flags,
697        int pc)
698 {
699         if (likely(!atomic_read(&data->disabled)))
700                 trace_function(tr, data, ip, parent_ip, flags, pc);
701 }
702
703 static void ftrace_trace_stack(struct trace_array *tr,
704                                struct trace_array_cpu *data,
705                                unsigned long flags,
706                                int skip, int pc)
707 {
708         struct ring_buffer_event *event;
709         struct stack_entry *entry;
710         struct stack_trace trace;
711         unsigned long irq_flags;
712
713         if (!(trace_flags & TRACE_ITER_STACKTRACE))
714                 return;
715
716         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
717                                          &irq_flags);
718         if (!event)
719                 return;
720         entry   = ring_buffer_event_data(event);
721         tracing_generic_entry_update(&entry->ent, flags, pc);
722         entry->ent.type         = TRACE_STACK;
723
724         memset(&entry->caller, 0, sizeof(entry->caller));
725
726         trace.nr_entries        = 0;
727         trace.max_entries       = FTRACE_STACK_ENTRIES;
728         trace.skip              = skip;
729         trace.entries           = entry->caller;
730
731         save_stack_trace(&trace);
732         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
733 }
734
735 void __trace_stack(struct trace_array *tr,
736                    struct trace_array_cpu *data,
737                    unsigned long flags,
738                    int skip)
739 {
740         ftrace_trace_stack(tr, data, flags, skip, preempt_count());
741 }
742
743 static void
744 ftrace_trace_special(void *__tr, void *__data,
745                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
746                      int pc)
747 {
748         struct ring_buffer_event *event;
749         struct trace_array_cpu *data = __data;
750         struct trace_array *tr = __tr;
751         struct special_entry *entry;
752         unsigned long irq_flags;
753
754         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
755                                          &irq_flags);
756         if (!event)
757                 return;
758         entry   = ring_buffer_event_data(event);
759         tracing_generic_entry_update(&entry->ent, 0, pc);
760         entry->ent.type                 = TRACE_SPECIAL;
761         entry->arg1                     = arg1;
762         entry->arg2                     = arg2;
763         entry->arg3                     = arg3;
764         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
765         ftrace_trace_stack(tr, data, irq_flags, 4, pc);
766
767         trace_wake_up();
768 }
769
770 void
771 __trace_special(void *__tr, void *__data,
772                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
773 {
774         ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
775 }
776
777 void
778 tracing_sched_switch_trace(struct trace_array *tr,
779                            struct trace_array_cpu *data,
780                            struct task_struct *prev,
781                            struct task_struct *next,
782                            unsigned long flags, int pc)
783 {
784         struct ring_buffer_event *event;
785         struct ctx_switch_entry *entry;
786         unsigned long irq_flags;
787
788         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
789                                            &irq_flags);
790         if (!event)
791                 return;
792         entry   = ring_buffer_event_data(event);
793         tracing_generic_entry_update(&entry->ent, flags, pc);
794         entry->ent.type                 = TRACE_CTX;
795         entry->prev_pid                 = prev->pid;
796         entry->prev_prio                = prev->prio;
797         entry->prev_state               = prev->state;
798         entry->next_pid                 = next->pid;
799         entry->next_prio                = next->prio;
800         entry->next_state               = next->state;
801         entry->next_cpu = task_cpu(next);
802         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
803         ftrace_trace_stack(tr, data, flags, 5, pc);
804 }
805
806 void
807 tracing_sched_wakeup_trace(struct trace_array *tr,
808                            struct trace_array_cpu *data,
809                            struct task_struct *wakee,
810                            struct task_struct *curr,
811                            unsigned long flags, int pc)
812 {
813         struct ring_buffer_event *event;
814         struct ctx_switch_entry *entry;
815         unsigned long irq_flags;
816
817         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
818                                            &irq_flags);
819         if (!event)
820                 return;
821         entry   = ring_buffer_event_data(event);
822         tracing_generic_entry_update(&entry->ent, flags, pc);
823         entry->ent.type                 = TRACE_WAKE;
824         entry->prev_pid                 = curr->pid;
825         entry->prev_prio                = curr->prio;
826         entry->prev_state               = curr->state;
827         entry->next_pid                 = wakee->pid;
828         entry->next_prio                = wakee->prio;
829         entry->next_state               = wakee->state;
830         entry->next_cpu                 = task_cpu(wakee);
831         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
832         ftrace_trace_stack(tr, data, flags, 6, pc);
833
834         trace_wake_up();
835 }
836
837 void
838 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
839 {
840         struct trace_array *tr = &global_trace;
841         struct trace_array_cpu *data;
842         long disabled;
843         int cpu;
844         int pc;
845
846         if (tracing_disabled || !tr->ctrl)
847                 return;
848
849         pc = preempt_count();
850         preempt_disable_notrace();
851         cpu = raw_smp_processor_id();
852         data = tr->data[cpu];
853         disabled = atomic_inc_return(&data->disabled);
854
855         if (likely(disabled == 1))
856                 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
857
858         atomic_dec(&data->disabled);
859         preempt_enable_notrace();
860 }
861
862 #ifdef CONFIG_FTRACE
863 static void
864 function_trace_call(unsigned long ip, unsigned long parent_ip)
865 {
866         struct trace_array *tr = &global_trace;
867         struct trace_array_cpu *data;
868         unsigned long flags;
869         long disabled;
870         int cpu, resched;
871         int pc;
872
873         if (unlikely(!ftrace_function_enabled))
874                 return;
875
876         if (skip_trace(ip))
877                 return;
878
879         pc = preempt_count();
880         resched = need_resched();
881         preempt_disable_notrace();
882         local_save_flags(flags);
883         cpu = raw_smp_processor_id();
884         data = tr->data[cpu];
885         disabled = atomic_inc_return(&data->disabled);
886
887         if (likely(disabled == 1))
888                 trace_function(tr, data, ip, parent_ip, flags, pc);
889
890         atomic_dec(&data->disabled);
891         if (resched)
892                 preempt_enable_no_resched_notrace();
893         else
894                 preempt_enable_notrace();
895 }
896
897 static struct ftrace_ops trace_ops __read_mostly =
898 {
899         .func = function_trace_call,
900 };
901
902 void tracing_start_function_trace(void)
903 {
904         ftrace_function_enabled = 0;
905         register_ftrace_function(&trace_ops);
906         if (tracer_enabled)
907                 ftrace_function_enabled = 1;
908 }
909
910 void tracing_stop_function_trace(void)
911 {
912         ftrace_function_enabled = 0;
913         unregister_ftrace_function(&trace_ops);
914 }
915 #endif
916
917 enum trace_file_type {
918         TRACE_FILE_LAT_FMT      = 1,
919 };
920
921 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
922 {
923         /* Don't allow ftrace to trace into the ring buffers */
924         ftrace_disable_cpu();
925
926         iter->idx++;
927         if (iter->buffer_iter[iter->cpu])
928                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
929
930         ftrace_enable_cpu();
931 }
932
933 static struct trace_entry *
934 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
935 {
936         struct ring_buffer_event *event;
937         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
938
939         /* Don't allow ftrace to trace into the ring buffers */
940         ftrace_disable_cpu();
941
942         if (buf_iter)
943                 event = ring_buffer_iter_peek(buf_iter, ts);
944         else
945                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
946
947         ftrace_enable_cpu();
948
949         return event ? ring_buffer_event_data(event) : NULL;
950 }
951
952 static struct trace_entry *
953 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
954 {
955         struct ring_buffer *buffer = iter->tr->buffer;
956         struct trace_entry *ent, *next = NULL;
957         u64 next_ts = 0, ts;
958         int next_cpu = -1;
959         int cpu;
960
961         for_each_tracing_cpu(cpu) {
962
963                 if (ring_buffer_empty_cpu(buffer, cpu))
964                         continue;
965
966                 ent = peek_next_entry(iter, cpu, &ts);
967
968                 /*
969                  * Pick the entry with the smallest timestamp:
970                  */
971                 if (ent && (!next || ts < next_ts)) {
972                         next = ent;
973                         next_cpu = cpu;
974                         next_ts = ts;
975                 }
976         }
977
978         if (ent_cpu)
979                 *ent_cpu = next_cpu;
980
981         if (ent_ts)
982                 *ent_ts = next_ts;
983
984         return next;
985 }
986
987 /* Find the next real entry, without updating the iterator itself */
988 static struct trace_entry *
989 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
990 {
991         return __find_next_entry(iter, ent_cpu, ent_ts);
992 }
993
994 /* Find the next real entry, and increment the iterator to the next entry */
995 static void *find_next_entry_inc(struct trace_iterator *iter)
996 {
997         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
998
999         if (iter->ent)
1000                 trace_iterator_increment(iter, iter->cpu);
1001
1002         return iter->ent ? iter : NULL;
1003 }
1004
1005 static void trace_consume(struct trace_iterator *iter)
1006 {
1007         /* Don't allow ftrace to trace into the ring buffers */
1008         ftrace_disable_cpu();
1009         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1010         ftrace_enable_cpu();
1011 }
1012
1013 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1014 {
1015         struct trace_iterator *iter = m->private;
1016         int i = (int)*pos;
1017         void *ent;
1018
1019         (*pos)++;
1020
1021         /* can't go backwards */
1022         if (iter->idx > i)
1023                 return NULL;
1024
1025         if (iter->idx < 0)
1026                 ent = find_next_entry_inc(iter);
1027         else
1028                 ent = iter;
1029
1030         while (ent && iter->idx < i)
1031                 ent = find_next_entry_inc(iter);
1032
1033         iter->pos = *pos;
1034
1035         return ent;
1036 }
1037
1038 static void *s_start(struct seq_file *m, loff_t *pos)
1039 {
1040         struct trace_iterator *iter = m->private;
1041         void *p = NULL;
1042         loff_t l = 0;
1043         int cpu;
1044
1045         mutex_lock(&trace_types_lock);
1046
1047         if (!current_trace || current_trace != iter->trace) {
1048                 mutex_unlock(&trace_types_lock);
1049                 return NULL;
1050         }
1051
1052         atomic_inc(&trace_record_cmdline_disabled);
1053
1054         /* let the tracer grab locks here if needed */
1055         if (current_trace->start)
1056                 current_trace->start(iter);
1057
1058         if (*pos != iter->pos) {
1059                 iter->ent = NULL;
1060                 iter->cpu = 0;
1061                 iter->idx = -1;
1062
1063                 ftrace_disable_cpu();
1064
1065                 for_each_tracing_cpu(cpu) {
1066                         ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1067                 }
1068
1069                 ftrace_enable_cpu();
1070
1071                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1072                         ;
1073
1074         } else {
1075                 l = *pos - 1;
1076                 p = s_next(m, p, &l);
1077         }
1078
1079         return p;
1080 }
1081
1082 static void s_stop(struct seq_file *m, void *p)
1083 {
1084         struct trace_iterator *iter = m->private;
1085
1086         atomic_dec(&trace_record_cmdline_disabled);
1087
1088         /* let the tracer release locks here if needed */
1089         if (current_trace && current_trace == iter->trace && iter->trace->stop)
1090                 iter->trace->stop(iter);
1091
1092         mutex_unlock(&trace_types_lock);
1093 }
1094
1095 #define KRETPROBE_MSG "[unknown/kretprobe'd]"
1096
1097 #ifdef CONFIG_KRETPROBES
1098 static inline int kretprobed(unsigned long addr)
1099 {
1100         return addr == (unsigned long)kretprobe_trampoline;
1101 }
1102 #else
1103 static inline int kretprobed(unsigned long addr)
1104 {
1105         return 0;
1106 }
1107 #endif /* CONFIG_KRETPROBES */
1108
1109 static int
1110 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1111 {
1112 #ifdef CONFIG_KALLSYMS
1113         char str[KSYM_SYMBOL_LEN];
1114
1115         kallsyms_lookup(address, NULL, NULL, NULL, str);
1116
1117         return trace_seq_printf(s, fmt, str);
1118 #endif
1119         return 1;
1120 }
1121
1122 static int
1123 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1124                      unsigned long address)
1125 {
1126 #ifdef CONFIG_KALLSYMS
1127         char str[KSYM_SYMBOL_LEN];
1128
1129         sprint_symbol(str, address);
1130         return trace_seq_printf(s, fmt, str);
1131 #endif
1132         return 1;
1133 }
1134
1135 #ifndef CONFIG_64BIT
1136 # define IP_FMT "%08lx"
1137 #else
1138 # define IP_FMT "%016lx"
1139 #endif
1140
1141 static int
1142 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1143 {
1144         int ret;
1145
1146         if (!ip)
1147                 return trace_seq_printf(s, "0");
1148
1149         if (sym_flags & TRACE_ITER_SYM_OFFSET)
1150                 ret = seq_print_sym_offset(s, "%s", ip);
1151         else
1152                 ret = seq_print_sym_short(s, "%s", ip);
1153
1154         if (!ret)
1155                 return 0;
1156
1157         if (sym_flags & TRACE_ITER_SYM_ADDR)
1158                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1159         return ret;
1160 }
1161
1162 static void print_lat_help_header(struct seq_file *m)
1163 {
1164         seq_puts(m, "#                  _------=> CPU#            \n");
1165         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1166         seq_puts(m, "#                | / _----=> need-resched    \n");
1167         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1168         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1169         seq_puts(m, "#                |||| /                      \n");
1170         seq_puts(m, "#                |||||     delay             \n");
1171         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1172         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1173 }
1174
1175 static void print_func_help_header(struct seq_file *m)
1176 {
1177         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1178         seq_puts(m, "#              | |       |          |         |\n");
1179 }
1180
1181
1182 static void
1183 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1184 {
1185         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1186         struct trace_array *tr = iter->tr;
1187         struct trace_array_cpu *data = tr->data[tr->cpu];
1188         struct tracer *type = current_trace;
1189         unsigned long total;
1190         unsigned long entries;
1191         const char *name = "preemption";
1192
1193         if (type)
1194                 name = type->name;
1195
1196         entries = ring_buffer_entries(iter->tr->buffer);
1197         total = entries +
1198                 ring_buffer_overruns(iter->tr->buffer);
1199
1200         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1201                    name, UTS_RELEASE);
1202         seq_puts(m, "-----------------------------------"
1203                  "---------------------------------\n");
1204         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1205                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1206                    nsecs_to_usecs(data->saved_latency),
1207                    entries,
1208                    total,
1209                    tr->cpu,
1210 #if defined(CONFIG_PREEMPT_NONE)
1211                    "server",
1212 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1213                    "desktop",
1214 #elif defined(CONFIG_PREEMPT)
1215                    "preempt",
1216 #else
1217                    "unknown",
1218 #endif
1219                    /* These are reserved for later use */
1220                    0, 0, 0, 0);
1221 #ifdef CONFIG_SMP
1222         seq_printf(m, " #P:%d)\n", num_online_cpus());
1223 #else
1224         seq_puts(m, ")\n");
1225 #endif
1226         seq_puts(m, "    -----------------\n");
1227         seq_printf(m, "    | task: %.16s-%d "
1228                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1229                    data->comm, data->pid, data->uid, data->nice,
1230                    data->policy, data->rt_priority);
1231         seq_puts(m, "    -----------------\n");
1232
1233         if (data->critical_start) {
1234                 seq_puts(m, " => started at: ");
1235                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1236                 trace_print_seq(m, &iter->seq);
1237                 seq_puts(m, "\n => ended at:   ");
1238                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1239                 trace_print_seq(m, &iter->seq);
1240                 seq_puts(m, "\n");
1241         }
1242
1243         seq_puts(m, "\n");
1244 }
1245
1246 static void
1247 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1248 {
1249         int hardirq, softirq;
1250         char *comm;
1251
1252         comm = trace_find_cmdline(entry->pid);
1253
1254         trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1255         trace_seq_printf(s, "%3d", cpu);
1256         trace_seq_printf(s, "%c%c",
1257                         (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1258                         ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1259
1260         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1261         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1262         if (hardirq && softirq) {
1263                 trace_seq_putc(s, 'H');
1264         } else {
1265                 if (hardirq) {
1266                         trace_seq_putc(s, 'h');
1267                 } else {
1268                         if (softirq)
1269                                 trace_seq_putc(s, 's');
1270                         else
1271                                 trace_seq_putc(s, '.');
1272                 }
1273         }
1274
1275         if (entry->preempt_count)
1276                 trace_seq_printf(s, "%x", entry->preempt_count);
1277         else
1278                 trace_seq_puts(s, ".");
1279 }
1280
1281 unsigned long preempt_mark_thresh = 100;
1282
1283 static void
1284 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1285                     unsigned long rel_usecs)
1286 {
1287         trace_seq_printf(s, " %4lldus", abs_usecs);
1288         if (rel_usecs > preempt_mark_thresh)
1289                 trace_seq_puts(s, "!: ");
1290         else if (rel_usecs > 1)
1291                 trace_seq_puts(s, "+: ");
1292         else
1293                 trace_seq_puts(s, " : ");
1294 }
1295
1296 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1297
1298 /*
1299  * The message is supposed to contain an ending newline.
1300  * If the printing stops prematurely, try to add a newline of our own.
1301  */
1302 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1303 {
1304         struct trace_entry *ent;
1305         struct trace_field_cont *cont;
1306         bool ok = true;
1307
1308         ent = peek_next_entry(iter, iter->cpu, NULL);
1309         if (!ent || ent->type != TRACE_CONT) {
1310                 trace_seq_putc(s, '\n');
1311                 return;
1312         }
1313
1314         do {
1315                 cont = (struct trace_field_cont *)ent;
1316                 if (ok)
1317                         ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1318
1319                 ftrace_disable_cpu();
1320
1321                 if (iter->buffer_iter[iter->cpu])
1322                         ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1323                 else
1324                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1325
1326                 ftrace_enable_cpu();
1327
1328                 ent = peek_next_entry(iter, iter->cpu, NULL);
1329         } while (ent && ent->type == TRACE_CONT);
1330
1331         if (!ok)
1332                 trace_seq_putc(s, '\n');
1333 }
1334
1335 static enum print_line_t
1336 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1337 {
1338         struct trace_seq *s = &iter->seq;
1339         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1340         struct trace_entry *next_entry;
1341         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1342         struct trace_entry *entry = iter->ent;
1343         unsigned long abs_usecs;
1344         unsigned long rel_usecs;
1345         u64 next_ts;
1346         char *comm;
1347         int S, T;
1348         int i;
1349         unsigned state;
1350
1351         if (entry->type == TRACE_CONT)
1352                 return TRACE_TYPE_HANDLED;
1353
1354         next_entry = find_next_entry(iter, NULL, &next_ts);
1355         if (!next_entry)
1356                 next_ts = iter->ts;
1357         rel_usecs = ns2usecs(next_ts - iter->ts);
1358         abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1359
1360         if (verbose) {
1361                 comm = trace_find_cmdline(entry->pid);
1362                 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1363                                  " %ld.%03ldms (+%ld.%03ldms): ",
1364                                  comm,
1365                                  entry->pid, cpu, entry->flags,
1366                                  entry->preempt_count, trace_idx,
1367                                  ns2usecs(iter->ts),
1368                                  abs_usecs/1000,
1369                                  abs_usecs % 1000, rel_usecs/1000,
1370                                  rel_usecs % 1000);
1371         } else {
1372                 lat_print_generic(s, entry, cpu);
1373                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1374         }
1375         switch (entry->type) {
1376         case TRACE_FN: {
1377                 struct ftrace_entry *field;
1378
1379                 trace_assign_type(field, entry);
1380
1381                 seq_print_ip_sym(s, field->ip, sym_flags);
1382                 trace_seq_puts(s, " (");
1383                 if (kretprobed(field->parent_ip))
1384                         trace_seq_puts(s, KRETPROBE_MSG);
1385                 else
1386                         seq_print_ip_sym(s, field->parent_ip, sym_flags);
1387                 trace_seq_puts(s, ")\n");
1388                 break;
1389         }
1390         case TRACE_CTX:
1391         case TRACE_WAKE: {
1392                 struct ctx_switch_entry *field;
1393
1394                 trace_assign_type(field, entry);
1395
1396                 T = field->next_state < sizeof(state_to_char) ?
1397                         state_to_char[field->next_state] : 'X';
1398
1399                 state = field->prev_state ?
1400                         __ffs(field->prev_state) + 1 : 0;
1401                 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1402                 comm = trace_find_cmdline(field->next_pid);
1403                 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1404                                  field->prev_pid,
1405                                  field->prev_prio,
1406                                  S, entry->type == TRACE_CTX ? "==>" : "  +",
1407                                  field->next_cpu,
1408                                  field->next_pid,
1409                                  field->next_prio,
1410                                  T, comm);
1411                 break;
1412         }
1413         case TRACE_SPECIAL: {
1414                 struct special_entry *field;
1415
1416                 trace_assign_type(field, entry);
1417
1418                 trace_seq_printf(s, "# %ld %ld %ld\n",
1419                                  field->arg1,
1420                                  field->arg2,
1421                                  field->arg3);
1422                 break;
1423         }
1424         case TRACE_STACK: {
1425                 struct stack_entry *field;
1426
1427                 trace_assign_type(field, entry);
1428
1429                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1430                         if (i)
1431                                 trace_seq_puts(s, " <= ");
1432                         seq_print_ip_sym(s, field->caller[i], sym_flags);
1433                 }
1434                 trace_seq_puts(s, "\n");
1435                 break;
1436         }
1437         case TRACE_PRINT: {
1438                 struct print_entry *field;
1439
1440                 trace_assign_type(field, entry);
1441
1442                 seq_print_ip_sym(s, field->ip, sym_flags);
1443                 trace_seq_printf(s, ": %s", field->buf);
1444                 if (entry->flags & TRACE_FLAG_CONT)
1445                         trace_seq_print_cont(s, iter);
1446                 break;
1447         }
1448         default:
1449                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1450         }
1451         return TRACE_TYPE_HANDLED;
1452 }
1453
1454 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1455 {
1456         struct trace_seq *s = &iter->seq;
1457         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1458         struct trace_entry *entry;
1459         unsigned long usec_rem;
1460         unsigned long long t;
1461         unsigned long secs;
1462         char *comm;
1463         int ret;
1464         int S, T;
1465         int i;
1466
1467         entry = iter->ent;
1468
1469         if (entry->type == TRACE_CONT)
1470                 return TRACE_TYPE_HANDLED;
1471
1472         comm = trace_find_cmdline(iter->ent->pid);
1473
1474         t = ns2usecs(iter->ts);
1475         usec_rem = do_div(t, 1000000ULL);
1476         secs = (unsigned long)t;
1477
1478         ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1479         if (!ret)
1480                 return TRACE_TYPE_PARTIAL_LINE;
1481         ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1482         if (!ret)
1483                 return TRACE_TYPE_PARTIAL_LINE;
1484         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1485         if (!ret)
1486                 return TRACE_TYPE_PARTIAL_LINE;
1487
1488         switch (entry->type) {
1489         case TRACE_FN: {
1490                 struct ftrace_entry *field;
1491
1492                 trace_assign_type(field, entry);
1493
1494                 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1495                 if (!ret)
1496                         return TRACE_TYPE_PARTIAL_LINE;
1497                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1498                                                 field->parent_ip) {
1499                         ret = trace_seq_printf(s, " <-");
1500                         if (!ret)
1501                                 return TRACE_TYPE_PARTIAL_LINE;
1502                         if (kretprobed(field->parent_ip))
1503                                 ret = trace_seq_puts(s, KRETPROBE_MSG);
1504                         else
1505                                 ret = seq_print_ip_sym(s,
1506                                                        field->parent_ip,
1507                                                        sym_flags);
1508                         if (!ret)
1509                                 return TRACE_TYPE_PARTIAL_LINE;
1510                 }
1511                 ret = trace_seq_printf(s, "\n");
1512                 if (!ret)
1513                         return TRACE_TYPE_PARTIAL_LINE;
1514                 break;
1515         }
1516         case TRACE_CTX:
1517         case TRACE_WAKE: {
1518                 struct ctx_switch_entry *field;
1519
1520                 trace_assign_type(field, entry);
1521
1522                 S = field->prev_state < sizeof(state_to_char) ?
1523                         state_to_char[field->prev_state] : 'X';
1524                 T = field->next_state < sizeof(state_to_char) ?
1525                         state_to_char[field->next_state] : 'X';
1526                 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1527                                        field->prev_pid,
1528                                        field->prev_prio,
1529                                        S,
1530                                        entry->type == TRACE_CTX ? "==>" : "  +",
1531                                        field->next_cpu,
1532                                        field->next_pid,
1533                                        field->next_prio,
1534                                        T);
1535                 if (!ret)
1536                         return TRACE_TYPE_PARTIAL_LINE;
1537                 break;
1538         }
1539         case TRACE_SPECIAL: {
1540                 struct special_entry *field;
1541
1542                 trace_assign_type(field, entry);
1543
1544                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1545                                  field->arg1,
1546                                  field->arg2,
1547                                  field->arg3);
1548                 if (!ret)
1549                         return TRACE_TYPE_PARTIAL_LINE;
1550                 break;
1551         }
1552         case TRACE_STACK: {
1553                 struct stack_entry *field;
1554
1555                 trace_assign_type(field, entry);
1556
1557                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1558                         if (i) {
1559                                 ret = trace_seq_puts(s, " <= ");
1560                                 if (!ret)
1561                                         return TRACE_TYPE_PARTIAL_LINE;
1562                         }
1563                         ret = seq_print_ip_sym(s, field->caller[i],
1564                                                sym_flags);
1565                         if (!ret)
1566                                 return TRACE_TYPE_PARTIAL_LINE;
1567                 }
1568                 ret = trace_seq_puts(s, "\n");
1569                 if (!ret)
1570                         return TRACE_TYPE_PARTIAL_LINE;
1571                 break;
1572         }
1573         case TRACE_PRINT: {
1574                 struct print_entry *field;
1575
1576                 trace_assign_type(field, entry);
1577
1578                 seq_print_ip_sym(s, field->ip, sym_flags);
1579                 trace_seq_printf(s, ": %s", field->buf);
1580                 if (entry->flags & TRACE_FLAG_CONT)
1581                         trace_seq_print_cont(s, iter);
1582                 break;
1583         }
1584         }
1585         return TRACE_TYPE_HANDLED;
1586 }
1587
1588 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1589 {
1590         struct trace_seq *s = &iter->seq;
1591         struct trace_entry *entry;
1592         int ret;
1593         int S, T;
1594
1595         entry = iter->ent;
1596
1597         if (entry->type == TRACE_CONT)
1598                 return TRACE_TYPE_HANDLED;
1599
1600         ret = trace_seq_printf(s, "%d %d %llu ",
1601                 entry->pid, iter->cpu, iter->ts);
1602         if (!ret)
1603                 return TRACE_TYPE_PARTIAL_LINE;
1604
1605         switch (entry->type) {
1606         case TRACE_FN: {
1607                 struct ftrace_entry *field;
1608
1609                 trace_assign_type(field, entry);
1610
1611                 ret = trace_seq_printf(s, "%x %x\n",
1612                                         field->ip,
1613                                         field->parent_ip);
1614                 if (!ret)
1615                         return TRACE_TYPE_PARTIAL_LINE;
1616                 break;
1617         }
1618         case TRACE_CTX:
1619         case TRACE_WAKE: {
1620                 struct ctx_switch_entry *field;
1621
1622                 trace_assign_type(field, entry);
1623
1624                 S = field->prev_state < sizeof(state_to_char) ?
1625                         state_to_char[field->prev_state] : 'X';
1626                 T = field->next_state < sizeof(state_to_char) ?
1627                         state_to_char[field->next_state] : 'X';
1628                 if (entry->type == TRACE_WAKE)
1629                         S = '+';
1630                 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1631                                        field->prev_pid,
1632                                        field->prev_prio,
1633                                        S,
1634                                        field->next_cpu,
1635                                        field->next_pid,
1636                                        field->next_prio,
1637                                        T);
1638                 if (!ret)
1639                         return TRACE_TYPE_PARTIAL_LINE;
1640                 break;
1641         }
1642         case TRACE_SPECIAL:
1643         case TRACE_STACK: {
1644                 struct special_entry *field;
1645
1646                 trace_assign_type(field, entry);
1647
1648                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1649                                  field->arg1,
1650                                  field->arg2,
1651                                  field->arg3);
1652                 if (!ret)
1653                         return TRACE_TYPE_PARTIAL_LINE;
1654                 break;
1655         }
1656         case TRACE_PRINT: {
1657                 struct print_entry *field;
1658
1659                 trace_assign_type(field, entry);
1660
1661                 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1662                 if (entry->flags & TRACE_FLAG_CONT)
1663                         trace_seq_print_cont(s, iter);
1664                 break;
1665         }
1666         }
1667         return TRACE_TYPE_HANDLED;
1668 }
1669
1670 #define SEQ_PUT_FIELD_RET(s, x)                         \
1671 do {                                                    \
1672         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1673                 return 0;                               \
1674 } while (0)
1675
1676 #define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
1677 do {                                                    \
1678         if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
1679                 return 0;                               \
1680 } while (0)
1681
1682 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1683 {
1684         struct trace_seq *s = &iter->seq;
1685         unsigned char newline = '\n';
1686         struct trace_entry *entry;
1687         int S, T;
1688
1689         entry = iter->ent;
1690
1691         if (entry->type == TRACE_CONT)
1692                 return TRACE_TYPE_HANDLED;
1693
1694         SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1695         SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1696         SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1697
1698         switch (entry->type) {
1699         case TRACE_FN: {
1700                 struct ftrace_entry *field;
1701
1702                 trace_assign_type(field, entry);
1703
1704                 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1705                 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1706                 break;
1707         }
1708         case TRACE_CTX:
1709         case TRACE_WAKE: {
1710                 struct ctx_switch_entry *field;
1711
1712                 trace_assign_type(field, entry);
1713
1714                 S = field->prev_state < sizeof(state_to_char) ?
1715                         state_to_char[field->prev_state] : 'X';
1716                 T = field->next_state < sizeof(state_to_char) ?
1717                         state_to_char[field->next_state] : 'X';
1718                 if (entry->type == TRACE_WAKE)
1719                         S = '+';
1720                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1721                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1722                 SEQ_PUT_HEX_FIELD_RET(s, S);
1723                 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1724                 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1725                 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1726                 SEQ_PUT_HEX_FIELD_RET(s, T);
1727                 break;
1728         }
1729         case TRACE_SPECIAL:
1730         case TRACE_STACK: {
1731                 struct special_entry *field;
1732
1733                 trace_assign_type(field, entry);
1734
1735                 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1736                 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1737                 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1738                 break;
1739         }
1740         }
1741         SEQ_PUT_FIELD_RET(s, newline);
1742
1743         return TRACE_TYPE_HANDLED;
1744 }
1745
1746 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1747 {
1748         struct trace_seq *s = &iter->seq;
1749         struct trace_entry *entry;
1750
1751         entry = iter->ent;
1752
1753         if (entry->type == TRACE_CONT)
1754                 return TRACE_TYPE_HANDLED;
1755
1756         SEQ_PUT_FIELD_RET(s, entry->pid);
1757         SEQ_PUT_FIELD_RET(s, iter->cpu);
1758         SEQ_PUT_FIELD_RET(s, iter->ts);
1759
1760         switch (entry->type) {
1761         case TRACE_FN: {
1762                 struct ftrace_entry *field;
1763
1764                 trace_assign_type(field, entry);
1765
1766                 SEQ_PUT_FIELD_RET(s, field->ip);
1767                 SEQ_PUT_FIELD_RET(s, field->parent_ip);
1768                 break;
1769         }
1770         case TRACE_CTX: {
1771                 struct ctx_switch_entry *field;
1772
1773                 trace_assign_type(field, entry);
1774
1775                 SEQ_PUT_FIELD_RET(s, field->prev_pid);
1776                 SEQ_PUT_FIELD_RET(s, field->prev_prio);
1777                 SEQ_PUT_FIELD_RET(s, field->prev_state);
1778                 SEQ_PUT_FIELD_RET(s, field->next_pid);
1779                 SEQ_PUT_FIELD_RET(s, field->next_prio);
1780                 SEQ_PUT_FIELD_RET(s, field->next_state);
1781                 break;
1782         }
1783         case TRACE_SPECIAL:
1784         case TRACE_STACK: {
1785                 struct special_entry *field;
1786
1787                 trace_assign_type(field, entry);
1788
1789                 SEQ_PUT_FIELD_RET(s, field->arg1);
1790                 SEQ_PUT_FIELD_RET(s, field->arg2);
1791                 SEQ_PUT_FIELD_RET(s, field->arg3);
1792                 break;
1793         }
1794         }
1795         return 1;
1796 }
1797
1798 static int trace_empty(struct trace_iterator *iter)
1799 {
1800         int cpu;
1801
1802         for_each_tracing_cpu(cpu) {
1803                 if (iter->buffer_iter[cpu]) {
1804                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1805                                 return 0;
1806                 } else {
1807                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1808                                 return 0;
1809                 }
1810         }
1811
1812         return 1;
1813 }
1814
1815 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1816 {
1817         enum print_line_t ret;
1818
1819         if (iter->trace && iter->trace->print_line) {
1820                 ret = iter->trace->print_line(iter);
1821                 if (ret != TRACE_TYPE_UNHANDLED)
1822                         return ret;
1823         }
1824
1825         if (trace_flags & TRACE_ITER_BIN)
1826                 return print_bin_fmt(iter);
1827
1828         if (trace_flags & TRACE_ITER_HEX)
1829                 return print_hex_fmt(iter);
1830
1831         if (trace_flags & TRACE_ITER_RAW)
1832                 return print_raw_fmt(iter);
1833
1834         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1835                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1836
1837         return print_trace_fmt(iter);
1838 }
1839
1840 static int s_show(struct seq_file *m, void *v)
1841 {
1842         struct trace_iterator *iter = v;
1843
1844         if (iter->ent == NULL) {
1845                 if (iter->tr) {
1846                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1847                         seq_puts(m, "#\n");
1848                 }
1849                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1850                         /* print nothing if the buffers are empty */
1851                         if (trace_empty(iter))
1852                                 return 0;
1853                         print_trace_header(m, iter);
1854                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1855                                 print_lat_help_header(m);
1856                 } else {
1857                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1858                                 print_func_help_header(m);
1859                 }
1860         } else {
1861                 print_trace_line(iter);
1862                 trace_print_seq(m, &iter->seq);
1863         }
1864
1865         return 0;
1866 }
1867
1868 static struct seq_operations tracer_seq_ops = {
1869         .start          = s_start,
1870         .next           = s_next,
1871         .stop           = s_stop,
1872         .show           = s_show,
1873 };
1874
1875 static struct trace_iterator *
1876 __tracing_open(struct inode *inode, struct file *file, int *ret)
1877 {
1878         struct trace_iterator *iter;
1879         struct seq_file *m;
1880         int cpu;
1881
1882         if (tracing_disabled) {
1883                 *ret = -ENODEV;
1884                 return NULL;
1885         }
1886
1887         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1888         if (!iter) {
1889                 *ret = -ENOMEM;
1890                 goto out;
1891         }
1892
1893         mutex_lock(&trace_types_lock);
1894         if (current_trace && current_trace->print_max)
1895                 iter->tr = &max_tr;
1896         else
1897                 iter->tr = inode->i_private;
1898         iter->trace = current_trace;
1899         iter->pos = -1;
1900
1901         for_each_tracing_cpu(cpu) {
1902
1903                 iter->buffer_iter[cpu] =
1904                         ring_buffer_read_start(iter->tr->buffer, cpu);
1905
1906                 if (!iter->buffer_iter[cpu])
1907                         goto fail_buffer;
1908         }
1909
1910         /* TODO stop tracer */
1911         *ret = seq_open(file, &tracer_seq_ops);
1912         if (*ret)
1913                 goto fail_buffer;
1914
1915         m = file->private_data;
1916         m->private = iter;
1917
1918         /* stop the trace while dumping */
1919         if (iter->tr->ctrl) {
1920                 tracer_enabled = 0;
1921                 ftrace_function_enabled = 0;
1922         }
1923
1924         if (iter->trace && iter->trace->open)
1925                         iter->trace->open(iter);
1926
1927         mutex_unlock(&trace_types_lock);
1928
1929  out:
1930         return iter;
1931
1932  fail_buffer:
1933         for_each_tracing_cpu(cpu) {
1934                 if (iter->buffer_iter[cpu])
1935                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1936         }
1937         mutex_unlock(&trace_types_lock);
1938
1939         return ERR_PTR(-ENOMEM);
1940 }
1941
1942 int tracing_open_generic(struct inode *inode, struct file *filp)
1943 {
1944         if (tracing_disabled)
1945                 return -ENODEV;
1946
1947         filp->private_data = inode->i_private;
1948         return 0;
1949 }
1950
1951 int tracing_release(struct inode *inode, struct file *file)
1952 {
1953         struct seq_file *m = (struct seq_file *)file->private_data;
1954         struct trace_iterator *iter = m->private;
1955         int cpu;
1956
1957         mutex_lock(&trace_types_lock);
1958         for_each_tracing_cpu(cpu) {
1959                 if (iter->buffer_iter[cpu])
1960                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1961         }
1962
1963         if (iter->trace && iter->trace->close)
1964                 iter->trace->close(iter);
1965
1966         /* reenable tracing if it was previously enabled */
1967         if (iter->tr->ctrl) {
1968                 tracer_enabled = 1;
1969                 /*
1970                  * It is safe to enable function tracing even if it
1971                  * isn't used
1972                  */
1973                 ftrace_function_enabled = 1;
1974         }
1975         mutex_unlock(&trace_types_lock);
1976
1977         seq_release(inode, file);
1978         kfree(iter);
1979         return 0;
1980 }
1981
1982 static int tracing_open(struct inode *inode, struct file *file)
1983 {
1984         int ret;
1985
1986         __tracing_open(inode, file, &ret);
1987
1988         return ret;
1989 }
1990
1991 static int tracing_lt_open(struct inode *inode, struct file *file)
1992 {
1993         struct trace_iterator *iter;
1994         int ret;
1995
1996         iter = __tracing_open(inode, file, &ret);
1997
1998         if (!ret)
1999                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2000
2001         return ret;
2002 }
2003
2004
2005 static void *
2006 t_next(struct seq_file *m, void *v, loff_t *pos)
2007 {
2008         struct tracer *t = m->private;
2009
2010         (*pos)++;
2011
2012         if (t)
2013                 t = t->next;
2014
2015         m->private = t;
2016
2017         return t;
2018 }
2019
2020 static void *t_start(struct seq_file *m, loff_t *pos)
2021 {
2022         struct tracer *t = m->private;
2023         loff_t l = 0;
2024
2025         mutex_lock(&trace_types_lock);
2026         for (; t && l < *pos; t = t_next(m, t, &l))
2027                 ;
2028
2029         return t;
2030 }
2031
2032 static void t_stop(struct seq_file *m, void *p)
2033 {
2034         mutex_unlock(&trace_types_lock);
2035 }
2036
2037 static int t_show(struct seq_file *m, void *v)
2038 {
2039         struct tracer *t = v;
2040
2041         if (!t)
2042                 return 0;
2043
2044         seq_printf(m, "%s", t->name);
2045         if (t->next)
2046                 seq_putc(m, ' ');
2047         else
2048                 seq_putc(m, '\n');
2049
2050         return 0;
2051 }
2052
2053 static struct seq_operations show_traces_seq_ops = {
2054         .start          = t_start,
2055         .next           = t_next,
2056         .stop           = t_stop,
2057         .show           = t_show,
2058 };
2059
2060 static int show_traces_open(struct inode *inode, struct file *file)
2061 {
2062         int ret;
2063
2064         if (tracing_disabled)
2065                 return -ENODEV;
2066
2067         ret = seq_open(file, &show_traces_seq_ops);
2068         if (!ret) {
2069                 struct seq_file *m = file->private_data;
2070                 m->private = trace_types;
2071         }
2072
2073         return ret;
2074 }
2075
2076 static struct file_operations tracing_fops = {
2077         .open           = tracing_open,
2078         .read           = seq_read,
2079         .llseek         = seq_lseek,
2080         .release        = tracing_release,
2081 };
2082
2083 static struct file_operations tracing_lt_fops = {
2084         .open           = tracing_lt_open,
2085         .read           = seq_read,
2086         .llseek         = seq_lseek,
2087         .release        = tracing_release,
2088 };
2089
2090 static struct file_operations show_traces_fops = {
2091         .open           = show_traces_open,
2092         .read           = seq_read,
2093         .release        = seq_release,
2094 };
2095
2096 /*
2097  * Only trace on a CPU if the bitmask is set:
2098  */
2099 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2100
2101 /*
2102  * When tracing/tracing_cpu_mask is modified then this holds
2103  * the new bitmask we are about to install:
2104  */
2105 static cpumask_t tracing_cpumask_new;
2106
2107 /*
2108  * The tracer itself will not take this lock, but still we want
2109  * to provide a consistent cpumask to user-space:
2110  */
2111 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2112
2113 /*
2114  * Temporary storage for the character representation of the
2115  * CPU bitmask (and one more byte for the newline):
2116  */
2117 static char mask_str[NR_CPUS + 1];
2118
2119 static ssize_t
2120 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2121                      size_t count, loff_t *ppos)
2122 {
2123         int len;
2124
2125         mutex_lock(&tracing_cpumask_update_lock);
2126
2127         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2128         if (count - len < 2) {
2129                 count = -EINVAL;
2130                 goto out_err;
2131         }
2132         len += sprintf(mask_str + len, "\n");
2133         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2134
2135 out_err:
2136         mutex_unlock(&tracing_cpumask_update_lock);
2137
2138         return count;
2139 }
2140
2141 static ssize_t
2142 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2143                       size_t count, loff_t *ppos)
2144 {
2145         int err, cpu;
2146
2147         mutex_lock(&tracing_cpumask_update_lock);
2148         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2149         if (err)
2150                 goto err_unlock;
2151
2152         raw_local_irq_disable();
2153         __raw_spin_lock(&ftrace_max_lock);
2154         for_each_tracing_cpu(cpu) {
2155                 /*
2156                  * Increase/decrease the disabled counter if we are
2157                  * about to flip a bit in the cpumask:
2158                  */
2159                 if (cpu_isset(cpu, tracing_cpumask) &&
2160                                 !cpu_isset(cpu, tracing_cpumask_new)) {
2161                         atomic_inc(&global_trace.data[cpu]->disabled);
2162                 }
2163                 if (!cpu_isset(cpu, tracing_cpumask) &&
2164                                 cpu_isset(cpu, tracing_cpumask_new)) {
2165                         atomic_dec(&global_trace.data[cpu]->disabled);
2166                 }
2167         }
2168         __raw_spin_unlock(&ftrace_max_lock);
2169         raw_local_irq_enable();
2170
2171         tracing_cpumask = tracing_cpumask_new;
2172
2173         mutex_unlock(&tracing_cpumask_update_lock);
2174
2175         return count;
2176
2177 err_unlock:
2178         mutex_unlock(&tracing_cpumask_update_lock);
2179
2180         return err;
2181 }
2182
2183 static struct file_operations tracing_cpumask_fops = {
2184         .open           = tracing_open_generic,
2185         .read           = tracing_cpumask_read,
2186         .write          = tracing_cpumask_write,
2187 };
2188
2189 static ssize_t
2190 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2191                        size_t cnt, loff_t *ppos)
2192 {
2193         char *buf;
2194         int r = 0;
2195         int len = 0;
2196         int i;
2197
2198         /* calulate max size */
2199         for (i = 0; trace_options[i]; i++) {
2200                 len += strlen(trace_options[i]);
2201                 len += 3; /* "no" and space */
2202         }
2203
2204         /* +2 for \n and \0 */
2205         buf = kmalloc(len + 2, GFP_KERNEL);
2206         if (!buf)
2207                 return -ENOMEM;
2208
2209         for (i = 0; trace_options[i]; i++) {
2210                 if (trace_flags & (1 << i))
2211                         r += sprintf(buf + r, "%s ", trace_options[i]);
2212                 else
2213                         r += sprintf(buf + r, "no%s ", trace_options[i]);
2214         }
2215
2216         r += sprintf(buf + r, "\n");
2217         WARN_ON(r >= len + 2);
2218
2219         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2220
2221         kfree(buf);
2222
2223         return r;
2224 }
2225
2226 static ssize_t
2227 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2228                         size_t cnt, loff_t *ppos)
2229 {
2230         char buf[64];
2231         char *cmp = buf;
2232         int neg = 0;
2233         int i;
2234
2235         if (cnt >= sizeof(buf))
2236                 return -EINVAL;
2237
2238         if (copy_from_user(&buf, ubuf, cnt))
2239                 return -EFAULT;
2240
2241         buf[cnt] = 0;
2242
2243         if (strncmp(buf, "no", 2) == 0) {
2244                 neg = 1;
2245                 cmp += 2;
2246         }
2247
2248         for (i = 0; trace_options[i]; i++) {
2249                 int len = strlen(trace_options[i]);
2250
2251                 if (strncmp(cmp, trace_options[i], len) == 0) {
2252                         if (neg)
2253                                 trace_flags &= ~(1 << i);
2254                         else
2255                                 trace_flags |= (1 << i);
2256                         break;
2257                 }
2258         }
2259         /*
2260          * If no option could be set, return an error:
2261          */
2262         if (!trace_options[i])
2263                 return -EINVAL;
2264
2265         filp->f_pos += cnt;
2266
2267         return cnt;
2268 }
2269
2270 static struct file_operations tracing_iter_fops = {
2271         .open           = tracing_open_generic,
2272         .read           = tracing_iter_ctrl_read,
2273         .write          = tracing_iter_ctrl_write,
2274 };
2275
2276 static const char readme_msg[] =
2277         "tracing mini-HOWTO:\n\n"
2278         "# mkdir /debug\n"
2279         "# mount -t debugfs nodev /debug\n\n"
2280         "# cat /debug/tracing/available_tracers\n"
2281         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2282         "# cat /debug/tracing/current_tracer\n"
2283         "none\n"
2284         "# echo sched_switch > /debug/tracing/current_tracer\n"
2285         "# cat /debug/tracing/current_tracer\n"
2286         "sched_switch\n"
2287         "# cat /debug/tracing/iter_ctrl\n"
2288         "noprint-parent nosym-offset nosym-addr noverbose\n"
2289         "# echo print-parent > /debug/tracing/iter_ctrl\n"
2290         "# echo 1 > /debug/tracing/tracing_enabled\n"
2291         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2292         "echo 0 > /debug/tracing/tracing_enabled\n"
2293 ;
2294
2295 static ssize_t
2296 tracing_readme_read(struct file *filp, char __user *ubuf,
2297                        size_t cnt, loff_t *ppos)
2298 {
2299         return simple_read_from_buffer(ubuf, cnt, ppos,
2300                                         readme_msg, strlen(readme_msg));
2301 }
2302
2303 static struct file_operations tracing_readme_fops = {
2304         .open           = tracing_open_generic,
2305         .read           = tracing_readme_read,
2306 };
2307
2308 static ssize_t
2309 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2310                   size_t cnt, loff_t *ppos)
2311 {
2312         struct trace_array *tr = filp->private_data;
2313         char buf[64];
2314         int r;
2315
2316         r = sprintf(buf, "%ld\n", tr->ctrl);
2317         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2318 }
2319
2320 static ssize_t
2321 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2322                    size_t cnt, loff_t *ppos)
2323 {
2324         struct trace_array *tr = filp->private_data;
2325         char buf[64];
2326         long val;
2327         int ret;
2328
2329         if (cnt >= sizeof(buf))
2330                 return -EINVAL;
2331
2332         if (copy_from_user(&buf, ubuf, cnt))
2333                 return -EFAULT;
2334
2335         buf[cnt] = 0;
2336
2337         ret = strict_strtoul(buf, 10, &val);
2338         if (ret < 0)
2339                 return ret;
2340
2341         val = !!val;
2342
2343         mutex_lock(&trace_types_lock);
2344         if (tr->ctrl ^ val) {
2345                 if (val)
2346                         tracer_enabled = 1;
2347                 else
2348                         tracer_enabled = 0;
2349
2350                 tr->ctrl = val;
2351
2352                 if (current_trace && current_trace->ctrl_update)
2353                         current_trace->ctrl_update(tr);
2354         }
2355         mutex_unlock(&trace_types_lock);
2356
2357         filp->f_pos += cnt;
2358
2359         return cnt;
2360 }
2361
2362 static ssize_t
2363 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2364                        size_t cnt, loff_t *ppos)
2365 {
2366         char buf[max_tracer_type_len+2];
2367         int r;
2368
2369         mutex_lock(&trace_types_lock);
2370         if (current_trace)
2371                 r = sprintf(buf, "%s\n", current_trace->name);
2372         else
2373                 r = sprintf(buf, "\n");
2374         mutex_unlock(&trace_types_lock);
2375
2376         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2377 }
2378
2379 static ssize_t
2380 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2381                         size_t cnt, loff_t *ppos)
2382 {
2383         struct trace_array *tr = &global_trace;
2384         struct tracer *t;
2385         char buf[max_tracer_type_len+1];
2386         int i;
2387
2388         if (cnt > max_tracer_type_len)
2389                 cnt = max_tracer_type_len;
2390
2391         if (copy_from_user(&buf, ubuf, cnt))
2392                 return -EFAULT;
2393
2394         buf[cnt] = 0;
2395
2396         /* strip ending whitespace. */
2397         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2398                 buf[i] = 0;
2399
2400         mutex_lock(&trace_types_lock);
2401         for (t = trace_types; t; t = t->next) {
2402                 if (strcmp(t->name, buf) == 0)
2403                         break;
2404         }
2405         if (!t || t == current_trace)
2406                 goto out;
2407
2408         if (current_trace && current_trace->reset)
2409                 current_trace->reset(tr);
2410
2411         current_trace = t;
2412         if (t->init)
2413                 t->init(tr);
2414
2415  out:
2416         mutex_unlock(&trace_types_lock);
2417
2418         filp->f_pos += cnt;
2419
2420         return cnt;
2421 }
2422
2423 static ssize_t
2424 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2425                      size_t cnt, loff_t *ppos)
2426 {
2427         unsigned long *ptr = filp->private_data;
2428         char buf[64];
2429         int r;
2430
2431         r = snprintf(buf, sizeof(buf), "%ld\n",
2432                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2433         if (r > sizeof(buf))
2434                 r = sizeof(buf);
2435         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2436 }
2437
2438 static ssize_t
2439 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2440                       size_t cnt, loff_t *ppos)
2441 {
2442         long *ptr = filp->private_data;
2443         char buf[64];
2444         long val;
2445         int ret;
2446
2447         if (cnt >= sizeof(buf))
2448                 return -EINVAL;
2449
2450         if (copy_from_user(&buf, ubuf, cnt))
2451                 return -EFAULT;
2452
2453         buf[cnt] = 0;
2454
2455         ret = strict_strtoul(buf, 10, &val);
2456         if (ret < 0)
2457                 return ret;
2458
2459         *ptr = val * 1000;
2460
2461         return cnt;
2462 }
2463
2464 static atomic_t tracing_reader;
2465
2466 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2467 {
2468         struct trace_iterator *iter;
2469
2470         if (tracing_disabled)
2471                 return -ENODEV;
2472
2473         /* We only allow for reader of the pipe */
2474         if (atomic_inc_return(&tracing_reader) != 1) {
2475                 atomic_dec(&tracing_reader);
2476                 return -EBUSY;
2477         }
2478
2479         /* create a buffer to store the information to pass to userspace */
2480         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2481         if (!iter)
2482                 return -ENOMEM;
2483
2484         mutex_lock(&trace_types_lock);
2485         iter->tr = &global_trace;
2486         iter->trace = current_trace;
2487         filp->private_data = iter;
2488
2489         if (iter->trace->pipe_open)
2490                 iter->trace->pipe_open(iter);
2491         mutex_unlock(&trace_types_lock);
2492
2493         return 0;
2494 }
2495
2496 static int tracing_release_pipe(struct inode *inode, struct file *file)
2497 {
2498         struct trace_iterator *iter = file->private_data;
2499
2500         kfree(iter);
2501         atomic_dec(&tracing_reader);
2502
2503         return 0;
2504 }
2505
2506 static unsigned int
2507 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2508 {
2509         struct trace_iterator *iter = filp->private_data;
2510
2511         if (trace_flags & TRACE_ITER_BLOCK) {
2512                 /*
2513                  * Always select as readable when in blocking mode
2514                  */
2515                 return POLLIN | POLLRDNORM;
2516         } else {
2517                 if (!trace_empty(iter))
2518                         return POLLIN | POLLRDNORM;
2519                 poll_wait(filp, &trace_wait, poll_table);
2520                 if (!trace_empty(iter))
2521                         return POLLIN | POLLRDNORM;
2522
2523                 return 0;
2524         }
2525 }
2526
2527 /*
2528  * Consumer reader.
2529  */
2530 static ssize_t
2531 tracing_read_pipe(struct file *filp, char __user *ubuf,
2532                   size_t cnt, loff_t *ppos)
2533 {
2534         struct trace_iterator *iter = filp->private_data;
2535         ssize_t sret;
2536
2537         /* return any leftover data */
2538         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2539         if (sret != -EBUSY)
2540                 return sret;
2541
2542         trace_seq_reset(&iter->seq);
2543
2544         mutex_lock(&trace_types_lock);
2545         if (iter->trace->read) {
2546                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2547                 if (sret)
2548                         goto out;
2549         }
2550
2551 waitagain:
2552         sret = 0;
2553         while (trace_empty(iter)) {
2554
2555                 if ((filp->f_flags & O_NONBLOCK)) {
2556                         sret = -EAGAIN;
2557                         goto out;
2558                 }
2559
2560                 /*
2561                  * This is a make-shift waitqueue. The reason we don't use
2562                  * an actual wait queue is because:
2563                  *  1) we only ever have one waiter
2564                  *  2) the tracing, traces all functions, we don't want
2565                  *     the overhead of calling wake_up and friends
2566                  *     (and tracing them too)
2567                  *     Anyway, this is really very primitive wakeup.
2568                  */
2569                 set_current_state(TASK_INTERRUPTIBLE);
2570                 iter->tr->waiter = current;
2571
2572                 mutex_unlock(&trace_types_lock);
2573
2574                 /* sleep for 100 msecs, and try again. */
2575                 schedule_timeout(HZ/10);
2576
2577                 mutex_lock(&trace_types_lock);
2578
2579                 iter->tr->waiter = NULL;
2580
2581                 if (signal_pending(current)) {
2582                         sret = -EINTR;
2583                         goto out;
2584                 }
2585
2586                 if (iter->trace != current_trace)
2587                         goto out;
2588
2589                 /*
2590                  * We block until we read something and tracing is disabled.
2591                  * We still block if tracing is disabled, but we have never
2592                  * read anything. This allows a user to cat this file, and
2593                  * then enable tracing. But after we have read something,
2594                  * we give an EOF when tracing is again disabled.
2595                  *
2596                  * iter->pos will be 0 if we haven't read anything.
2597                  */
2598                 if (!tracer_enabled && iter->pos)
2599                         break;
2600
2601                 continue;
2602         }
2603
2604         /* stop when tracing is finished */
2605         if (trace_empty(iter))
2606                 goto out;
2607
2608         if (cnt >= PAGE_SIZE)
2609                 cnt = PAGE_SIZE - 1;
2610
2611         /* reset all but tr, trace, and overruns */
2612         memset(&iter->seq, 0,
2613                sizeof(struct trace_iterator) -
2614                offsetof(struct trace_iterator, seq));
2615         iter->pos = -1;
2616
2617         while (find_next_entry_inc(iter) != NULL) {
2618                 enum print_line_t ret;
2619                 int len = iter->seq.len;
2620
2621                 ret = print_trace_line(iter);
2622                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2623                         /* don't print partial lines */
2624                         iter->seq.len = len;
2625                         break;
2626                 }
2627
2628                 trace_consume(iter);
2629
2630                 if (iter->seq.len >= cnt)
2631                         break;
2632         }
2633
2634         /* Now copy what we have to the user */
2635         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2636         if (iter->seq.readpos >= iter->seq.len)
2637                 trace_seq_reset(&iter->seq);
2638
2639         /*
2640          * If there was nothing to send to user, inspite of consuming trace
2641          * entries, go back to wait for more entries.
2642          */
2643         if (sret == -EBUSY)
2644                 goto waitagain;
2645
2646 out:
2647         mutex_unlock(&trace_types_lock);
2648
2649         return sret;
2650 }
2651
2652 static ssize_t
2653 tracing_entries_read(struct file *filp, char __user *ubuf,
2654                      size_t cnt, loff_t *ppos)
2655 {
2656         struct trace_array *tr = filp->private_data;
2657         char buf[64];
2658         int r;
2659
2660         r = sprintf(buf, "%lu\n", tr->entries);
2661         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2662 }
2663
2664 static ssize_t
2665 tracing_entries_write(struct file *filp, const char __user *ubuf,
2666                       size_t cnt, loff_t *ppos)
2667 {
2668         unsigned long val;
2669         char buf[64];
2670         int ret;
2671         struct trace_array *tr = filp->private_data;
2672
2673         if (cnt >= sizeof(buf))
2674                 return -EINVAL;
2675
2676         if (copy_from_user(&buf, ubuf, cnt))
2677                 return -EFAULT;
2678
2679         buf[cnt] = 0;
2680
2681         ret = strict_strtoul(buf, 10, &val);
2682         if (ret < 0)
2683                 return ret;
2684
2685         /* must have at least 1 entry */
2686         if (!val)
2687                 return -EINVAL;
2688
2689         mutex_lock(&trace_types_lock);
2690
2691         if (tr->ctrl) {
2692                 cnt = -EBUSY;
2693                 pr_info("ftrace: please disable tracing"
2694                         " before modifying buffer size\n");
2695                 goto out;
2696         }
2697
2698         if (val != global_trace.entries) {
2699                 ret = ring_buffer_resize(global_trace.buffer, val);
2700                 if (ret < 0) {
2701                         cnt = ret;
2702                         goto out;
2703                 }
2704
2705                 ret = ring_buffer_resize(max_tr.buffer, val);
2706                 if (ret < 0) {
2707                         int r;
2708                         cnt = ret;
2709                         r = ring_buffer_resize(global_trace.buffer,
2710                                                global_trace.entries);
2711                         if (r < 0) {
2712                                 /* AARGH! We are left with different
2713                                  * size max buffer!!!! */
2714                                 WARN_ON(1);
2715                                 tracing_disabled = 1;
2716                         }
2717                         goto out;
2718                 }
2719
2720                 global_trace.entries = val;
2721         }
2722
2723         filp->f_pos += cnt;
2724
2725         /* If check pages failed, return ENOMEM */
2726         if (tracing_disabled)
2727                 cnt = -ENOMEM;
2728  out:
2729         max_tr.entries = global_trace.entries;
2730         mutex_unlock(&trace_types_lock);
2731
2732         return cnt;
2733 }
2734
2735 static int mark_printk(const char *fmt, ...)
2736 {
2737         int ret;
2738         va_list args;
2739         va_start(args, fmt);
2740         ret = trace_vprintk(0, fmt, args);
2741         va_end(args);
2742         return ret;
2743 }
2744
2745 static ssize_t
2746 tracing_mark_write(struct file *filp, const char __user *ubuf,
2747                                         size_t cnt, loff_t *fpos)
2748 {
2749         char *buf;
2750         char *end;
2751         struct trace_array *tr = &global_trace;
2752
2753         if (!tr->ctrl || tracing_disabled)
2754                 return -EINVAL;
2755
2756         if (cnt > TRACE_BUF_SIZE)
2757                 cnt = TRACE_BUF_SIZE;
2758
2759         buf = kmalloc(cnt + 1, GFP_KERNEL);
2760         if (buf == NULL)
2761                 return -ENOMEM;
2762
2763         if (copy_from_user(buf, ubuf, cnt)) {
2764                 kfree(buf);
2765                 return -EFAULT;
2766         }
2767
2768         /* Cut from the first nil or newline. */
2769         buf[cnt] = '\0';
2770         end = strchr(buf, '\n');
2771         if (end)
2772                 *end = '\0';
2773
2774         cnt = mark_printk("%s\n", buf);
2775         kfree(buf);
2776         *fpos += cnt;
2777
2778         return cnt;
2779 }
2780
2781 static struct file_operations tracing_max_lat_fops = {
2782         .open           = tracing_open_generic,
2783         .read           = tracing_max_lat_read,
2784         .write          = tracing_max_lat_write,
2785 };
2786
2787 static struct file_operations tracing_ctrl_fops = {
2788         .open           = tracing_open_generic,
2789         .read           = tracing_ctrl_read,
2790         .write          = tracing_ctrl_write,
2791 };
2792
2793 static struct file_operations set_tracer_fops = {
2794         .open           = tracing_open_generic,
2795         .read           = tracing_set_trace_read,
2796         .write          = tracing_set_trace_write,
2797 };
2798
2799 static struct file_operations tracing_pipe_fops = {
2800         .open           = tracing_open_pipe,
2801         .poll           = tracing_poll_pipe,
2802         .read           = tracing_read_pipe,
2803         .release        = tracing_release_pipe,
2804 };
2805
2806 static struct file_operations tracing_entries_fops = {
2807         .open           = tracing_open_generic,
2808         .read           = tracing_entries_read,
2809         .write          = tracing_entries_write,
2810 };
2811
2812 static struct file_operations tracing_mark_fops = {
2813         .open           = tracing_open_generic,
2814         .write          = tracing_mark_write,
2815 };
2816
2817 #ifdef CONFIG_DYNAMIC_FTRACE
2818
2819 static ssize_t
2820 tracing_read_long(struct file *filp, char __user *ubuf,
2821                   size_t cnt, loff_t *ppos)
2822 {
2823         unsigned long *p = filp->private_data;
2824         char buf[64];
2825         int r;
2826
2827         r = sprintf(buf, "%ld\n", *p);
2828
2829         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2830 }
2831
2832 static struct file_operations tracing_read_long_fops = {
2833         .open           = tracing_open_generic,
2834         .read           = tracing_read_long,
2835 };
2836 #endif
2837
2838 static struct dentry *d_tracer;
2839
2840 struct dentry *tracing_init_dentry(void)
2841 {
2842         static int once;
2843
2844         if (d_tracer)
2845                 return d_tracer;
2846
2847         d_tracer = debugfs_create_dir("tracing", NULL);
2848
2849         if (!d_tracer && !once) {
2850                 once = 1;
2851                 pr_warning("Could not create debugfs directory 'tracing'\n");
2852                 return NULL;
2853         }
2854
2855         return d_tracer;
2856 }
2857
2858 #ifdef CONFIG_FTRACE_SELFTEST
2859 /* Let selftest have access to static functions in this file */
2860 #include "trace_selftest.c"
2861 #endif
2862
2863 static __init int tracer_init_debugfs(void)
2864 {
2865         struct dentry *d_tracer;
2866         struct dentry *entry;
2867
2868         d_tracer = tracing_init_dentry();
2869
2870         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2871                                     &global_trace, &tracing_ctrl_fops);
2872         if (!entry)
2873                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2874
2875         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2876                                     NULL, &tracing_iter_fops);
2877         if (!entry)
2878                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2879
2880         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2881                                     NULL, &tracing_cpumask_fops);
2882         if (!entry)
2883                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2884
2885         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2886                                     &global_trace, &tracing_lt_fops);
2887         if (!entry)
2888                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2889
2890         entry = debugfs_create_file("trace", 0444, d_tracer,
2891                                     &global_trace, &tracing_fops);
2892         if (!entry)
2893                 pr_warning("Could not create debugfs 'trace' entry\n");
2894
2895         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2896                                     &global_trace, &show_traces_fops);
2897         if (!entry)
2898                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
2899
2900         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2901                                     &global_trace, &set_tracer_fops);
2902         if (!entry)
2903                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
2904
2905         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2906                                     &tracing_max_latency,
2907                                     &tracing_max_lat_fops);
2908         if (!entry)
2909                 pr_warning("Could not create debugfs "
2910                            "'tracing_max_latency' entry\n");
2911
2912         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2913                                     &tracing_thresh, &tracing_max_lat_fops);
2914         if (!entry)
2915                 pr_warning("Could not create debugfs "
2916                            "'tracing_thresh' entry\n");
2917         entry = debugfs_create_file("README", 0644, d_tracer,
2918                                     NULL, &tracing_readme_fops);
2919         if (!entry)
2920                 pr_warning("Could not create debugfs 'README' entry\n");
2921
2922         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2923                                     NULL, &tracing_pipe_fops);
2924         if (!entry)
2925                 pr_warning("Could not create debugfs "
2926                            "'trace_pipe' entry\n");
2927
2928         entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2929                                     &global_trace, &tracing_entries_fops);
2930         if (!entry)
2931                 pr_warning("Could not create debugfs "
2932                            "'trace_entries' entry\n");
2933
2934         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
2935                                     NULL, &tracing_mark_fops);
2936         if (!entry)
2937                 pr_warning("Could not create debugfs "
2938                            "'trace_marker' entry\n");
2939
2940 #ifdef CONFIG_DYNAMIC_FTRACE
2941         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2942                                     &ftrace_update_tot_cnt,
2943                                     &tracing_read_long_fops);
2944         if (!entry)
2945                 pr_warning("Could not create debugfs "
2946                            "'dyn_ftrace_total_info' entry\n");
2947 #endif
2948 #ifdef CONFIG_SYSPROF_TRACER
2949         init_tracer_sysprof_debugfs(d_tracer);
2950 #endif
2951         return 0;
2952 }
2953
2954 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2955 {
2956         static DEFINE_SPINLOCK(trace_buf_lock);
2957         static char trace_buf[TRACE_BUF_SIZE];
2958
2959         struct ring_buffer_event *event;
2960         struct trace_array *tr = &global_trace;
2961         struct trace_array_cpu *data;
2962         struct print_entry *entry;
2963         unsigned long flags, irq_flags;
2964         long disabled;
2965         int cpu, len = 0, size, pc;
2966
2967         if (!tr->ctrl || tracing_disabled)
2968                 return 0;
2969
2970         pc = preempt_count();
2971         preempt_disable_notrace();
2972         cpu = raw_smp_processor_id();
2973         data = tr->data[cpu];
2974         disabled = atomic_inc_return(&data->disabled);
2975
2976         if (unlikely(disabled != 1))
2977                 goto out;
2978
2979         spin_lock_irqsave(&trace_buf_lock, flags);
2980         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
2981
2982         len = min(len, TRACE_BUF_SIZE-1);
2983         trace_buf[len] = 0;
2984
2985         size = sizeof(*entry) + len + 1;
2986         event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
2987         if (!event)
2988                 goto out_unlock;
2989         entry = ring_buffer_event_data(event);
2990         tracing_generic_entry_update(&entry->ent, flags, pc);
2991         entry->ent.type                 = TRACE_PRINT;
2992         entry->ip                       = ip;
2993
2994         memcpy(&entry->buf, trace_buf, len);
2995         entry->buf[len] = 0;
2996         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
2997
2998  out_unlock:
2999         spin_unlock_irqrestore(&trace_buf_lock, flags);
3000
3001  out:
3002         atomic_dec(&data->disabled);
3003         preempt_enable_notrace();
3004
3005         return len;
3006 }
3007 EXPORT_SYMBOL_GPL(trace_vprintk);
3008
3009 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3010 {
3011         int ret;
3012         va_list ap;
3013
3014         if (!(trace_flags & TRACE_ITER_PRINTK))
3015                 return 0;
3016
3017         va_start(ap, fmt);
3018         ret = trace_vprintk(ip, fmt, ap);
3019         va_end(ap);
3020         return ret;
3021 }
3022 EXPORT_SYMBOL_GPL(__ftrace_printk);
3023
3024 static int trace_panic_handler(struct notifier_block *this,
3025                                unsigned long event, void *unused)
3026 {
3027         ftrace_dump();
3028         return NOTIFY_OK;
3029 }
3030
3031 static struct notifier_block trace_panic_notifier = {
3032         .notifier_call  = trace_panic_handler,
3033         .next           = NULL,
3034         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
3035 };
3036
3037 static int trace_die_handler(struct notifier_block *self,
3038                              unsigned long val,
3039                              void *data)
3040 {
3041         switch (val) {
3042         case DIE_OOPS:
3043                 ftrace_dump();
3044                 break;
3045         default:
3046                 break;
3047         }
3048         return NOTIFY_OK;
3049 }
3050
3051 static struct notifier_block trace_die_notifier = {
3052         .notifier_call = trace_die_handler,
3053         .priority = 200
3054 };
3055
3056 /*
3057  * printk is set to max of 1024, we really don't need it that big.
3058  * Nothing should be printing 1000 characters anyway.
3059  */
3060 #define TRACE_MAX_PRINT         1000
3061
3062 /*
3063  * Define here KERN_TRACE so that we have one place to modify
3064  * it if we decide to change what log level the ftrace dump
3065  * should be at.
3066  */
3067 #define KERN_TRACE              KERN_INFO
3068
3069 static void
3070 trace_printk_seq(struct trace_seq *s)
3071 {
3072         /* Probably should print a warning here. */
3073         if (s->len >= 1000)
3074                 s->len = 1000;
3075
3076         /* should be zero ended, but we are paranoid. */
3077         s->buffer[s->len] = 0;
3078
3079         printk(KERN_TRACE "%s", s->buffer);
3080
3081         trace_seq_reset(s);
3082 }
3083
3084
3085 void ftrace_dump(void)
3086 {
3087         static DEFINE_SPINLOCK(ftrace_dump_lock);
3088         /* use static because iter can be a bit big for the stack */
3089         static struct trace_iterator iter;
3090         static cpumask_t mask;
3091         static int dump_ran;
3092         unsigned long flags;
3093         int cnt = 0, cpu;
3094
3095         /* only one dump */
3096         spin_lock_irqsave(&ftrace_dump_lock, flags);
3097         if (dump_ran)
3098                 goto out;
3099
3100         dump_ran = 1;
3101
3102         /* No turning back! */
3103         ftrace_kill_atomic();
3104
3105         for_each_tracing_cpu(cpu) {
3106                 atomic_inc(&global_trace.data[cpu]->disabled);
3107         }
3108
3109         printk(KERN_TRACE "Dumping ftrace buffer:\n");
3110
3111         iter.tr = &global_trace;
3112         iter.trace = current_trace;
3113
3114         /*
3115          * We need to stop all tracing on all CPUS to read the
3116          * the next buffer. This is a bit expensive, but is
3117          * not done often. We fill all what we can read,
3118          * and then release the locks again.
3119          */
3120
3121         cpus_clear(mask);
3122
3123         while (!trace_empty(&iter)) {
3124
3125                 if (!cnt)
3126                         printk(KERN_TRACE "---------------------------------\n");
3127
3128                 cnt++;
3129
3130                 /* reset all but tr, trace, and overruns */
3131                 memset(&iter.seq, 0,
3132                        sizeof(struct trace_iterator) -
3133                        offsetof(struct trace_iterator, seq));
3134                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3135                 iter.pos = -1;
3136
3137                 if (find_next_entry_inc(&iter) != NULL) {
3138                         print_trace_line(&iter);
3139                         trace_consume(&iter);
3140                 }
3141
3142                 trace_printk_seq(&iter.seq);
3143         }
3144
3145         if (!cnt)
3146                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3147         else
3148                 printk(KERN_TRACE "---------------------------------\n");
3149
3150  out:
3151         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3152 }
3153
3154 __init static int tracer_alloc_buffers(void)
3155 {
3156         struct trace_array_cpu *data;
3157         int i;
3158
3159         /* TODO: make the number of buffers hot pluggable with CPUS */
3160         tracing_buffer_mask = cpu_possible_map;
3161
3162         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3163                                                    TRACE_BUFFER_FLAGS);
3164         if (!global_trace.buffer) {
3165                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3166                 WARN_ON(1);
3167                 return 0;
3168         }
3169         global_trace.entries = ring_buffer_size(global_trace.buffer);
3170
3171 #ifdef CONFIG_TRACER_MAX_TRACE
3172         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3173                                              TRACE_BUFFER_FLAGS);
3174         if (!max_tr.buffer) {
3175                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3176                 WARN_ON(1);
3177                 ring_buffer_free(global_trace.buffer);
3178                 return 0;
3179         }
3180         max_tr.entries = ring_buffer_size(max_tr.buffer);
3181         WARN_ON(max_tr.entries != global_trace.entries);
3182 #endif
3183
3184         /* Allocate the first page for all buffers */
3185         for_each_tracing_cpu(i) {
3186                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3187                 max_tr.data[i] = &per_cpu(max_data, i);
3188         }
3189
3190         trace_init_cmdlines();
3191
3192         register_tracer(&nop_trace);
3193 #ifdef CONFIG_BOOT_TRACER
3194         register_tracer(&boot_tracer);
3195         current_trace = &boot_tracer;
3196         current_trace->init(&global_trace);
3197 #else
3198         current_trace = &nop_trace;
3199 #endif
3200
3201         /* All seems OK, enable tracing */
3202         global_trace.ctrl = tracer_enabled;
3203         tracing_disabled = 0;
3204
3205         atomic_notifier_chain_register(&panic_notifier_list,
3206                                        &trace_panic_notifier);
3207
3208         register_die_notifier(&trace_die_notifier);
3209
3210         return 0;
3211 }
3212 early_initcall(tracer_alloc_buffers);
3213 fs_initcall(tracer_init_debugfs);