git.oblomov.eu Git - linux-2.6/blob - kernel/trace/trace.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally taken from the RT patch by:
   8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code from the latency_tracer, that is:
  11  *  Copyright (C) 2004-2006 Ingo Molnar
  12  *  Copyright (C) 2004 William Lee Irwin III
  13  */
  14 #include <linux/utsrelease.h>
  15 #include <linux/kallsyms.h>
  16 #include <linux/seq_file.h>
  17 #include <linux/notifier.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/hardirq.h>
  21 #include <linux/linkage.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/ftrace.h>
  24 #include <linux/module.h>
  25 #include <linux/percpu.h>
  26 #include <linux/kdebug.h>
  27 #include <linux/ctype.h>
  28 #include <linux/init.h>
  29 #include <linux/poll.h>
  30 #include <linux/gfp.h>
  31 #include <linux/fs.h>
  32 #include <linux/kprobes.h>
  33 #include <linux/writeback.h>
  34
  35 #include <linux/stacktrace.h>
  36 #include <linux/ring_buffer.h>
  37 #include <linux/irqflags.h>
  38
  39 #include "trace.h"
  40
  41 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
  42
  43 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
  44 unsigned long __read_mostly     tracing_thresh;
  45
  46 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
  47
  48 static inline void ftrace_disable_cpu(void)
  49 {
  50         preempt_disable();
  51         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
  52 }
  53
  54 static inline void ftrace_enable_cpu(void)
  55 {
  56         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
  57         preempt_enable();
  58 }
  59
  60 static cpumask_t __read_mostly          tracing_buffer_mask;
  61
  62 #define for_each_tracing_cpu(cpu)       \
  63         for_each_cpu_mask(cpu, tracing_buffer_mask)
  64
  65 static int tracing_disabled = 1;
  66
  67 /*
  68  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  69  *
  70  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  71  * is set, then ftrace_dump is called. This will output the contents
  72  * of the ftrace buffers to the console.  This is very useful for
  73  * capturing traces that lead to crashes and outputing it to a
  74  * serial console.
  75  *
  76  * It is default off, but you can enable it with either specifying
  77  * "ftrace_dump_on_oops" in the kernel command line, or setting
  78  * /proc/sys/kernel/ftrace_dump_on_oops to true.
  79  */
  80 int ftrace_dump_on_oops;
  81
  82 static int tracing_set_tracer(char *buf);
  83
  84 static int __init set_ftrace(char *str)
  85 {
  86         tracing_set_tracer(str);
  87         return 1;
  88 }
  89 __setup("ftrace", set_ftrace);
  90
  91 static int __init set_ftrace_dump_on_oops(char *str)
  92 {
  93         ftrace_dump_on_oops = 1;
  94         return 1;
  95 }
  96 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
  97
  98 long
  99 ns2usecs(cycle_t nsec)
 100 {
 101         nsec += 500;
 102         do_div(nsec, 1000);
 103         return nsec;
 104 }
 105
 106 cycle_t ftrace_now(int cpu)
 107 {
 108         u64 ts = ring_buffer_time_stamp(cpu);
 109         ring_buffer_normalize_time_stamp(cpu, &ts);
 110         return ts;
 111 }
 112
 113 /*
 114  * The global_trace is the descriptor that holds the tracing
 115  * buffers for the live tracing. For each CPU, it contains
 116  * a link list of pages that will store trace entries. The
 117  * page descriptor of the pages in the memory is used to hold
 118  * the link list by linking the lru item in the page descriptor
 119  * to each of the pages in the buffer per CPU.
 120  *
 121  * For each active CPU there is a data field that holds the
 122  * pages for the buffer for that CPU. Each CPU has the same number
 123  * of pages allocated for its buffer.
 124  */
 125 static struct trace_array       global_trace;
 126
 127 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
 128
 129 /*
 130  * The max_tr is used to snapshot the global_trace when a maximum
 131  * latency is reached. Some tracers will use this to store a maximum
 132  * trace while it continues examining live traces.
 133  *
 134  * The buffers for the max_tr are set up the same as the global_trace.
 135  * When a snapshot is taken, the link list of the max_tr is swapped
 136  * with the link list of the global_trace and the buffers are reset for
 137  * the global_trace so the tracing can continue.
 138  */
 139 static struct trace_array       max_tr;
 140
 141 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
 142
 143 /* tracer_enabled is used to toggle activation of a tracer */
 144 static int                      tracer_enabled = 1;
 145
 146 /* function tracing enabled */
 147 int                             ftrace_function_enabled;
 148
 149 /*
 150  * trace_buf_size is the size in bytes that is allocated
 151  * for a buffer. Note, the number of bytes is always rounded
 152  * to page size.
 153  *
 154  * This number is purposely set to a low number of 16384.
 155  * If the dump on oops happens, it will be much appreciated
 156  * to not have to wait for all that output. Anyway this can be
 157  * boot time and run time configurable.
 158  */
 159 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 160
 161 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 162
 163 /* trace_types holds a link list of available tracers. */
 164 static struct tracer            *trace_types __read_mostly;
 165
 166 /* current_trace points to the tracer that is currently active */
 167 static struct tracer            *current_trace __read_mostly;
 168
 169 /*
 170  * max_tracer_type_len is used to simplify the allocating of
 171  * buffers to read userspace tracer names. We keep track of
 172  * the longest tracer name registered.
 173  */
 174 static int                      max_tracer_type_len;
 175
 176 /*
 177  * trace_types_lock is used to protect the trace_types list.
 178  * This lock is also used to keep user access serialized.
 179  * Accesses from userspace will grab this lock while userspace
 180  * activities happen inside the kernel.
 181  */
 182 static DEFINE_MUTEX(trace_types_lock);
 183
 184 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
 185 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 186
 187 /* trace_flags holds iter_ctrl options */
 188 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
 189
 190 /**
 191  * trace_wake_up - wake up tasks waiting for trace input
 192  *
 193  * Simply wakes up any task that is blocked on the trace_wait
 194  * queue. These is used with trace_poll for tasks polling the trace.
 195  */
 196 void trace_wake_up(void)
 197 {
 198         /*
 199          * The runqueue_is_locked() can fail, but this is the best we
 200          * have for now:
 201          */
 202         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
 203                 wake_up(&trace_wait);
 204 }
 205
 206 static int __init set_buf_size(char *str)
 207 {
 208         unsigned long buf_size;
 209         int ret;
 210
 211         if (!str)
 212                 return 0;
 213         ret = strict_strtoul(str, 0, &buf_size);
 214         /* nr_entries can not be zero */
 215         if (ret < 0 || buf_size == 0)
 216                 return 0;
 217         trace_buf_size = buf_size;
 218         return 1;
 219 }
 220 __setup("trace_buf_size=", set_buf_size);
 221
 222 unsigned long nsecs_to_usecs(unsigned long nsecs)
 223 {
 224         return nsecs / 1000;
 225 }
 226
 227 /*
 228  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
 229  * control the output of kernel symbols.
 230  */
 231 #define TRACE_ITER_SYM_MASK \
 232         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
 233
 234 /* These must match the bit postions in trace_iterator_flags */
 235 static const char *trace_options[] = {
 236         "print-parent",
 237         "sym-offset",
 238         "sym-addr",
 239         "verbose",
 240         "raw",
 241         "hex",
 242         "bin",
 243         "block",
 244         "stacktrace",
 245         "sched-tree",
 246         "ftrace_printk",
 247         "ftrace_preempt",
 248         NULL
 249 };
 250
 251 /*
 252  * ftrace_max_lock is used to protect the swapping of buffers
 253  * when taking a max snapshot. The buffers themselves are
 254  * protected by per_cpu spinlocks. But the action of the swap
 255  * needs its own lock.
 256  *
 257  * This is defined as a raw_spinlock_t in order to help
 258  * with performance when lockdep debugging is enabled.
 259  */
 260 static raw_spinlock_t ftrace_max_lock =
 261         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
 262
 263 /*
 264  * Copy the new maximum trace into the separate maximum-trace
 265  * structure. (this way the maximum trace is permanently saved,
 266  * for later retrieval via /debugfs/tracing/latency_trace)
 267  */
 268 static void
 269 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 270 {
 271         struct trace_array_cpu *data = tr->data[cpu];
 272
 273         max_tr.cpu = cpu;
 274         max_tr.time_start = data->preempt_timestamp;
 275
 276         data = max_tr.data[cpu];
 277         data->saved_latency = tracing_max_latency;
 278
 279         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
 280         data->pid = tsk->pid;
 281         data->uid = tsk->uid;
 282         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 283         data->policy = tsk->policy;
 284         data->rt_priority = tsk->rt_priority;
 285
 286         /* record this tasks comm */
 287         tracing_record_cmdline(current);
 288 }
 289
 290 /**
 291  * trace_seq_printf - sequence printing of trace information
 292  * @s: trace sequence descriptor
 293  * @fmt: printf format string
 294  *
 295  * The tracer may use either sequence operations or its own
 296  * copy to user routines. To simplify formating of a trace
 297  * trace_seq_printf is used to store strings into a special
 298  * buffer (@s). Then the output may be either used by
 299  * the sequencer or pulled into another buffer.
 300  */
 301 int
 302 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
 303 {
 304         int len = (PAGE_SIZE - 1) - s->len;
 305         va_list ap;
 306         int ret;
 307
 308         if (!len)
 309                 return 0;
 310
 311         va_start(ap, fmt);
 312         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
 313         va_end(ap);
 314
 315         /* If we can't write it all, don't bother writing anything */
 316         if (ret >= len)
 317                 return 0;
 318
 319         s->len += ret;
 320
 321         return len;
 322 }
 323
 324 /**
 325  * trace_seq_puts - trace sequence printing of simple string
 326  * @s: trace sequence descriptor
 327  * @str: simple string to record
 328  *
 329  * The tracer may use either the sequence operations or its own
 330  * copy to user routines. This function records a simple string
 331  * into a special buffer (@s) for later retrieval by a sequencer
 332  * or other mechanism.
 333  */
 334 static int
 335 trace_seq_puts(struct trace_seq *s, const char *str)
 336 {
 337         int len = strlen(str);
 338
 339         if (len > ((PAGE_SIZE - 1) - s->len))
 340                 return 0;
 341
 342         memcpy(s->buffer + s->len, str, len);
 343         s->len += len;
 344
 345         return len;
 346 }
 347
 348 static int
 349 trace_seq_putc(struct trace_seq *s, unsigned char c)
 350 {
 351         if (s->len >= (PAGE_SIZE - 1))
 352                 return 0;
 353
 354         s->buffer[s->len++] = c;
 355
 356         return 1;
 357 }
 358
 359 static int
 360 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
 361 {
 362         if (len > ((PAGE_SIZE - 1) - s->len))
 363                 return 0;
 364
 365         memcpy(s->buffer + s->len, mem, len);
 366         s->len += len;
 367
 368         return len;
 369 }
 370
 371 #define MAX_MEMHEX_BYTES        8
 372 #define HEX_CHARS               (MAX_MEMHEX_BYTES*2 + 1)
 373
 374 static int
 375 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
 376 {
 377         unsigned char hex[HEX_CHARS];
 378         unsigned char *data = mem;
 379         int i, j;
 380
 381 #ifdef __BIG_ENDIAN
 382         for (i = 0, j = 0; i < len; i++) {
 383 #else
 384         for (i = len-1, j = 0; i >= 0; i--) {
 385 #endif
 386                 hex[j++] = hex_asc_hi(data[i]);
 387                 hex[j++] = hex_asc_lo(data[i]);
 388         }
 389         hex[j++] = ' ';
 390
 391         return trace_seq_putmem(s, hex, j);
 392 }
 393
 394 static void
 395 trace_seq_reset(struct trace_seq *s)
 396 {
 397         s->len = 0;
 398         s->readpos = 0;
 399 }
 400
 401 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 402 {
 403         int len;
 404         int ret;
 405
 406         if (s->len <= s->readpos)
 407                 return -EBUSY;
 408
 409         len = s->len - s->readpos;
 410         if (cnt > len)
 411                 cnt = len;
 412         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 413         if (ret)
 414                 return -EFAULT;
 415
 416         s->readpos += len;
 417         return cnt;
 418 }
 419
 420 static void
 421 trace_print_seq(struct seq_file *m, struct trace_seq *s)
 422 {
 423         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
 424
 425         s->buffer[len] = 0;
 426         seq_puts(m, s->buffer);
 427
 428         trace_seq_reset(s);
 429 }
 430
 431 /**
 432  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 433  * @tr: tracer
 434  * @tsk: the task with the latency
 435  * @cpu: The cpu that initiated the trace.
 436  *
 437  * Flip the buffers between the @tr and the max_tr and record information
 438  * about which task was the cause of this latency.
 439  */
 440 void
 441 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 442 {
 443         struct ring_buffer *buf = tr->buffer;
 444
 445         WARN_ON_ONCE(!irqs_disabled());
 446         __raw_spin_lock(&ftrace_max_lock);
 447
 448         tr->buffer = max_tr.buffer;
 449         max_tr.buffer = buf;
 450
 451         ftrace_disable_cpu();
 452         ring_buffer_reset(tr->buffer);
 453         ftrace_enable_cpu();
 454
 455         __update_max_tr(tr, tsk, cpu);
 456         __raw_spin_unlock(&ftrace_max_lock);
 457 }
 458
 459 /**
 460  * update_max_tr_single - only copy one trace over, and reset the rest
 461  * @tr - tracer
 462  * @tsk - task with the latency
 463  * @cpu - the cpu of the buffer to copy.
 464  *
 465  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 466  */
 467 void
 468 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 469 {
 470         int ret;
 471
 472         WARN_ON_ONCE(!irqs_disabled());
 473         __raw_spin_lock(&ftrace_max_lock);
 474
 475         ftrace_disable_cpu();
 476
 477         ring_buffer_reset(max_tr.buffer);
 478         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 479
 480         ftrace_enable_cpu();
 481
 482         WARN_ON_ONCE(ret);
 483
 484         __update_max_tr(tr, tsk, cpu);
 485         __raw_spin_unlock(&ftrace_max_lock);
 486 }
 487
 488 /**
 489  * register_tracer - register a tracer with the ftrace system.
 490  * @type - the plugin for the tracer
 491  *
 492  * Register a new plugin tracer.
 493  */
 494 int register_tracer(struct tracer *type)
 495 {
 496         struct tracer *t;
 497         int len;
 498         int ret = 0;
 499
 500         if (!type->name) {
 501                 pr_info("Tracer must have a name\n");
 502                 return -1;
 503         }
 504
 505         mutex_lock(&trace_types_lock);
 506         for (t = trace_types; t; t = t->next) {
 507                 if (strcmp(type->name, t->name) == 0) {
 508                         /* already found */
 509                         pr_info("Trace %s already registered\n",
 510                                 type->name);
 511                         ret = -1;
 512                         goto out;
 513                 }
 514         }
 515
 516 #ifdef CONFIG_FTRACE_STARTUP_TEST
 517         if (type->selftest) {
 518                 struct tracer *saved_tracer = current_trace;
 519                 struct trace_array *tr = &global_trace;
 520                 int saved_ctrl = tr->ctrl;
 521                 int i;
 522                 /*
 523                  * Run a selftest on this tracer.
 524                  * Here we reset the trace buffer, and set the current
 525                  * tracer to be this tracer. The tracer can then run some
 526                  * internal tracing to verify that everything is in order.
 527                  * If we fail, we do not register this tracer.
 528                  */
 529                 for_each_tracing_cpu(i) {
 530                         tracing_reset(tr, i);
 531                 }
 532                 current_trace = type;
 533                 tr->ctrl = 0;
 534                 /* the test is responsible for initializing and enabling */
 535                 pr_info("Testing tracer %s: ", type->name);
 536                 ret = type->selftest(type, tr);
 537                 /* the test is responsible for resetting too */
 538                 current_trace = saved_tracer;
 539                 tr->ctrl = saved_ctrl;
 540                 if (ret) {
 541                         printk(KERN_CONT "FAILED!\n");
 542                         goto out;
 543                 }
 544                 /* Only reset on passing, to avoid touching corrupted buffers */
 545                 for_each_tracing_cpu(i) {
 546                         tracing_reset(tr, i);
 547                 }
 548                 printk(KERN_CONT "PASSED\n");
 549         }
 550 #endif
 551
 552         type->next = trace_types;
 553         trace_types = type;
 554         len = strlen(type->name);
 555         if (len > max_tracer_type_len)
 556                 max_tracer_type_len = len;
 557
 558  out:
 559         mutex_unlock(&trace_types_lock);
 560
 561         return ret;
 562 }
 563
 564 void unregister_tracer(struct tracer *type)
 565 {
 566         struct tracer **t;
 567         int len;
 568
 569         mutex_lock(&trace_types_lock);
 570         for (t = &trace_types; *t; t = &(*t)->next) {
 571                 if (*t == type)
 572                         goto found;
 573         }
 574         pr_info("Trace %s not registered\n", type->name);
 575         goto out;
 576
 577  found:
 578         *t = (*t)->next;
 579         if (strlen(type->name) != max_tracer_type_len)
 580                 goto out;
 581
 582         max_tracer_type_len = 0;
 583         for (t = &trace_types; *t; t = &(*t)->next) {
 584                 len = strlen((*t)->name);
 585                 if (len > max_tracer_type_len)
 586                         max_tracer_type_len = len;
 587         }
 588  out:
 589         mutex_unlock(&trace_types_lock);
 590 }
 591
 592 void tracing_reset(struct trace_array *tr, int cpu)
 593 {
 594         ftrace_disable_cpu();
 595         ring_buffer_reset_cpu(tr->buffer, cpu);
 596         ftrace_enable_cpu();
 597 }
 598
 599 #define SAVED_CMDLINES 128
 600 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 601 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 602 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 603 static int cmdline_idx;
 604 static DEFINE_SPINLOCK(trace_cmdline_lock);
 605
 606 /* temporary disable recording */
 607 atomic_t trace_record_cmdline_disabled __read_mostly;
 608
 609 static void trace_init_cmdlines(void)
 610 {
 611         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
 612         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
 613         cmdline_idx = 0;
 614 }
 615
 616 void trace_stop_cmdline_recording(void);
 617
 618 static void trace_save_cmdline(struct task_struct *tsk)
 619 {
 620         unsigned map;
 621         unsigned idx;
 622
 623         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
 624                 return;
 625
 626         /*
 627          * It's not the end of the world if we don't get
 628          * the lock, but we also don't want to spin
 629          * nor do we want to disable interrupts,
 630          * so if we miss here, then better luck next time.
 631          */
 632         if (!spin_trylock(&trace_cmdline_lock))
 633                 return;
 634
 635         idx = map_pid_to_cmdline[tsk->pid];
 636         if (idx >= SAVED_CMDLINES) {
 637                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 638
 639                 map = map_cmdline_to_pid[idx];
 640                 if (map <= PID_MAX_DEFAULT)
 641                         map_pid_to_cmdline[map] = (unsigned)-1;
 642
 643                 map_pid_to_cmdline[tsk->pid] = idx;
 644
 645                 cmdline_idx = idx;
 646         }
 647
 648         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 649
 650         spin_unlock(&trace_cmdline_lock);
 651 }
 652
 653 static char *trace_find_cmdline(int pid)
 654 {
 655         char *cmdline = "<...>";
 656         unsigned map;
 657
 658         if (!pid)
 659                 return "<idle>";
 660
 661         if (pid > PID_MAX_DEFAULT)
 662                 goto out;
 663
 664         map = map_pid_to_cmdline[pid];
 665         if (map >= SAVED_CMDLINES)
 666                 goto out;
 667
 668         cmdline = saved_cmdlines[map];
 669
 670  out:
 671         return cmdline;
 672 }
 673
 674 void tracing_record_cmdline(struct task_struct *tsk)
 675 {
 676         if (atomic_read(&trace_record_cmdline_disabled))
 677                 return;
 678
 679         trace_save_cmdline(tsk);
 680 }
 681
 682 void
 683 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 684                              int pc)
 685 {
 686         struct task_struct *tsk = current;
 687
 688         entry->preempt_count            = pc & 0xff;
 689         entry->pid                      = (tsk) ? tsk->pid : 0;
 690         entry->flags =
 691 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 692                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
 693 #else
 694                 TRACE_FLAG_IRQS_NOSUPPORT |
 695 #endif
 696                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 697                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
 698                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 699 }
 700
 701 void
 702 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
 703                unsigned long ip, unsigned long parent_ip, unsigned long flags,
 704                int pc)
 705 {
 706         struct ring_buffer_event *event;
 707         struct ftrace_entry *entry;
 708         unsigned long irq_flags;
 709
 710         /* If we are reading the ring buffer, don't trace */
 711         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
 712                 return;
 713
 714         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 715                                          &irq_flags);
 716         if (!event)
 717                 return;
 718         entry   = ring_buffer_event_data(event);
 719         tracing_generic_entry_update(&entry->ent, flags, pc);
 720         entry->ent.type                 = TRACE_FN;
 721         entry->ip                       = ip;
 722         entry->parent_ip                = parent_ip;
 723         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 724 }
 725
 726 void
 727 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
 728        unsigned long ip, unsigned long parent_ip, unsigned long flags,
 729        int pc)
 730 {
 731         if (likely(!atomic_read(&data->disabled)))
 732                 trace_function(tr, data, ip, parent_ip, flags, pc);
 733 }
 734
 735 static void ftrace_trace_stack(struct trace_array *tr,
 736                                struct trace_array_cpu *data,
 737                                unsigned long flags,
 738                                int skip, int pc)
 739 {
 740 #ifdef CONFIG_STACKTRACE
 741         struct ring_buffer_event *event;
 742         struct stack_entry *entry;
 743         struct stack_trace trace;
 744         unsigned long irq_flags;
 745
 746         if (!(trace_flags & TRACE_ITER_STACKTRACE))
 747                 return;
 748
 749         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 750                                          &irq_flags);
 751         if (!event)
 752                 return;
 753         entry   = ring_buffer_event_data(event);
 754         tracing_generic_entry_update(&entry->ent, flags, pc);
 755         entry->ent.type         = TRACE_STACK;
 756
 757         memset(&entry->caller, 0, sizeof(entry->caller));
 758
 759         trace.nr_entries        = 0;
 760         trace.max_entries       = FTRACE_STACK_ENTRIES;
 761         trace.skip              = skip;
 762         trace.entries           = entry->caller;
 763
 764         save_stack_trace(&trace);
 765         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 766 #endif
 767 }
 768
 769 void __trace_stack(struct trace_array *tr,
 770                    struct trace_array_cpu *data,
 771                    unsigned long flags,
 772                    int skip)
 773 {
 774         ftrace_trace_stack(tr, data, flags, skip, preempt_count());
 775 }
 776
 777 static void
 778 ftrace_trace_special(void *__tr, void *__data,
 779                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
 780                      int pc)
 781 {
 782         struct ring_buffer_event *event;
 783         struct trace_array_cpu *data = __data;
 784         struct trace_array *tr = __tr;
 785         struct special_entry *entry;
 786         unsigned long irq_flags;
 787
 788         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 789                                          &irq_flags);
 790         if (!event)
 791                 return;
 792         entry   = ring_buffer_event_data(event);
 793         tracing_generic_entry_update(&entry->ent, 0, pc);
 794         entry->ent.type                 = TRACE_SPECIAL;
 795         entry->arg1                     = arg1;
 796         entry->arg2                     = arg2;
 797         entry->arg3                     = arg3;
 798         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 799         ftrace_trace_stack(tr, data, irq_flags, 4, pc);
 800
 801         trace_wake_up();
 802 }
 803
 804 void
 805 __trace_special(void *__tr, void *__data,
 806                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
 807 {
 808         ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
 809 }
 810
 811 void
 812 tracing_sched_switch_trace(struct trace_array *tr,
 813                            struct trace_array_cpu *data,
 814                            struct task_struct *prev,
 815                            struct task_struct *next,
 816                            unsigned long flags, int pc)
 817 {
 818         struct ring_buffer_event *event;
 819         struct ctx_switch_entry *entry;
 820         unsigned long irq_flags;
 821
 822         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 823                                            &irq_flags);
 824         if (!event)
 825                 return;
 826         entry   = ring_buffer_event_data(event);
 827         tracing_generic_entry_update(&entry->ent, flags, pc);
 828         entry->ent.type                 = TRACE_CTX;
 829         entry->prev_pid                 = prev->pid;
 830         entry->prev_prio                = prev->prio;
 831         entry->prev_state               = prev->state;
 832         entry->next_pid                 = next->pid;
 833         entry->next_prio                = next->prio;
 834         entry->next_state               = next->state;
 835         entry->next_cpu = task_cpu(next);
 836         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 837         ftrace_trace_stack(tr, data, flags, 5, pc);
 838 }
 839
 840 void
 841 tracing_sched_wakeup_trace(struct trace_array *tr,
 842                            struct trace_array_cpu *data,
 843                            struct task_struct *wakee,
 844                            struct task_struct *curr,
 845                            unsigned long flags, int pc)
 846 {
 847         struct ring_buffer_event *event;
 848         struct ctx_switch_entry *entry;
 849         unsigned long irq_flags;
 850
 851         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 852                                            &irq_flags);
 853         if (!event)
 854                 return;
 855         entry   = ring_buffer_event_data(event);
 856         tracing_generic_entry_update(&entry->ent, flags, pc);
 857         entry->ent.type                 = TRACE_WAKE;
 858         entry->prev_pid                 = curr->pid;
 859         entry->prev_prio                = curr->prio;
 860         entry->prev_state               = curr->state;
 861         entry->next_pid                 = wakee->pid;
 862         entry->next_prio                = wakee->prio;
 863         entry->next_state               = wakee->state;
 864         entry->next_cpu                 = task_cpu(wakee);
 865         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 866         ftrace_trace_stack(tr, data, flags, 6, pc);
 867
 868         trace_wake_up();
 869 }
 870
 871 void
 872 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
 873 {
 874         struct trace_array *tr = &global_trace;
 875         struct trace_array_cpu *data;
 876         int cpu;
 877         int pc;
 878
 879         if (tracing_disabled || !tr->ctrl)
 880                 return;
 881
 882         pc = preempt_count();
 883         preempt_disable_notrace();
 884         cpu = raw_smp_processor_id();
 885         data = tr->data[cpu];
 886
 887         if (likely(!atomic_read(&data->disabled)))
 888                 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
 889
 890         preempt_enable_notrace();
 891 }
 892
 893 #ifdef CONFIG_FUNCTION_TRACER
 894 static void
 895 function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
 896 {
 897         struct trace_array *tr = &global_trace;
 898         struct trace_array_cpu *data;
 899         unsigned long flags;
 900         long disabled;
 901         int cpu, resched;
 902         int pc;
 903
 904         if (unlikely(!ftrace_function_enabled))
 905                 return;
 906
 907         pc = preempt_count();
 908         resched = ftrace_preempt_disable();
 909         local_save_flags(flags);
 910         cpu = raw_smp_processor_id();
 911         data = tr->data[cpu];
 912         disabled = atomic_inc_return(&data->disabled);
 913
 914         if (likely(disabled == 1))
 915                 trace_function(tr, data, ip, parent_ip, flags, pc);
 916
 917         atomic_dec(&data->disabled);
 918         ftrace_preempt_enable(resched);
 919 }
 920
 921 static void
 922 function_trace_call(unsigned long ip, unsigned long parent_ip)
 923 {
 924         struct trace_array *tr = &global_trace;
 925         struct trace_array_cpu *data;
 926         unsigned long flags;
 927         long disabled;
 928         int cpu;
 929         int pc;
 930
 931         if (unlikely(!ftrace_function_enabled))
 932                 return;
 933
 934         /*
 935          * Need to use raw, since this must be called before the
 936          * recursive protection is performed.
 937          */
 938         raw_local_irq_save(flags);
 939         cpu = raw_smp_processor_id();
 940         data = tr->data[cpu];
 941         disabled = atomic_inc_return(&data->disabled);
 942
 943         if (likely(disabled == 1)) {
 944                 pc = preempt_count();
 945                 trace_function(tr, data, ip, parent_ip, flags, pc);
 946         }
 947
 948         atomic_dec(&data->disabled);
 949         raw_local_irq_restore(flags);
 950 }
 951
 952 static struct ftrace_ops trace_ops __read_mostly =
 953 {
 954         .func = function_trace_call,
 955 };
 956
 957 void tracing_start_function_trace(void)
 958 {
 959         ftrace_function_enabled = 0;
 960
 961         if (trace_flags & TRACE_ITER_PREEMPTONLY)
 962                 trace_ops.func = function_trace_call_preempt_only;
 963         else
 964                 trace_ops.func = function_trace_call;
 965
 966         register_ftrace_function(&trace_ops);
 967         if (tracer_enabled)
 968                 ftrace_function_enabled = 1;
 969 }
 970
 971 void tracing_stop_function_trace(void)
 972 {
 973         ftrace_function_enabled = 0;
 974         unregister_ftrace_function(&trace_ops);
 975 }
 976 #endif
 977
 978 enum trace_file_type {
 979         TRACE_FILE_LAT_FMT      = 1,
 980 };
 981
 982 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
 983 {
 984         /* Don't allow ftrace to trace into the ring buffers */
 985         ftrace_disable_cpu();
 986
 987         iter->idx++;
 988         if (iter->buffer_iter[iter->cpu])
 989                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
 990
 991         ftrace_enable_cpu();
 992 }
 993
 994 static struct trace_entry *
 995 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
 996 {
 997         struct ring_buffer_event *event;
 998         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
 999
1000         /* Don't allow ftrace to trace into the ring buffers */
1001         ftrace_disable_cpu();
1002
1003         if (buf_iter)
1004                 event = ring_buffer_iter_peek(buf_iter, ts);
1005         else
1006                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1007
1008         ftrace_enable_cpu();
1009
1010         return event ? ring_buffer_event_data(event) : NULL;
1011 }
1012
1013 static struct trace_entry *
1014 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1015 {
1016         struct ring_buffer *buffer = iter->tr->buffer;
1017         struct trace_entry *ent, *next = NULL;
1018         u64 next_ts = 0, ts;
1019         int next_cpu = -1;
1020         int cpu;
1021
1022         for_each_tracing_cpu(cpu) {
1023
1024                 if (ring_buffer_empty_cpu(buffer, cpu))
1025                         continue;
1026
1027                 ent = peek_next_entry(iter, cpu, &ts);
1028
1029                 /*
1030                  * Pick the entry with the smallest timestamp:
1031                  */
1032                 if (ent && (!next || ts < next_ts)) {
1033                         next = ent;
1034                         next_cpu = cpu;
1035                         next_ts = ts;
1036                 }
1037         }
1038
1039         if (ent_cpu)
1040                 *ent_cpu = next_cpu;
1041
1042         if (ent_ts)
1043                 *ent_ts = next_ts;
1044
1045         return next;
1046 }
1047
1048 /* Find the next real entry, without updating the iterator itself */
1049 static struct trace_entry *
1050 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1051 {
1052         return __find_next_entry(iter, ent_cpu, ent_ts);
1053 }
1054
1055 /* Find the next real entry, and increment the iterator to the next entry */
1056 static void *find_next_entry_inc(struct trace_iterator *iter)
1057 {
1058         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1059
1060         if (iter->ent)
1061                 trace_iterator_increment(iter, iter->cpu);
1062
1063         return iter->ent ? iter : NULL;
1064 }
1065
1066 static void trace_consume(struct trace_iterator *iter)
1067 {
1068         /* Don't allow ftrace to trace into the ring buffers */
1069         ftrace_disable_cpu();
1070         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1071         ftrace_enable_cpu();
1072 }
1073
1074 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1075 {
1076         struct trace_iterator *iter = m->private;
1077         int i = (int)*pos;
1078         void *ent;
1079
1080         (*pos)++;
1081
1082         /* can't go backwards */
1083         if (iter->idx > i)
1084                 return NULL;
1085
1086         if (iter->idx < 0)
1087                 ent = find_next_entry_inc(iter);
1088         else
1089                 ent = iter;
1090
1091         while (ent && iter->idx < i)
1092                 ent = find_next_entry_inc(iter);
1093
1094         iter->pos = *pos;
1095
1096         return ent;
1097 }
1098
1099 static void *s_start(struct seq_file *m, loff_t *pos)
1100 {
1101         struct trace_iterator *iter = m->private;
1102         void *p = NULL;
1103         loff_t l = 0;
1104         int cpu;
1105
1106         mutex_lock(&trace_types_lock);
1107
1108         if (!current_trace || current_trace != iter->trace) {
1109                 mutex_unlock(&trace_types_lock);
1110                 return NULL;
1111         }
1112
1113         atomic_inc(&trace_record_cmdline_disabled);
1114
1115         /* let the tracer grab locks here if needed */
1116         if (current_trace->start)
1117                 current_trace->start(iter);
1118
1119         if (*pos != iter->pos) {
1120                 iter->ent = NULL;
1121                 iter->cpu = 0;
1122                 iter->idx = -1;
1123
1124                 ftrace_disable_cpu();
1125
1126                 for_each_tracing_cpu(cpu) {
1127                         ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1128                 }
1129
1130                 ftrace_enable_cpu();
1131
1132                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1133                         ;
1134
1135         } else {
1136                 l = *pos - 1;
1137                 p = s_next(m, p, &l);
1138         }
1139
1140         return p;
1141 }
1142
1143 static void s_stop(struct seq_file *m, void *p)
1144 {
1145         struct trace_iterator *iter = m->private;
1146
1147         atomic_dec(&trace_record_cmdline_disabled);
1148
1149         /* let the tracer release locks here if needed */
1150         if (current_trace && current_trace == iter->trace && iter->trace->stop)
1151                 iter->trace->stop(iter);
1152
1153         mutex_unlock(&trace_types_lock);
1154 }
1155
1156 #define KRETPROBE_MSG "[unknown/kretprobe'd]"
1157
1158 #ifdef CONFIG_KRETPROBES
1159 static inline int kretprobed(unsigned long addr)
1160 {
1161         return addr == (unsigned long)kretprobe_trampoline;
1162 }
1163 #else
1164 static inline int kretprobed(unsigned long addr)
1165 {
1166         return 0;
1167 }
1168 #endif /* CONFIG_KRETPROBES */
1169
1170 static int
1171 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1172 {
1173 #ifdef CONFIG_KALLSYMS
1174         char str[KSYM_SYMBOL_LEN];
1175
1176         kallsyms_lookup(address, NULL, NULL, NULL, str);
1177
1178         return trace_seq_printf(s, fmt, str);
1179 #endif
1180         return 1;
1181 }
1182
1183 static int
1184 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1185                      unsigned long address)
1186 {
1187 #ifdef CONFIG_KALLSYMS
1188         char str[KSYM_SYMBOL_LEN];
1189
1190         sprint_symbol(str, address);
1191         return trace_seq_printf(s, fmt, str);
1192 #endif
1193         return 1;
1194 }
1195
1196 #ifndef CONFIG_64BIT
1197 # define IP_FMT "%08lx"
1198 #else
1199 # define IP_FMT "%016lx"
1200 #endif
1201
1202 static int
1203 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1204 {
1205         int ret;
1206
1207         if (!ip)
1208                 return trace_seq_printf(s, "0");
1209
1210         if (sym_flags & TRACE_ITER_SYM_OFFSET)
1211                 ret = seq_print_sym_offset(s, "%s", ip);
1212         else
1213                 ret = seq_print_sym_short(s, "%s", ip);
1214
1215         if (!ret)
1216                 return 0;
1217
1218         if (sym_flags & TRACE_ITER_SYM_ADDR)
1219                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1220         return ret;
1221 }
1222
1223 static void print_lat_help_header(struct seq_file *m)
1224 {
1225         seq_puts(m, "#                  _------=> CPU#            \n");
1226         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1227         seq_puts(m, "#                | / _----=> need-resched    \n");
1228         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1229         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1230         seq_puts(m, "#                |||| /                      \n");
1231         seq_puts(m, "#                |||||     delay             \n");
1232         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1233         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1234 }
1235
1236 static void print_func_help_header(struct seq_file *m)
1237 {
1238         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1239         seq_puts(m, "#              | |       |          |         |\n");
1240 }
1241
1242
1243 static void
1244 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1245 {
1246         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1247         struct trace_array *tr = iter->tr;
1248         struct trace_array_cpu *data = tr->data[tr->cpu];
1249         struct tracer *type = current_trace;
1250         unsigned long total;
1251         unsigned long entries;
1252         const char *name = "preemption";
1253
1254         if (type)
1255                 name = type->name;
1256
1257         entries = ring_buffer_entries(iter->tr->buffer);
1258         total = entries +
1259                 ring_buffer_overruns(iter->tr->buffer);
1260
1261         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1262                    name, UTS_RELEASE);
1263         seq_puts(m, "-----------------------------------"
1264                  "---------------------------------\n");
1265         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1266                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1267                    nsecs_to_usecs(data->saved_latency),
1268                    entries,
1269                    total,
1270                    tr->cpu,
1271 #if defined(CONFIG_PREEMPT_NONE)
1272                    "server",
1273 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1274                    "desktop",
1275 #elif defined(CONFIG_PREEMPT)
1276                    "preempt",
1277 #else
1278                    "unknown",
1279 #endif
1280                    /* These are reserved for later use */
1281                    0, 0, 0, 0);
1282 #ifdef CONFIG_SMP
1283         seq_printf(m, " #P:%d)\n", num_online_cpus());
1284 #else
1285         seq_puts(m, ")\n");
1286 #endif
1287         seq_puts(m, "    -----------------\n");
1288         seq_printf(m, "    | task: %.16s-%d "
1289                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1290                    data->comm, data->pid, data->uid, data->nice,
1291                    data->policy, data->rt_priority);
1292         seq_puts(m, "    -----------------\n");
1293
1294         if (data->critical_start) {
1295                 seq_puts(m, " => started at: ");
1296                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1297                 trace_print_seq(m, &iter->seq);
1298                 seq_puts(m, "\n => ended at:   ");
1299                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1300                 trace_print_seq(m, &iter->seq);
1301                 seq_puts(m, "\n");
1302         }
1303
1304         seq_puts(m, "\n");
1305 }
1306
1307 static void
1308 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1309 {
1310         int hardirq, softirq;
1311         char *comm;
1312
1313         comm = trace_find_cmdline(entry->pid);
1314
1315         trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1316         trace_seq_printf(s, "%3d", cpu);
1317         trace_seq_printf(s, "%c%c",
1318                         (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1319                          (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1320                         ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1321
1322         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1323         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1324         if (hardirq && softirq) {
1325                 trace_seq_putc(s, 'H');
1326         } else {
1327                 if (hardirq) {
1328                         trace_seq_putc(s, 'h');
1329                 } else {
1330                         if (softirq)
1331                                 trace_seq_putc(s, 's');
1332                         else
1333                                 trace_seq_putc(s, '.');
1334                 }
1335         }
1336
1337         if (entry->preempt_count)
1338                 trace_seq_printf(s, "%x", entry->preempt_count);
1339         else
1340                 trace_seq_puts(s, ".");
1341 }
1342
1343 unsigned long preempt_mark_thresh = 100;
1344
1345 static void
1346 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1347                     unsigned long rel_usecs)
1348 {
1349         trace_seq_printf(s, " %4lldus", abs_usecs);
1350         if (rel_usecs > preempt_mark_thresh)
1351                 trace_seq_puts(s, "!: ");
1352         else if (rel_usecs > 1)
1353                 trace_seq_puts(s, "+: ");
1354         else
1355                 trace_seq_puts(s, " : ");
1356 }
1357
1358 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1359
1360 /*
1361  * The message is supposed to contain an ending newline.
1362  * If the printing stops prematurely, try to add a newline of our own.
1363  */
1364 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1365 {
1366         struct trace_entry *ent;
1367         struct trace_field_cont *cont;
1368         bool ok = true;
1369
1370         ent = peek_next_entry(iter, iter->cpu, NULL);
1371         if (!ent || ent->type != TRACE_CONT) {
1372                 trace_seq_putc(s, '\n');
1373                 return;
1374         }
1375
1376         do {
1377                 cont = (struct trace_field_cont *)ent;
1378                 if (ok)
1379                         ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1380
1381                 ftrace_disable_cpu();
1382
1383                 if (iter->buffer_iter[iter->cpu])
1384                         ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1385                 else
1386                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1387
1388                 ftrace_enable_cpu();
1389
1390                 ent = peek_next_entry(iter, iter->cpu, NULL);
1391         } while (ent && ent->type == TRACE_CONT);
1392
1393         if (!ok)
1394                 trace_seq_putc(s, '\n');
1395 }
1396
1397 static enum print_line_t
1398 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1399 {
1400         struct trace_seq *s = &iter->seq;
1401         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1402         struct trace_entry *next_entry;
1403         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1404         struct trace_entry *entry = iter->ent;
1405         unsigned long abs_usecs;
1406         unsigned long rel_usecs;
1407         u64 next_ts;
1408         char *comm;
1409         int S, T;
1410         int i;
1411         unsigned state;
1412
1413         if (entry->type == TRACE_CONT)
1414                 return TRACE_TYPE_HANDLED;
1415
1416         next_entry = find_next_entry(iter, NULL, &next_ts);
1417         if (!next_entry)
1418                 next_ts = iter->ts;
1419         rel_usecs = ns2usecs(next_ts - iter->ts);
1420         abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1421
1422         if (verbose) {
1423                 comm = trace_find_cmdline(entry->pid);
1424                 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1425                                  " %ld.%03ldms (+%ld.%03ldms): ",
1426                                  comm,
1427                                  entry->pid, cpu, entry->flags,
1428                                  entry->preempt_count, trace_idx,
1429                                  ns2usecs(iter->ts),
1430                                  abs_usecs/1000,
1431                                  abs_usecs % 1000, rel_usecs/1000,
1432                                  rel_usecs % 1000);
1433         } else {
1434                 lat_print_generic(s, entry, cpu);
1435                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1436         }
1437         switch (entry->type) {
1438         case TRACE_FN: {
1439                 struct ftrace_entry *field;
1440
1441                 trace_assign_type(field, entry);
1442
1443                 seq_print_ip_sym(s, field->ip, sym_flags);
1444                 trace_seq_puts(s, " (");
1445                 if (kretprobed(field->parent_ip))
1446                         trace_seq_puts(s, KRETPROBE_MSG);
1447                 else
1448                         seq_print_ip_sym(s, field->parent_ip, sym_flags);
1449                 trace_seq_puts(s, ")\n");
1450                 break;
1451         }
1452         case TRACE_CTX:
1453         case TRACE_WAKE: {
1454                 struct ctx_switch_entry *field;
1455
1456                 trace_assign_type(field, entry);
1457
1458                 T = field->next_state < sizeof(state_to_char) ?
1459                         state_to_char[field->next_state] : 'X';
1460
1461                 state = field->prev_state ?
1462                         __ffs(field->prev_state) + 1 : 0;
1463                 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1464                 comm = trace_find_cmdline(field->next_pid);
1465                 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1466                                  field->prev_pid,
1467                                  field->prev_prio,
1468                                  S, entry->type == TRACE_CTX ? "==>" : "  +",
1469                                  field->next_cpu,
1470                                  field->next_pid,
1471                                  field->next_prio,
1472                                  T, comm);
1473                 break;
1474         }
1475         case TRACE_SPECIAL: {
1476                 struct special_entry *field;
1477
1478                 trace_assign_type(field, entry);
1479
1480                 trace_seq_printf(s, "# %ld %ld %ld\n",
1481                                  field->arg1,
1482                                  field->arg2,
1483                                  field->arg3);
1484                 break;
1485         }
1486         case TRACE_STACK: {
1487                 struct stack_entry *field;
1488
1489                 trace_assign_type(field, entry);
1490
1491                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1492                         if (i)
1493                                 trace_seq_puts(s, " <= ");
1494                         seq_print_ip_sym(s, field->caller[i], sym_flags);
1495                 }
1496                 trace_seq_puts(s, "\n");
1497                 break;
1498         }
1499         case TRACE_PRINT: {
1500                 struct print_entry *field;
1501
1502                 trace_assign_type(field, entry);
1503
1504                 seq_print_ip_sym(s, field->ip, sym_flags);
1505                 trace_seq_printf(s, ": %s", field->buf);
1506                 if (entry->flags & TRACE_FLAG_CONT)
1507                         trace_seq_print_cont(s, iter);
1508                 break;
1509         }
1510         default:
1511                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1512         }
1513         return TRACE_TYPE_HANDLED;
1514 }
1515
1516 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1517 {
1518         struct trace_seq *s = &iter->seq;
1519         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1520         struct trace_entry *entry;
1521         unsigned long usec_rem;
1522         unsigned long long t;
1523         unsigned long secs;
1524         char *comm;
1525         int ret;
1526         int S, T;
1527         int i;
1528
1529         entry = iter->ent;
1530
1531         if (entry->type == TRACE_CONT)
1532                 return TRACE_TYPE_HANDLED;
1533
1534         comm = trace_find_cmdline(iter->ent->pid);
1535
1536         t = ns2usecs(iter->ts);
1537         usec_rem = do_div(t, 1000000ULL);
1538         secs = (unsigned long)t;
1539
1540         ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1541         if (!ret)
1542                 return TRACE_TYPE_PARTIAL_LINE;
1543         ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1544         if (!ret)
1545                 return TRACE_TYPE_PARTIAL_LINE;
1546         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1547         if (!ret)
1548                 return TRACE_TYPE_PARTIAL_LINE;
1549
1550         switch (entry->type) {
1551         case TRACE_FN: {
1552                 struct ftrace_entry *field;
1553
1554                 trace_assign_type(field, entry);
1555
1556                 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1557                 if (!ret)
1558                         return TRACE_TYPE_PARTIAL_LINE;
1559                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1560                                                 field->parent_ip) {
1561                         ret = trace_seq_printf(s, " <-");
1562                         if (!ret)
1563                                 return TRACE_TYPE_PARTIAL_LINE;
1564                         if (kretprobed(field->parent_ip))
1565                                 ret = trace_seq_puts(s, KRETPROBE_MSG);
1566                         else
1567                                 ret = seq_print_ip_sym(s,
1568                                                        field->parent_ip,
1569                                                        sym_flags);
1570                         if (!ret)
1571                                 return TRACE_TYPE_PARTIAL_LINE;
1572                 }
1573                 ret = trace_seq_printf(s, "\n");
1574                 if (!ret)
1575                         return TRACE_TYPE_PARTIAL_LINE;
1576                 break;
1577         }
1578         case TRACE_CTX:
1579         case TRACE_WAKE: {
1580                 struct ctx_switch_entry *field;
1581
1582                 trace_assign_type(field, entry);
1583
1584                 S = field->prev_state < sizeof(state_to_char) ?
1585                         state_to_char[field->prev_state] : 'X';
1586                 T = field->next_state < sizeof(state_to_char) ?
1587                         state_to_char[field->next_state] : 'X';
1588                 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1589                                        field->prev_pid,
1590                                        field->prev_prio,
1591                                        S,
1592                                        entry->type == TRACE_CTX ? "==>" : "  +",
1593                                        field->next_cpu,
1594                                        field->next_pid,
1595                                        field->next_prio,
1596                                        T);
1597                 if (!ret)
1598                         return TRACE_TYPE_PARTIAL_LINE;
1599                 break;
1600         }
1601         case TRACE_SPECIAL: {
1602                 struct special_entry *field;
1603
1604                 trace_assign_type(field, entry);
1605
1606                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1607                                  field->arg1,
1608                                  field->arg2,
1609                                  field->arg3);
1610                 if (!ret)
1611                         return TRACE_TYPE_PARTIAL_LINE;
1612                 break;
1613         }
1614         case TRACE_STACK: {
1615                 struct stack_entry *field;
1616
1617                 trace_assign_type(field, entry);
1618
1619                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1620                         if (i) {
1621                                 ret = trace_seq_puts(s, " <= ");
1622                                 if (!ret)
1623                                         return TRACE_TYPE_PARTIAL_LINE;
1624                         }
1625                         ret = seq_print_ip_sym(s, field->caller[i],
1626                                                sym_flags);
1627                         if (!ret)
1628                                 return TRACE_TYPE_PARTIAL_LINE;
1629                 }
1630                 ret = trace_seq_puts(s, "\n");
1631                 if (!ret)
1632                         return TRACE_TYPE_PARTIAL_LINE;
1633                 break;
1634         }
1635         case TRACE_PRINT: {
1636                 struct print_entry *field;
1637
1638                 trace_assign_type(field, entry);
1639
1640                 seq_print_ip_sym(s, field->ip, sym_flags);
1641                 trace_seq_printf(s, ": %s", field->buf);
1642                 if (entry->flags & TRACE_FLAG_CONT)
1643                         trace_seq_print_cont(s, iter);
1644                 break;
1645         }
1646         }
1647         return TRACE_TYPE_HANDLED;
1648 }
1649
1650 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1651 {
1652         struct trace_seq *s = &iter->seq;
1653         struct trace_entry *entry;
1654         int ret;
1655         int S, T;
1656
1657         entry = iter->ent;
1658
1659         if (entry->type == TRACE_CONT)
1660                 return TRACE_TYPE_HANDLED;
1661
1662         ret = trace_seq_printf(s, "%d %d %llu ",
1663                 entry->pid, iter->cpu, iter->ts);
1664         if (!ret)
1665                 return TRACE_TYPE_PARTIAL_LINE;
1666
1667         switch (entry->type) {
1668         case TRACE_FN: {
1669                 struct ftrace_entry *field;
1670
1671                 trace_assign_type(field, entry);
1672
1673                 ret = trace_seq_printf(s, "%x %x\n",
1674                                         field->ip,
1675                                         field->parent_ip);
1676                 if (!ret)
1677                         return TRACE_TYPE_PARTIAL_LINE;
1678                 break;
1679         }
1680         case TRACE_CTX:
1681         case TRACE_WAKE: {
1682                 struct ctx_switch_entry *field;
1683
1684                 trace_assign_type(field, entry);
1685
1686                 S = field->prev_state < sizeof(state_to_char) ?
1687                         state_to_char[field->prev_state] : 'X';
1688                 T = field->next_state < sizeof(state_to_char) ?
1689                         state_to_char[field->next_state] : 'X';
1690                 if (entry->type == TRACE_WAKE)
1691                         S = '+';
1692                 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1693                                        field->prev_pid,
1694                                        field->prev_prio,
1695                                        S,
1696                                        field->next_cpu,
1697                                        field->next_pid,
1698                                        field->next_prio,
1699                                        T);
1700                 if (!ret)
1701                         return TRACE_TYPE_PARTIAL_LINE;
1702                 break;
1703         }
1704         case TRACE_SPECIAL:
1705         case TRACE_STACK: {
1706                 struct special_entry *field;
1707
1708                 trace_assign_type(field, entry);
1709
1710                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1711                                  field->arg1,
1712                                  field->arg2,
1713                                  field->arg3);
1714                 if (!ret)
1715                         return TRACE_TYPE_PARTIAL_LINE;
1716                 break;
1717         }
1718         case TRACE_PRINT: {
1719                 struct print_entry *field;
1720
1721                 trace_assign_type(field, entry);
1722
1723                 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1724                 if (entry->flags & TRACE_FLAG_CONT)
1725                         trace_seq_print_cont(s, iter);
1726                 break;
1727         }
1728         }
1729         return TRACE_TYPE_HANDLED;
1730 }
1731
1732 #define SEQ_PUT_FIELD_RET(s, x)                         \
1733 do {                                                    \
1734         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1735                 return 0;                               \
1736 } while (0)
1737
1738 #define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
1739 do {                                                    \
1740         BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES);     \
1741         if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
1742                 return 0;                               \
1743 } while (0)
1744
1745 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1746 {
1747         struct trace_seq *s = &iter->seq;
1748         unsigned char newline = '\n';
1749         struct trace_entry *entry;
1750         int S, T;
1751
1752         entry = iter->ent;
1753
1754         if (entry->type == TRACE_CONT)
1755                 return TRACE_TYPE_HANDLED;
1756
1757         SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1758         SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1759         SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1760
1761         switch (entry->type) {
1762         case TRACE_FN: {
1763                 struct ftrace_entry *field;
1764
1765                 trace_assign_type(field, entry);
1766
1767                 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1768                 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1769                 break;
1770         }
1771         case TRACE_CTX:
1772         case TRACE_WAKE: {
1773                 struct ctx_switch_entry *field;
1774
1775                 trace_assign_type(field, entry);
1776
1777                 S = field->prev_state < sizeof(state_to_char) ?
1778                         state_to_char[field->prev_state] : 'X';
1779                 T = field->next_state < sizeof(state_to_char) ?
1780                         state_to_char[field->next_state] : 'X';
1781                 if (entry->type == TRACE_WAKE)
1782                         S = '+';
1783                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1784                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1785                 SEQ_PUT_HEX_FIELD_RET(s, S);
1786                 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1787                 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1788                 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1789                 SEQ_PUT_HEX_FIELD_RET(s, T);
1790                 break;
1791         }
1792         case TRACE_SPECIAL:
1793         case TRACE_STACK: {
1794                 struct special_entry *field;
1795
1796                 trace_assign_type(field, entry);
1797
1798                 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1799                 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1800                 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1801                 break;
1802         }
1803         }
1804         SEQ_PUT_FIELD_RET(s, newline);
1805
1806         return TRACE_TYPE_HANDLED;
1807 }
1808
1809 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1810 {
1811         struct trace_seq *s = &iter->seq;
1812         struct trace_entry *entry;
1813
1814         entry = iter->ent;
1815
1816         if (entry->type == TRACE_CONT)
1817                 return TRACE_TYPE_HANDLED;
1818
1819         SEQ_PUT_FIELD_RET(s, entry->pid);
1820         SEQ_PUT_FIELD_RET(s, iter->cpu);
1821         SEQ_PUT_FIELD_RET(s, iter->ts);
1822
1823         switch (entry->type) {
1824         case TRACE_FN: {
1825                 struct ftrace_entry *field;
1826
1827                 trace_assign_type(field, entry);
1828
1829                 SEQ_PUT_FIELD_RET(s, field->ip);
1830                 SEQ_PUT_FIELD_RET(s, field->parent_ip);
1831                 break;
1832         }
1833         case TRACE_CTX: {
1834                 struct ctx_switch_entry *field;
1835
1836                 trace_assign_type(field, entry);
1837
1838                 SEQ_PUT_FIELD_RET(s, field->prev_pid);
1839                 SEQ_PUT_FIELD_RET(s, field->prev_prio);
1840                 SEQ_PUT_FIELD_RET(s, field->prev_state);
1841                 SEQ_PUT_FIELD_RET(s, field->next_pid);
1842                 SEQ_PUT_FIELD_RET(s, field->next_prio);
1843                 SEQ_PUT_FIELD_RET(s, field->next_state);
1844                 break;
1845         }
1846         case TRACE_SPECIAL:
1847         case TRACE_STACK: {
1848                 struct special_entry *field;
1849
1850                 trace_assign_type(field, entry);
1851
1852                 SEQ_PUT_FIELD_RET(s, field->arg1);
1853                 SEQ_PUT_FIELD_RET(s, field->arg2);
1854                 SEQ_PUT_FIELD_RET(s, field->arg3);
1855                 break;
1856         }
1857         }
1858         return 1;
1859 }
1860
1861 static int trace_empty(struct trace_iterator *iter)
1862 {
1863         int cpu;
1864
1865         for_each_tracing_cpu(cpu) {
1866                 if (iter->buffer_iter[cpu]) {
1867                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1868                                 return 0;
1869                 } else {
1870                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1871                                 return 0;
1872                 }
1873         }
1874
1875         return 1;
1876 }
1877
1878 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1879 {
1880         enum print_line_t ret;
1881
1882         if (iter->trace && iter->trace->print_line) {
1883                 ret = iter->trace->print_line(iter);
1884                 if (ret != TRACE_TYPE_UNHANDLED)
1885                         return ret;
1886         }
1887
1888         if (trace_flags & TRACE_ITER_BIN)
1889                 return print_bin_fmt(iter);
1890
1891         if (trace_flags & TRACE_ITER_HEX)
1892                 return print_hex_fmt(iter);
1893
1894         if (trace_flags & TRACE_ITER_RAW)
1895                 return print_raw_fmt(iter);
1896
1897         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1898                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1899
1900         return print_trace_fmt(iter);
1901 }
1902
1903 static int s_show(struct seq_file *m, void *v)
1904 {
1905         struct trace_iterator *iter = v;
1906
1907         if (iter->ent == NULL) {
1908                 if (iter->tr) {
1909                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1910                         seq_puts(m, "#\n");
1911                 }
1912                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1913                         /* print nothing if the buffers are empty */
1914                         if (trace_empty(iter))
1915                                 return 0;
1916                         print_trace_header(m, iter);
1917                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1918                                 print_lat_help_header(m);
1919                 } else {
1920                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1921                                 print_func_help_header(m);
1922                 }
1923         } else {
1924                 print_trace_line(iter);
1925                 trace_print_seq(m, &iter->seq);
1926         }
1927
1928         return 0;
1929 }
1930
1931 static struct seq_operations tracer_seq_ops = {
1932         .start          = s_start,
1933         .next           = s_next,
1934         .stop           = s_stop,
1935         .show           = s_show,
1936 };
1937
1938 static struct trace_iterator *
1939 __tracing_open(struct inode *inode, struct file *file, int *ret)
1940 {
1941         struct trace_iterator *iter;
1942         struct seq_file *m;
1943         int cpu;
1944
1945         if (tracing_disabled) {
1946                 *ret = -ENODEV;
1947                 return NULL;
1948         }
1949
1950         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1951         if (!iter) {
1952                 *ret = -ENOMEM;
1953                 goto out;
1954         }
1955
1956         mutex_lock(&trace_types_lock);
1957         if (current_trace && current_trace->print_max)
1958                 iter->tr = &max_tr;
1959         else
1960                 iter->tr = inode->i_private;
1961         iter->trace = current_trace;
1962         iter->pos = -1;
1963
1964         for_each_tracing_cpu(cpu) {
1965
1966                 iter->buffer_iter[cpu] =
1967                         ring_buffer_read_start(iter->tr->buffer, cpu);
1968
1969                 if (!iter->buffer_iter[cpu])
1970                         goto fail_buffer;
1971         }
1972
1973         /* TODO stop tracer */
1974         *ret = seq_open(file, &tracer_seq_ops);
1975         if (*ret)
1976                 goto fail_buffer;
1977
1978         m = file->private_data;
1979         m->private = iter;
1980
1981         /* stop the trace while dumping */
1982         if (iter->tr->ctrl) {
1983                 tracer_enabled = 0;
1984                 ftrace_function_enabled = 0;
1985         }
1986
1987         if (iter->trace && iter->trace->open)
1988                         iter->trace->open(iter);
1989
1990         mutex_unlock(&trace_types_lock);
1991
1992  out:
1993         return iter;
1994
1995  fail_buffer:
1996         for_each_tracing_cpu(cpu) {
1997                 if (iter->buffer_iter[cpu])
1998                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1999         }
2000         mutex_unlock(&trace_types_lock);
2001
2002         return ERR_PTR(-ENOMEM);
2003 }
2004
2005 int tracing_open_generic(struct inode *inode, struct file *filp)
2006 {
2007         if (tracing_disabled)
2008                 return -ENODEV;
2009
2010         filp->private_data = inode->i_private;
2011         return 0;
2012 }
2013
2014 int tracing_release(struct inode *inode, struct file *file)
2015 {
2016         struct seq_file *m = (struct seq_file *)file->private_data;
2017         struct trace_iterator *iter = m->private;
2018         int cpu;
2019
2020         mutex_lock(&trace_types_lock);
2021         for_each_tracing_cpu(cpu) {
2022                 if (iter->buffer_iter[cpu])
2023                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2024         }
2025
2026         if (iter->trace && iter->trace->close)
2027                 iter->trace->close(iter);
2028
2029         /* reenable tracing if it was previously enabled */
2030         if (iter->tr->ctrl) {
2031                 tracer_enabled = 1;
2032                 /*
2033                  * It is safe to enable function tracing even if it
2034                  * isn't used
2035                  */
2036                 ftrace_function_enabled = 1;
2037         }
2038         mutex_unlock(&trace_types_lock);
2039
2040         seq_release(inode, file);
2041         kfree(iter);
2042         return 0;
2043 }
2044
2045 static int tracing_open(struct inode *inode, struct file *file)
2046 {
2047         int ret;
2048
2049         __tracing_open(inode, file, &ret);
2050
2051         return ret;
2052 }
2053
2054 static int tracing_lt_open(struct inode *inode, struct file *file)
2055 {
2056         struct trace_iterator *iter;
2057         int ret;
2058
2059         iter = __tracing_open(inode, file, &ret);
2060
2061         if (!ret)
2062                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2063
2064         return ret;
2065 }
2066
2067
2068 static void *
2069 t_next(struct seq_file *m, void *v, loff_t *pos)
2070 {
2071         struct tracer *t = m->private;
2072
2073         (*pos)++;
2074
2075         if (t)
2076                 t = t->next;
2077
2078         m->private = t;
2079
2080         return t;
2081 }
2082
2083 static void *t_start(struct seq_file *m, loff_t *pos)
2084 {
2085         struct tracer *t = m->private;
2086         loff_t l = 0;
2087
2088         mutex_lock(&trace_types_lock);
2089         for (; t && l < *pos; t = t_next(m, t, &l))
2090                 ;
2091
2092         return t;
2093 }
2094
2095 static void t_stop(struct seq_file *m, void *p)
2096 {
2097         mutex_unlock(&trace_types_lock);
2098 }
2099
2100 static int t_show(struct seq_file *m, void *v)
2101 {
2102         struct tracer *t = v;
2103
2104         if (!t)
2105                 return 0;
2106
2107         seq_printf(m, "%s", t->name);
2108         if (t->next)
2109                 seq_putc(m, ' ');
2110         else
2111                 seq_putc(m, '\n');
2112
2113         return 0;
2114 }
2115
2116 static struct seq_operations show_traces_seq_ops = {
2117         .start          = t_start,
2118         .next           = t_next,
2119         .stop           = t_stop,
2120         .show           = t_show,
2121 };
2122
2123 static int show_traces_open(struct inode *inode, struct file *file)
2124 {
2125         int ret;
2126
2127         if (tracing_disabled)
2128                 return -ENODEV;
2129
2130         ret = seq_open(file, &show_traces_seq_ops);
2131         if (!ret) {
2132                 struct seq_file *m = file->private_data;
2133                 m->private = trace_types;
2134         }
2135
2136         return ret;
2137 }
2138
2139 static struct file_operations tracing_fops = {
2140         .open           = tracing_open,
2141         .read           = seq_read,
2142         .llseek         = seq_lseek,
2143         .release        = tracing_release,
2144 };
2145
2146 static struct file_operations tracing_lt_fops = {
2147         .open           = tracing_lt_open,
2148         .read           = seq_read,
2149         .llseek         = seq_lseek,
2150         .release        = tracing_release,
2151 };
2152
2153 static struct file_operations show_traces_fops = {
2154         .open           = show_traces_open,
2155         .read           = seq_read,
2156         .release        = seq_release,
2157 };
2158
2159 /*
2160  * Only trace on a CPU if the bitmask is set:
2161  */
2162 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2163
2164 /*
2165  * When tracing/tracing_cpu_mask is modified then this holds
2166  * the new bitmask we are about to install:
2167  */
2168 static cpumask_t tracing_cpumask_new;
2169
2170 /*
2171  * The tracer itself will not take this lock, but still we want
2172  * to provide a consistent cpumask to user-space:
2173  */
2174 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2175
2176 /*
2177  * Temporary storage for the character representation of the
2178  * CPU bitmask (and one more byte for the newline):
2179  */
2180 static char mask_str[NR_CPUS + 1];
2181
2182 static ssize_t
2183 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2184                      size_t count, loff_t *ppos)
2185 {
2186         int len;
2187
2188         mutex_lock(&tracing_cpumask_update_lock);
2189
2190         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2191         if (count - len < 2) {
2192                 count = -EINVAL;
2193                 goto out_err;
2194         }
2195         len += sprintf(mask_str + len, "\n");
2196         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2197
2198 out_err:
2199         mutex_unlock(&tracing_cpumask_update_lock);
2200
2201         return count;
2202 }
2203
2204 static ssize_t
2205 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2206                       size_t count, loff_t *ppos)
2207 {
2208         int err, cpu;
2209
2210         mutex_lock(&tracing_cpumask_update_lock);
2211         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2212         if (err)
2213                 goto err_unlock;
2214
2215         raw_local_irq_disable();
2216         __raw_spin_lock(&ftrace_max_lock);
2217         for_each_tracing_cpu(cpu) {
2218                 /*
2219                  * Increase/decrease the disabled counter if we are
2220                  * about to flip a bit in the cpumask:
2221                  */
2222                 if (cpu_isset(cpu, tracing_cpumask) &&
2223                                 !cpu_isset(cpu, tracing_cpumask_new)) {
2224                         atomic_inc(&global_trace.data[cpu]->disabled);
2225                 }
2226                 if (!cpu_isset(cpu, tracing_cpumask) &&
2227                                 cpu_isset(cpu, tracing_cpumask_new)) {
2228                         atomic_dec(&global_trace.data[cpu]->disabled);
2229                 }
2230         }
2231         __raw_spin_unlock(&ftrace_max_lock);
2232         raw_local_irq_enable();
2233
2234         tracing_cpumask = tracing_cpumask_new;
2235
2236         mutex_unlock(&tracing_cpumask_update_lock);
2237
2238         return count;
2239
2240 err_unlock:
2241         mutex_unlock(&tracing_cpumask_update_lock);
2242
2243         return err;
2244 }
2245
2246 static struct file_operations tracing_cpumask_fops = {
2247         .open           = tracing_open_generic,
2248         .read           = tracing_cpumask_read,
2249         .write          = tracing_cpumask_write,
2250 };
2251
2252 static ssize_t
2253 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2254                        size_t cnt, loff_t *ppos)
2255 {
2256         char *buf;
2257         int r = 0;
2258         int len = 0;
2259         int i;
2260
2261         /* calulate max size */
2262         for (i = 0; trace_options[i]; i++) {
2263                 len += strlen(trace_options[i]);
2264                 len += 3; /* "no" and space */
2265         }
2266
2267         /* +2 for \n and \0 */
2268         buf = kmalloc(len + 2, GFP_KERNEL);
2269         if (!buf)
2270                 return -ENOMEM;
2271
2272         for (i = 0; trace_options[i]; i++) {
2273                 if (trace_flags & (1 << i))
2274                         r += sprintf(buf + r, "%s ", trace_options[i]);
2275                 else
2276                         r += sprintf(buf + r, "no%s ", trace_options[i]);
2277         }
2278
2279         r += sprintf(buf + r, "\n");
2280         WARN_ON(r >= len + 2);
2281
2282         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2283
2284         kfree(buf);
2285
2286         return r;
2287 }
2288
2289 static ssize_t
2290 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2291                         size_t cnt, loff_t *ppos)
2292 {
2293         char buf[64];
2294         char *cmp = buf;
2295         int neg = 0;
2296         int i;
2297
2298         if (cnt >= sizeof(buf))
2299                 return -EINVAL;
2300
2301         if (copy_from_user(&buf, ubuf, cnt))
2302                 return -EFAULT;
2303
2304         buf[cnt] = 0;
2305
2306         if (strncmp(buf, "no", 2) == 0) {
2307                 neg = 1;
2308                 cmp += 2;
2309         }
2310
2311         for (i = 0; trace_options[i]; i++) {
2312                 int len = strlen(trace_options[i]);
2313
2314                 if (strncmp(cmp, trace_options[i], len) == 0) {
2315                         if (neg)
2316                                 trace_flags &= ~(1 << i);
2317                         else
2318                                 trace_flags |= (1 << i);
2319                         break;
2320                 }
2321         }
2322         /*
2323          * If no option could be set, return an error:
2324          */
2325         if (!trace_options[i])
2326                 return -EINVAL;
2327
2328         filp->f_pos += cnt;
2329
2330         return cnt;
2331 }
2332
2333 static struct file_operations tracing_iter_fops = {
2334         .open           = tracing_open_generic,
2335         .read           = tracing_iter_ctrl_read,
2336         .write          = tracing_iter_ctrl_write,
2337 };
2338
2339 static const char readme_msg[] =
2340         "tracing mini-HOWTO:\n\n"
2341         "# mkdir /debug\n"
2342         "# mount -t debugfs nodev /debug\n\n"
2343         "# cat /debug/tracing/available_tracers\n"
2344         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2345         "# cat /debug/tracing/current_tracer\n"
2346         "none\n"
2347         "# echo sched_switch > /debug/tracing/current_tracer\n"
2348         "# cat /debug/tracing/current_tracer\n"
2349         "sched_switch\n"
2350         "# cat /debug/tracing/iter_ctrl\n"
2351         "noprint-parent nosym-offset nosym-addr noverbose\n"
2352         "# echo print-parent > /debug/tracing/iter_ctrl\n"
2353         "# echo 1 > /debug/tracing/tracing_enabled\n"
2354         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2355         "echo 0 > /debug/tracing/tracing_enabled\n"
2356 ;
2357
2358 static ssize_t
2359 tracing_readme_read(struct file *filp, char __user *ubuf,
2360                        size_t cnt, loff_t *ppos)
2361 {
2362         return simple_read_from_buffer(ubuf, cnt, ppos,
2363                                         readme_msg, strlen(readme_msg));
2364 }
2365
2366 static struct file_operations tracing_readme_fops = {
2367         .open           = tracing_open_generic,
2368         .read           = tracing_readme_read,
2369 };
2370
2371 static ssize_t
2372 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2373                   size_t cnt, loff_t *ppos)
2374 {
2375         struct trace_array *tr = filp->private_data;
2376         char buf[64];
2377         int r;
2378
2379         r = sprintf(buf, "%ld\n", tr->ctrl);
2380         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2381 }
2382
2383 static ssize_t
2384 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2385                    size_t cnt, loff_t *ppos)
2386 {
2387         struct trace_array *tr = filp->private_data;
2388         char buf[64];
2389         long val;
2390         int ret;
2391
2392         if (cnt >= sizeof(buf))
2393                 return -EINVAL;
2394
2395         if (copy_from_user(&buf, ubuf, cnt))
2396                 return -EFAULT;
2397
2398         buf[cnt] = 0;
2399
2400         ret = strict_strtoul(buf, 10, &val);
2401         if (ret < 0)
2402                 return ret;
2403
2404         val = !!val;
2405
2406         mutex_lock(&trace_types_lock);
2407         if (tr->ctrl ^ val) {
2408                 if (val)
2409                         tracer_enabled = 1;
2410                 else
2411                         tracer_enabled = 0;
2412
2413                 tr->ctrl = val;
2414
2415                 if (current_trace && current_trace->ctrl_update)
2416                         current_trace->ctrl_update(tr);
2417         }
2418         mutex_unlock(&trace_types_lock);
2419
2420         filp->f_pos += cnt;
2421
2422         return cnt;
2423 }
2424
2425 static ssize_t
2426 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2427                        size_t cnt, loff_t *ppos)
2428 {
2429         char buf[max_tracer_type_len+2];
2430         int r;
2431
2432         mutex_lock(&trace_types_lock);
2433         if (current_trace)
2434                 r = sprintf(buf, "%s\n", current_trace->name);
2435         else
2436                 r = sprintf(buf, "\n");
2437         mutex_unlock(&trace_types_lock);
2438
2439         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2440 }
2441
2442 static int tracing_set_tracer(char *buf)
2443 {
2444         struct trace_array *tr = &global_trace;
2445         struct tracer *t;
2446         int ret = 0;
2447
2448         mutex_lock(&trace_types_lock);
2449         for (t = trace_types; t; t = t->next) {
2450                 if (strcmp(t->name, buf) == 0)
2451                         break;
2452         }
2453         if (!t) {
2454                 ret = -EINVAL;
2455                 goto out;
2456         }
2457         if (t == current_trace)
2458                 goto out;
2459
2460         if (current_trace && current_trace->reset)
2461                 current_trace->reset(tr);
2462
2463         current_trace = t;
2464         if (t->init)
2465                 t->init(tr);
2466
2467  out:
2468         mutex_unlock(&trace_types_lock);
2469
2470         return ret;
2471 }
2472
2473 static ssize_t
2474 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2475                         size_t cnt, loff_t *ppos)
2476 {
2477         char buf[max_tracer_type_len+1];
2478         int i;
2479         size_t ret;
2480
2481         if (cnt > max_tracer_type_len)
2482                 cnt = max_tracer_type_len;
2483
2484         if (copy_from_user(&buf, ubuf, cnt))
2485                 return -EFAULT;
2486
2487         buf[cnt] = 0;
2488
2489         /* strip ending whitespace. */
2490         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2491                 buf[i] = 0;
2492
2493         ret = tracing_set_tracer(buf);
2494         if (!ret)
2495                 ret = cnt;
2496
2497         if (ret > 0)
2498                 filp->f_pos += ret;
2499
2500         return ret;
2501 }
2502
2503 static ssize_t
2504 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2505                      size_t cnt, loff_t *ppos)
2506 {
2507         unsigned long *ptr = filp->private_data;
2508         char buf[64];
2509         int r;
2510
2511         r = snprintf(buf, sizeof(buf), "%ld\n",
2512                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2513         if (r > sizeof(buf))
2514                 r = sizeof(buf);
2515         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2516 }
2517
2518 static ssize_t
2519 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2520                       size_t cnt, loff_t *ppos)
2521 {
2522         long *ptr = filp->private_data;
2523         char buf[64];
2524         long val;
2525         int ret;
2526
2527         if (cnt >= sizeof(buf))
2528                 return -EINVAL;
2529
2530         if (copy_from_user(&buf, ubuf, cnt))
2531                 return -EFAULT;
2532
2533         buf[cnt] = 0;
2534
2535         ret = strict_strtoul(buf, 10, &val);
2536         if (ret < 0)
2537                 return ret;
2538
2539         *ptr = val * 1000;
2540
2541         return cnt;
2542 }
2543
2544 static atomic_t tracing_reader;
2545
2546 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2547 {
2548         struct trace_iterator *iter;
2549
2550         if (tracing_disabled)
2551                 return -ENODEV;
2552
2553         /* We only allow for reader of the pipe */
2554         if (atomic_inc_return(&tracing_reader) != 1) {
2555                 atomic_dec(&tracing_reader);
2556                 return -EBUSY;
2557         }
2558
2559         /* create a buffer to store the information to pass to userspace */
2560         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2561         if (!iter)
2562                 return -ENOMEM;
2563
2564         mutex_lock(&trace_types_lock);
2565         iter->tr = &global_trace;
2566         iter->trace = current_trace;
2567         filp->private_data = iter;
2568
2569         if (iter->trace->pipe_open)
2570                 iter->trace->pipe_open(iter);
2571         mutex_unlock(&trace_types_lock);
2572
2573         return 0;
2574 }
2575
2576 static int tracing_release_pipe(struct inode *inode, struct file *file)
2577 {
2578         struct trace_iterator *iter = file->private_data;
2579
2580         kfree(iter);
2581         atomic_dec(&tracing_reader);
2582
2583         return 0;
2584 }
2585
2586 static unsigned int
2587 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2588 {
2589         struct trace_iterator *iter = filp->private_data;
2590
2591         if (trace_flags & TRACE_ITER_BLOCK) {
2592                 /*
2593                  * Always select as readable when in blocking mode
2594                  */
2595                 return POLLIN | POLLRDNORM;
2596         } else {
2597                 if (!trace_empty(iter))
2598                         return POLLIN | POLLRDNORM;
2599                 poll_wait(filp, &trace_wait, poll_table);
2600                 if (!trace_empty(iter))
2601                         return POLLIN | POLLRDNORM;
2602
2603                 return 0;
2604         }
2605 }
2606
2607 /*
2608  * Consumer reader.
2609  */
2610 static ssize_t
2611 tracing_read_pipe(struct file *filp, char __user *ubuf,
2612                   size_t cnt, loff_t *ppos)
2613 {
2614         struct trace_iterator *iter = filp->private_data;
2615         ssize_t sret;
2616
2617         /* return any leftover data */
2618         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2619         if (sret != -EBUSY)
2620                 return sret;
2621
2622         trace_seq_reset(&iter->seq);
2623
2624         mutex_lock(&trace_types_lock);
2625         if (iter->trace->read) {
2626                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2627                 if (sret)
2628                         goto out;
2629         }
2630
2631 waitagain:
2632         sret = 0;
2633         while (trace_empty(iter)) {
2634
2635                 if ((filp->f_flags & O_NONBLOCK)) {
2636                         sret = -EAGAIN;
2637                         goto out;
2638                 }
2639
2640                 /*
2641                  * This is a make-shift waitqueue. The reason we don't use
2642                  * an actual wait queue is because:
2643                  *  1) we only ever have one waiter
2644                  *  2) the tracing, traces all functions, we don't want
2645                  *     the overhead of calling wake_up and friends
2646                  *     (and tracing them too)
2647                  *     Anyway, this is really very primitive wakeup.
2648                  */
2649                 set_current_state(TASK_INTERRUPTIBLE);
2650                 iter->tr->waiter = current;
2651
2652                 mutex_unlock(&trace_types_lock);
2653
2654                 /* sleep for 100 msecs, and try again. */
2655                 schedule_timeout(HZ/10);
2656
2657                 mutex_lock(&trace_types_lock);
2658
2659                 iter->tr->waiter = NULL;
2660
2661                 if (signal_pending(current)) {
2662                         sret = -EINTR;
2663                         goto out;
2664                 }
2665
2666                 if (iter->trace != current_trace)
2667                         goto out;
2668
2669                 /*
2670                  * We block until we read something and tracing is disabled.
2671                  * We still block if tracing is disabled, but we have never
2672                  * read anything. This allows a user to cat this file, and
2673                  * then enable tracing. But after we have read something,
2674                  * we give an EOF when tracing is again disabled.
2675                  *
2676                  * iter->pos will be 0 if we haven't read anything.
2677                  */
2678                 if (!tracer_enabled && iter->pos)
2679                         break;
2680
2681                 continue;
2682         }
2683
2684         /* stop when tracing is finished */
2685         if (trace_empty(iter))
2686                 goto out;
2687
2688         if (cnt >= PAGE_SIZE)
2689                 cnt = PAGE_SIZE - 1;
2690
2691         /* reset all but tr, trace, and overruns */
2692         memset(&iter->seq, 0,
2693                sizeof(struct trace_iterator) -
2694                offsetof(struct trace_iterator, seq));
2695         iter->pos = -1;
2696
2697         while (find_next_entry_inc(iter) != NULL) {
2698                 enum print_line_t ret;
2699                 int len = iter->seq.len;
2700
2701                 ret = print_trace_line(iter);
2702                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2703                         /* don't print partial lines */
2704                         iter->seq.len = len;
2705                         break;
2706                 }
2707
2708                 trace_consume(iter);
2709
2710                 if (iter->seq.len >= cnt)
2711                         break;
2712         }
2713
2714         /* Now copy what we have to the user */
2715         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2716         if (iter->seq.readpos >= iter->seq.len)
2717                 trace_seq_reset(&iter->seq);
2718
2719         /*
2720          * If there was nothing to send to user, inspite of consuming trace
2721          * entries, go back to wait for more entries.
2722          */
2723         if (sret == -EBUSY)
2724                 goto waitagain;
2725
2726 out:
2727         mutex_unlock(&trace_types_lock);
2728
2729         return sret;
2730 }
2731
2732 static ssize_t
2733 tracing_entries_read(struct file *filp, char __user *ubuf,
2734                      size_t cnt, loff_t *ppos)
2735 {
2736         struct trace_array *tr = filp->private_data;
2737         char buf[64];
2738         int r;
2739
2740         r = sprintf(buf, "%lu\n", tr->entries);
2741         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2742 }
2743
2744 static ssize_t
2745 tracing_entries_write(struct file *filp, const char __user *ubuf,
2746                       size_t cnt, loff_t *ppos)
2747 {
2748         unsigned long val;
2749         char buf[64];
2750         int ret;
2751         struct trace_array *tr = filp->private_data;
2752
2753         if (cnt >= sizeof(buf))
2754                 return -EINVAL;
2755
2756         if (copy_from_user(&buf, ubuf, cnt))
2757                 return -EFAULT;
2758
2759         buf[cnt] = 0;
2760
2761         ret = strict_strtoul(buf, 10, &val);
2762         if (ret < 0)
2763                 return ret;
2764
2765         /* must have at least 1 entry */
2766         if (!val)
2767                 return -EINVAL;
2768
2769         mutex_lock(&trace_types_lock);
2770
2771         if (tr->ctrl) {
2772                 cnt = -EBUSY;
2773                 pr_info("ftrace: please disable tracing"
2774                         " before modifying buffer size\n");
2775                 goto out;
2776         }
2777
2778         if (val != global_trace.entries) {
2779                 ret = ring_buffer_resize(global_trace.buffer, val);
2780                 if (ret < 0) {
2781                         cnt = ret;
2782                         goto out;
2783                 }
2784
2785                 ret = ring_buffer_resize(max_tr.buffer, val);
2786                 if (ret < 0) {
2787                         int r;
2788                         cnt = ret;
2789                         r = ring_buffer_resize(global_trace.buffer,
2790                                                global_trace.entries);
2791                         if (r < 0) {
2792                                 /* AARGH! We are left with different
2793                                  * size max buffer!!!! */
2794                                 WARN_ON(1);
2795                                 tracing_disabled = 1;
2796                         }
2797                         goto out;
2798                 }
2799
2800                 global_trace.entries = val;
2801         }
2802
2803         filp->f_pos += cnt;
2804
2805         /* If check pages failed, return ENOMEM */
2806         if (tracing_disabled)
2807                 cnt = -ENOMEM;
2808  out:
2809         max_tr.entries = global_trace.entries;
2810         mutex_unlock(&trace_types_lock);
2811
2812         return cnt;
2813 }
2814
2815 static int mark_printk(const char *fmt, ...)
2816 {
2817         int ret;
2818         va_list args;
2819         va_start(args, fmt);
2820         ret = trace_vprintk(0, fmt, args);
2821         va_end(args);
2822         return ret;
2823 }
2824
2825 static ssize_t
2826 tracing_mark_write(struct file *filp, const char __user *ubuf,
2827                                         size_t cnt, loff_t *fpos)
2828 {
2829         char *buf;
2830         char *end;
2831         struct trace_array *tr = &global_trace;
2832
2833         if (!tr->ctrl || tracing_disabled)
2834                 return -EINVAL;
2835
2836         if (cnt > TRACE_BUF_SIZE)
2837                 cnt = TRACE_BUF_SIZE;
2838
2839         buf = kmalloc(cnt + 1, GFP_KERNEL);
2840         if (buf == NULL)
2841                 return -ENOMEM;
2842
2843         if (copy_from_user(buf, ubuf, cnt)) {
2844                 kfree(buf);
2845                 return -EFAULT;
2846         }
2847
2848         /* Cut from the first nil or newline. */
2849         buf[cnt] = '\0';
2850         end = strchr(buf, '\n');
2851         if (end)
2852                 *end = '\0';
2853
2854         cnt = mark_printk("%s\n", buf);
2855         kfree(buf);
2856         *fpos += cnt;
2857
2858         return cnt;
2859 }
2860
2861 static struct file_operations tracing_max_lat_fops = {
2862         .open           = tracing_open_generic,
2863         .read           = tracing_max_lat_read,
2864         .write          = tracing_max_lat_write,
2865 };
2866
2867 static struct file_operations tracing_ctrl_fops = {
2868         .open           = tracing_open_generic,
2869         .read           = tracing_ctrl_read,
2870         .write          = tracing_ctrl_write,
2871 };
2872
2873 static struct file_operations set_tracer_fops = {
2874         .open           = tracing_open_generic,
2875         .read           = tracing_set_trace_read,
2876         .write          = tracing_set_trace_write,
2877 };
2878
2879 static struct file_operations tracing_pipe_fops = {
2880         .open           = tracing_open_pipe,
2881         .poll           = tracing_poll_pipe,
2882         .read           = tracing_read_pipe,
2883         .release        = tracing_release_pipe,
2884 };
2885
2886 static struct file_operations tracing_entries_fops = {
2887         .open           = tracing_open_generic,
2888         .read           = tracing_entries_read,
2889         .write          = tracing_entries_write,
2890 };
2891
2892 static struct file_operations tracing_mark_fops = {
2893         .open           = tracing_open_generic,
2894         .write          = tracing_mark_write,
2895 };
2896
2897 #ifdef CONFIG_DYNAMIC_FTRACE
2898
2899 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
2900 {
2901         return 0;
2902 }
2903
2904 static ssize_t
2905 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
2906                   size_t cnt, loff_t *ppos)
2907 {
2908         static char ftrace_dyn_info_buffer[1024];
2909         static DEFINE_MUTEX(dyn_info_mutex);
2910         unsigned long *p = filp->private_data;
2911         char *buf = ftrace_dyn_info_buffer;
2912         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
2913         int r;
2914
2915         mutex_lock(&dyn_info_mutex);
2916         r = sprintf(buf, "%ld ", *p);
2917
2918         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
2919         buf[r++] = '\n';
2920
2921         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2922
2923         mutex_unlock(&dyn_info_mutex);
2924
2925         return r;
2926 }
2927
2928 static struct file_operations tracing_dyn_info_fops = {
2929         .open           = tracing_open_generic,
2930         .read           = tracing_read_dyn_info,
2931 };
2932 #endif
2933
2934 static struct dentry *d_tracer;
2935
2936 struct dentry *tracing_init_dentry(void)
2937 {
2938         static int once;
2939
2940         if (d_tracer)
2941                 return d_tracer;
2942
2943         d_tracer = debugfs_create_dir("tracing", NULL);
2944
2945         if (!d_tracer && !once) {
2946                 once = 1;
2947                 pr_warning("Could not create debugfs directory 'tracing'\n");
2948                 return NULL;
2949         }
2950
2951         return d_tracer;
2952 }
2953
2954 #ifdef CONFIG_FTRACE_SELFTEST
2955 /* Let selftest have access to static functions in this file */
2956 #include "trace_selftest.c"
2957 #endif
2958
2959 static __init int tracer_init_debugfs(void)
2960 {
2961         struct dentry *d_tracer;
2962         struct dentry *entry;
2963
2964         d_tracer = tracing_init_dentry();
2965
2966         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2967                                     &global_trace, &tracing_ctrl_fops);
2968         if (!entry)
2969                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2970
2971         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2972                                     NULL, &tracing_iter_fops);
2973         if (!entry)
2974                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2975
2976         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2977                                     NULL, &tracing_cpumask_fops);
2978         if (!entry)
2979                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2980
2981         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2982                                     &global_trace, &tracing_lt_fops);
2983         if (!entry)
2984                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2985
2986         entry = debugfs_create_file("trace", 0444, d_tracer,
2987                                     &global_trace, &tracing_fops);
2988         if (!entry)
2989                 pr_warning("Could not create debugfs 'trace' entry\n");
2990
2991         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2992                                     &global_trace, &show_traces_fops);
2993         if (!entry)
2994                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
2995
2996         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2997                                     &global_trace, &set_tracer_fops);
2998         if (!entry)
2999                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
3000
3001         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
3002                                     &tracing_max_latency,
3003                                     &tracing_max_lat_fops);
3004         if (!entry)
3005                 pr_warning("Could not create debugfs "
3006                            "'tracing_max_latency' entry\n");
3007
3008         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
3009                                     &tracing_thresh, &tracing_max_lat_fops);
3010         if (!entry)
3011                 pr_warning("Could not create debugfs "
3012                            "'tracing_thresh' entry\n");
3013         entry = debugfs_create_file("README", 0644, d_tracer,
3014                                     NULL, &tracing_readme_fops);
3015         if (!entry)
3016                 pr_warning("Could not create debugfs 'README' entry\n");
3017
3018         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
3019                                     NULL, &tracing_pipe_fops);
3020         if (!entry)
3021                 pr_warning("Could not create debugfs "
3022                            "'trace_pipe' entry\n");
3023
3024         entry = debugfs_create_file("trace_entries", 0644, d_tracer,
3025                                     &global_trace, &tracing_entries_fops);
3026         if (!entry)
3027                 pr_warning("Could not create debugfs "
3028                            "'trace_entries' entry\n");
3029
3030         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
3031                                     NULL, &tracing_mark_fops);
3032         if (!entry)
3033                 pr_warning("Could not create debugfs "
3034                            "'trace_marker' entry\n");
3035
3036 #ifdef CONFIG_DYNAMIC_FTRACE
3037         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3038                                     &ftrace_update_tot_cnt,
3039                                     &tracing_dyn_info_fops);
3040         if (!entry)
3041                 pr_warning("Could not create debugfs "
3042                            "'dyn_ftrace_total_info' entry\n");
3043 #endif
3044 #ifdef CONFIG_SYSPROF_TRACER
3045         init_tracer_sysprof_debugfs(d_tracer);
3046 #endif
3047         return 0;
3048 }
3049
3050 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3051 {
3052         static DEFINE_SPINLOCK(trace_buf_lock);
3053         static char trace_buf[TRACE_BUF_SIZE];
3054
3055         struct ring_buffer_event *event;
3056         struct trace_array *tr = &global_trace;
3057         struct trace_array_cpu *data;
3058         struct print_entry *entry;
3059         unsigned long flags, irq_flags;
3060         int cpu, len = 0, size, pc;
3061
3062         if (!tr->ctrl || tracing_disabled)
3063                 return 0;
3064
3065         pc = preempt_count();
3066         preempt_disable_notrace();
3067         cpu = raw_smp_processor_id();
3068         data = tr->data[cpu];
3069
3070         if (unlikely(atomic_read(&data->disabled)))
3071                 goto out;
3072
3073         spin_lock_irqsave(&trace_buf_lock, flags);
3074         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3075
3076         len = min(len, TRACE_BUF_SIZE-1);
3077         trace_buf[len] = 0;
3078
3079         size = sizeof(*entry) + len + 1;
3080         event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
3081         if (!event)
3082                 goto out_unlock;
3083         entry = ring_buffer_event_data(event);
3084         tracing_generic_entry_update(&entry->ent, flags, pc);
3085         entry->ent.type                 = TRACE_PRINT;
3086         entry->ip                       = ip;
3087
3088         memcpy(&entry->buf, trace_buf, len);
3089         entry->buf[len] = 0;
3090         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3091
3092  out_unlock:
3093         spin_unlock_irqrestore(&trace_buf_lock, flags);
3094
3095  out:
3096         preempt_enable_notrace();
3097
3098         return len;
3099 }
3100 EXPORT_SYMBOL_GPL(trace_vprintk);
3101
3102 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3103 {
3104         int ret;
3105         va_list ap;
3106
3107         if (!(trace_flags & TRACE_ITER_PRINTK))
3108                 return 0;
3109
3110         va_start(ap, fmt);
3111         ret = trace_vprintk(ip, fmt, ap);
3112         va_end(ap);
3113         return ret;
3114 }
3115 EXPORT_SYMBOL_GPL(__ftrace_printk);
3116
3117 static int trace_panic_handler(struct notifier_block *this,
3118                                unsigned long event, void *unused)
3119 {
3120         if (ftrace_dump_on_oops)
3121                 ftrace_dump();
3122         return NOTIFY_OK;
3123 }
3124
3125 static struct notifier_block trace_panic_notifier = {
3126         .notifier_call  = trace_panic_handler,
3127         .next           = NULL,
3128         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
3129 };
3130
3131 static int trace_die_handler(struct notifier_block *self,
3132                              unsigned long val,
3133                              void *data)
3134 {
3135         switch (val) {
3136         case DIE_OOPS:
3137                 if (ftrace_dump_on_oops)
3138                         ftrace_dump();
3139                 break;
3140         default:
3141                 break;
3142         }
3143         return NOTIFY_OK;
3144 }
3145
3146 static struct notifier_block trace_die_notifier = {
3147         .notifier_call = trace_die_handler,
3148         .priority = 200
3149 };
3150
3151 /*
3152  * printk is set to max of 1024, we really don't need it that big.
3153  * Nothing should be printing 1000 characters anyway.
3154  */
3155 #define TRACE_MAX_PRINT         1000
3156
3157 /*
3158  * Define here KERN_TRACE so that we have one place to modify
3159  * it if we decide to change what log level the ftrace dump
3160  * should be at.
3161  */
3162 #define KERN_TRACE              KERN_INFO
3163
3164 static void
3165 trace_printk_seq(struct trace_seq *s)
3166 {
3167         /* Probably should print a warning here. */
3168         if (s->len >= 1000)
3169                 s->len = 1000;
3170
3171         /* should be zero ended, but we are paranoid. */
3172         s->buffer[s->len] = 0;
3173
3174         printk(KERN_TRACE "%s", s->buffer);
3175
3176         trace_seq_reset(s);
3177 }
3178
3179 void ftrace_dump(void)
3180 {
3181         static DEFINE_SPINLOCK(ftrace_dump_lock);
3182         /* use static because iter can be a bit big for the stack */
3183         static struct trace_iterator iter;
3184         static cpumask_t mask;
3185         static int dump_ran;
3186         unsigned long flags;
3187         int cnt = 0, cpu;
3188
3189         /* only one dump */
3190         spin_lock_irqsave(&ftrace_dump_lock, flags);
3191         if (dump_ran)
3192                 goto out;
3193
3194         dump_ran = 1;
3195
3196         /* No turning back! */
3197         ftrace_kill();
3198
3199         for_each_tracing_cpu(cpu) {
3200                 atomic_inc(&global_trace.data[cpu]->disabled);
3201         }
3202
3203         printk(KERN_TRACE "Dumping ftrace buffer:\n");
3204
3205         iter.tr = &global_trace;
3206         iter.trace = current_trace;
3207
3208         /*
3209          * We need to stop all tracing on all CPUS to read the
3210          * the next buffer. This is a bit expensive, but is
3211          * not done often. We fill all what we can read,
3212          * and then release the locks again.
3213          */
3214
3215         cpus_clear(mask);
3216
3217         while (!trace_empty(&iter)) {
3218
3219                 if (!cnt)
3220                         printk(KERN_TRACE "---------------------------------\n");
3221
3222                 cnt++;
3223
3224                 /* reset all but tr, trace, and overruns */
3225                 memset(&iter.seq, 0,
3226                        sizeof(struct trace_iterator) -
3227                        offsetof(struct trace_iterator, seq));
3228                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3229                 iter.pos = -1;
3230
3231                 if (find_next_entry_inc(&iter) != NULL) {
3232                         print_trace_line(&iter);
3233                         trace_consume(&iter);
3234                 }
3235
3236                 trace_printk_seq(&iter.seq);
3237         }
3238
3239         if (!cnt)
3240                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3241         else
3242                 printk(KERN_TRACE "---------------------------------\n");
3243
3244  out:
3245         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3246 }
3247
3248 __init static int tracer_alloc_buffers(void)
3249 {
3250         struct trace_array_cpu *data;
3251         int i;
3252
3253         /* TODO: make the number of buffers hot pluggable with CPUS */
3254         tracing_buffer_mask = cpu_possible_map;
3255
3256         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3257                                                    TRACE_BUFFER_FLAGS);
3258         if (!global_trace.buffer) {
3259                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3260                 WARN_ON(1);
3261                 return 0;
3262         }
3263         global_trace.entries = ring_buffer_size(global_trace.buffer);
3264
3265 #ifdef CONFIG_TRACER_MAX_TRACE
3266         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3267                                              TRACE_BUFFER_FLAGS);
3268         if (!max_tr.buffer) {
3269                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3270                 WARN_ON(1);
3271                 ring_buffer_free(global_trace.buffer);
3272                 return 0;
3273         }
3274         max_tr.entries = ring_buffer_size(max_tr.buffer);
3275         WARN_ON(max_tr.entries != global_trace.entries);
3276 #endif
3277
3278         /* Allocate the first page for all buffers */
3279         for_each_tracing_cpu(i) {
3280                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3281                 max_tr.data[i] = &per_cpu(max_data, i);
3282         }
3283
3284         trace_init_cmdlines();
3285
3286         register_tracer(&nop_trace);
3287 #ifdef CONFIG_BOOT_TRACER
3288         register_tracer(&boot_tracer);
3289         current_trace = &boot_tracer;
3290         current_trace->init(&global_trace);
3291 #else
3292         current_trace = &nop_trace;
3293 #endif
3294
3295         /* All seems OK, enable tracing */
3296         global_trace.ctrl = tracer_enabled;
3297         tracing_disabled = 0;
3298
3299         atomic_notifier_chain_register(&panic_notifier_list,
3300                                        &trace_panic_notifier);
3301
3302         register_die_notifier(&trace_die_notifier);
3303
3304         return 0;
3305 }
3306 early_initcall(tracer_alloc_buffers);
3307 fs_initcall(tracer_init_debugfs);