git.oblomov.eu Git - linux-2.6/blob - kernel/trace/trace.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally taken from the RT patch by:
   8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code from the latency_tracer, that is:
  11  *  Copyright (C) 2004-2006 Ingo Molnar
  12  *  Copyright (C) 2004 William Lee Irwin III
  13  */
  14 #include <linux/utsrelease.h>
  15 #include <linux/kallsyms.h>
  16 #include <linux/seq_file.h>
  17 #include <linux/notifier.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/hardirq.h>
  21 #include <linux/linkage.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/ftrace.h>
  24 #include <linux/module.h>
  25 #include <linux/percpu.h>
  26 #include <linux/kdebug.h>
  27 #include <linux/ctype.h>
  28 #include <linux/init.h>
  29 #include <linux/poll.h>
  30 #include <linux/gfp.h>
  31 #include <linux/fs.h>
  32 #include <linux/kprobes.h>
  33 #include <linux/writeback.h>
  34
  35 #include <linux/stacktrace.h>
  36 #include <linux/ring_buffer.h>
  37
  38 #include "trace.h"
  39
  40 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
  41
  42 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
  43 unsigned long __read_mostly     tracing_thresh;
  44
  45 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
  46
  47 static inline void ftrace_disable_cpu(void)
  48 {
  49         preempt_disable();
  50         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
  51 }
  52
  53 static inline void ftrace_enable_cpu(void)
  54 {
  55         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
  56         preempt_enable();
  57 }
  58
  59 static cpumask_t __read_mostly          tracing_buffer_mask;
  60
  61 #define for_each_tracing_cpu(cpu)       \
  62         for_each_cpu_mask(cpu, tracing_buffer_mask)
  63
  64 static int tracing_disabled = 1;
  65
  66 long
  67 ns2usecs(cycle_t nsec)
  68 {
  69         nsec += 500;
  70         do_div(nsec, 1000);
  71         return nsec;
  72 }
  73
  74 cycle_t ftrace_now(int cpu)
  75 {
  76         u64 ts = ring_buffer_time_stamp(cpu);
  77         ring_buffer_normalize_time_stamp(cpu, &ts);
  78         return ts;
  79 }
  80
  81 /*
  82  * The global_trace is the descriptor that holds the tracing
  83  * buffers for the live tracing. For each CPU, it contains
  84  * a link list of pages that will store trace entries. The
  85  * page descriptor of the pages in the memory is used to hold
  86  * the link list by linking the lru item in the page descriptor
  87  * to each of the pages in the buffer per CPU.
  88  *
  89  * For each active CPU there is a data field that holds the
  90  * pages for the buffer for that CPU. Each CPU has the same number
  91  * of pages allocated for its buffer.
  92  */
  93 static struct trace_array       global_trace;
  94
  95 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
  96
  97 /*
  98  * The max_tr is used to snapshot the global_trace when a maximum
  99  * latency is reached. Some tracers will use this to store a maximum
 100  * trace while it continues examining live traces.
 101  *
 102  * The buffers for the max_tr are set up the same as the global_trace.
 103  * When a snapshot is taken, the link list of the max_tr is swapped
 104  * with the link list of the global_trace and the buffers are reset for
 105  * the global_trace so the tracing can continue.
 106  */
 107 static struct trace_array       max_tr;
 108
 109 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
 110
 111 /* tracer_enabled is used to toggle activation of a tracer */
 112 static int                      tracer_enabled = 1;
 113
 114 /* function tracing enabled */
 115 int                             ftrace_function_enabled;
 116
 117 /*
 118  * trace_buf_size is the size in bytes that is allocated
 119  * for a buffer. Note, the number of bytes is always rounded
 120  * to page size.
 121  *
 122  * This number is purposely set to a low number of 16384.
 123  * If the dump on oops happens, it will be much appreciated
 124  * to not have to wait for all that output. Anyway this can be
 125  * boot time and run time configurable.
 126  */
 127 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 128
 129 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 130
 131 /* trace_types holds a link list of available tracers. */
 132 static struct tracer            *trace_types __read_mostly;
 133
 134 /* current_trace points to the tracer that is currently active */
 135 static struct tracer            *current_trace __read_mostly;
 136
 137 /*
 138  * max_tracer_type_len is used to simplify the allocating of
 139  * buffers to read userspace tracer names. We keep track of
 140  * the longest tracer name registered.
 141  */
 142 static int                      max_tracer_type_len;
 143
 144 /*
 145  * trace_types_lock is used to protect the trace_types list.
 146  * This lock is also used to keep user access serialized.
 147  * Accesses from userspace will grab this lock while userspace
 148  * activities happen inside the kernel.
 149  */
 150 static DEFINE_MUTEX(trace_types_lock);
 151
 152 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
 153 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 154
 155 /* trace_flags holds iter_ctrl options */
 156 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
 157
 158 /**
 159  * trace_wake_up - wake up tasks waiting for trace input
 160  *
 161  * Simply wakes up any task that is blocked on the trace_wait
 162  * queue. These is used with trace_poll for tasks polling the trace.
 163  */
 164 void trace_wake_up(void)
 165 {
 166         /*
 167          * The runqueue_is_locked() can fail, but this is the best we
 168          * have for now:
 169          */
 170         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
 171                 wake_up(&trace_wait);
 172 }
 173
 174 static int __init set_buf_size(char *str)
 175 {
 176         unsigned long buf_size;
 177         int ret;
 178
 179         if (!str)
 180                 return 0;
 181         ret = strict_strtoul(str, 0, &buf_size);
 182         /* nr_entries can not be zero */
 183         if (ret < 0 || buf_size == 0)
 184                 return 0;
 185         trace_buf_size = buf_size;
 186         return 1;
 187 }
 188 __setup("trace_buf_size=", set_buf_size);
 189
 190 unsigned long nsecs_to_usecs(unsigned long nsecs)
 191 {
 192         return nsecs / 1000;
 193 }
 194
 195 /*
 196  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
 197  * control the output of kernel symbols.
 198  */
 199 #define TRACE_ITER_SYM_MASK \
 200         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
 201
 202 /* These must match the bit postions in trace_iterator_flags */
 203 static const char *trace_options[] = {
 204         "print-parent",
 205         "sym-offset",
 206         "sym-addr",
 207         "verbose",
 208         "raw",
 209         "hex",
 210         "bin",
 211         "block",
 212         "stacktrace",
 213         "sched-tree",
 214         "ftrace_printk",
 215         NULL
 216 };
 217
 218 /*
 219  * ftrace_max_lock is used to protect the swapping of buffers
 220  * when taking a max snapshot. The buffers themselves are
 221  * protected by per_cpu spinlocks. But the action of the swap
 222  * needs its own lock.
 223  *
 224  * This is defined as a raw_spinlock_t in order to help
 225  * with performance when lockdep debugging is enabled.
 226  */
 227 static raw_spinlock_t ftrace_max_lock =
 228         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
 229
 230 /*
 231  * Copy the new maximum trace into the separate maximum-trace
 232  * structure. (this way the maximum trace is permanently saved,
 233  * for later retrieval via /debugfs/tracing/latency_trace)
 234  */
 235 static void
 236 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 237 {
 238         struct trace_array_cpu *data = tr->data[cpu];
 239
 240         max_tr.cpu = cpu;
 241         max_tr.time_start = data->preempt_timestamp;
 242
 243         data = max_tr.data[cpu];
 244         data->saved_latency = tracing_max_latency;
 245
 246         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
 247         data->pid = tsk->pid;
 248         data->uid = tsk->uid;
 249         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 250         data->policy = tsk->policy;
 251         data->rt_priority = tsk->rt_priority;
 252
 253         /* record this tasks comm */
 254         tracing_record_cmdline(current);
 255 }
 256
 257 /**
 258  * trace_seq_printf - sequence printing of trace information
 259  * @s: trace sequence descriptor
 260  * @fmt: printf format string
 261  *
 262  * The tracer may use either sequence operations or its own
 263  * copy to user routines. To simplify formating of a trace
 264  * trace_seq_printf is used to store strings into a special
 265  * buffer (@s). Then the output may be either used by
 266  * the sequencer or pulled into another buffer.
 267  */
 268 int
 269 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
 270 {
 271         int len = (PAGE_SIZE - 1) - s->len;
 272         va_list ap;
 273         int ret;
 274
 275         if (!len)
 276                 return 0;
 277
 278         va_start(ap, fmt);
 279         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
 280         va_end(ap);
 281
 282         /* If we can't write it all, don't bother writing anything */
 283         if (ret >= len)
 284                 return 0;
 285
 286         s->len += ret;
 287
 288         return len;
 289 }
 290
 291 /**
 292  * trace_seq_puts - trace sequence printing of simple string
 293  * @s: trace sequence descriptor
 294  * @str: simple string to record
 295  *
 296  * The tracer may use either the sequence operations or its own
 297  * copy to user routines. This function records a simple string
 298  * into a special buffer (@s) for later retrieval by a sequencer
 299  * or other mechanism.
 300  */
 301 static int
 302 trace_seq_puts(struct trace_seq *s, const char *str)
 303 {
 304         int len = strlen(str);
 305
 306         if (len > ((PAGE_SIZE - 1) - s->len))
 307                 return 0;
 308
 309         memcpy(s->buffer + s->len, str, len);
 310         s->len += len;
 311
 312         return len;
 313 }
 314
 315 static int
 316 trace_seq_putc(struct trace_seq *s, unsigned char c)
 317 {
 318         if (s->len >= (PAGE_SIZE - 1))
 319                 return 0;
 320
 321         s->buffer[s->len++] = c;
 322
 323         return 1;
 324 }
 325
 326 static int
 327 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
 328 {
 329         if (len > ((PAGE_SIZE - 1) - s->len))
 330                 return 0;
 331
 332         memcpy(s->buffer + s->len, mem, len);
 333         s->len += len;
 334
 335         return len;
 336 }
 337
 338 #define MAX_MEMHEX_BYTES        8
 339 #define HEX_CHARS               (MAX_MEMHEX_BYTES*2 + 1)
 340
 341 static int
 342 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
 343 {
 344         unsigned char hex[HEX_CHARS];
 345         unsigned char *data = mem;
 346         int i, j;
 347
 348 #ifdef __BIG_ENDIAN
 349         for (i = 0, j = 0; i < len; i++) {
 350 #else
 351         for (i = len-1, j = 0; i >= 0; i--) {
 352 #endif
 353                 hex[j++] = hex_asc_hi(data[i]);
 354                 hex[j++] = hex_asc_lo(data[i]);
 355         }
 356         hex[j++] = ' ';
 357
 358         return trace_seq_putmem(s, hex, j);
 359 }
 360
 361 static void
 362 trace_seq_reset(struct trace_seq *s)
 363 {
 364         s->len = 0;
 365         s->readpos = 0;
 366 }
 367
 368 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 369 {
 370         int len;
 371         int ret;
 372
 373         if (s->len <= s->readpos)
 374                 return -EBUSY;
 375
 376         len = s->len - s->readpos;
 377         if (cnt > len)
 378                 cnt = len;
 379         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 380         if (ret)
 381                 return -EFAULT;
 382
 383         s->readpos += len;
 384         return cnt;
 385 }
 386
 387 static void
 388 trace_print_seq(struct seq_file *m, struct trace_seq *s)
 389 {
 390         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
 391
 392         s->buffer[len] = 0;
 393         seq_puts(m, s->buffer);
 394
 395         trace_seq_reset(s);
 396 }
 397
 398 /**
 399  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 400  * @tr: tracer
 401  * @tsk: the task with the latency
 402  * @cpu: The cpu that initiated the trace.
 403  *
 404  * Flip the buffers between the @tr and the max_tr and record information
 405  * about which task was the cause of this latency.
 406  */
 407 void
 408 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 409 {
 410         struct ring_buffer *buf = tr->buffer;
 411
 412         WARN_ON_ONCE(!irqs_disabled());
 413         __raw_spin_lock(&ftrace_max_lock);
 414
 415         tr->buffer = max_tr.buffer;
 416         max_tr.buffer = buf;
 417
 418         ftrace_disable_cpu();
 419         ring_buffer_reset(tr->buffer);
 420         ftrace_enable_cpu();
 421
 422         __update_max_tr(tr, tsk, cpu);
 423         __raw_spin_unlock(&ftrace_max_lock);
 424 }
 425
 426 /**
 427  * update_max_tr_single - only copy one trace over, and reset the rest
 428  * @tr - tracer
 429  * @tsk - task with the latency
 430  * @cpu - the cpu of the buffer to copy.
 431  *
 432  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 433  */
 434 void
 435 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 436 {
 437         int ret;
 438
 439         WARN_ON_ONCE(!irqs_disabled());
 440         __raw_spin_lock(&ftrace_max_lock);
 441
 442         ftrace_disable_cpu();
 443
 444         ring_buffer_reset(max_tr.buffer);
 445         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 446
 447         ftrace_enable_cpu();
 448
 449         WARN_ON_ONCE(ret);
 450
 451         __update_max_tr(tr, tsk, cpu);
 452         __raw_spin_unlock(&ftrace_max_lock);
 453 }
 454
 455 /**
 456  * register_tracer - register a tracer with the ftrace system.
 457  * @type - the plugin for the tracer
 458  *
 459  * Register a new plugin tracer.
 460  */
 461 int register_tracer(struct tracer *type)
 462 {
 463         struct tracer *t;
 464         int len;
 465         int ret = 0;
 466
 467         if (!type->name) {
 468                 pr_info("Tracer must have a name\n");
 469                 return -1;
 470         }
 471
 472         mutex_lock(&trace_types_lock);
 473         for (t = trace_types; t; t = t->next) {
 474                 if (strcmp(type->name, t->name) == 0) {
 475                         /* already found */
 476                         pr_info("Trace %s already registered\n",
 477                                 type->name);
 478                         ret = -1;
 479                         goto out;
 480                 }
 481         }
 482
 483 #ifdef CONFIG_FTRACE_STARTUP_TEST
 484         if (type->selftest) {
 485                 struct tracer *saved_tracer = current_trace;
 486                 struct trace_array *tr = &global_trace;
 487                 int saved_ctrl = tr->ctrl;
 488                 int i;
 489                 /*
 490                  * Run a selftest on this tracer.
 491                  * Here we reset the trace buffer, and set the current
 492                  * tracer to be this tracer. The tracer can then run some
 493                  * internal tracing to verify that everything is in order.
 494                  * If we fail, we do not register this tracer.
 495                  */
 496                 for_each_tracing_cpu(i) {
 497                         tracing_reset(tr, i);
 498                 }
 499                 current_trace = type;
 500                 tr->ctrl = 0;
 501                 /* the test is responsible for initializing and enabling */
 502                 pr_info("Testing tracer %s: ", type->name);
 503                 ret = type->selftest(type, tr);
 504                 /* the test is responsible for resetting too */
 505                 current_trace = saved_tracer;
 506                 tr->ctrl = saved_ctrl;
 507                 if (ret) {
 508                         printk(KERN_CONT "FAILED!\n");
 509                         goto out;
 510                 }
 511                 /* Only reset on passing, to avoid touching corrupted buffers */
 512                 for_each_tracing_cpu(i) {
 513                         tracing_reset(tr, i);
 514                 }
 515                 printk(KERN_CONT "PASSED\n");
 516         }
 517 #endif
 518
 519         type->next = trace_types;
 520         trace_types = type;
 521         len = strlen(type->name);
 522         if (len > max_tracer_type_len)
 523                 max_tracer_type_len = len;
 524
 525  out:
 526         mutex_unlock(&trace_types_lock);
 527
 528         return ret;
 529 }
 530
 531 void unregister_tracer(struct tracer *type)
 532 {
 533         struct tracer **t;
 534         int len;
 535
 536         mutex_lock(&trace_types_lock);
 537         for (t = &trace_types; *t; t = &(*t)->next) {
 538                 if (*t == type)
 539                         goto found;
 540         }
 541         pr_info("Trace %s not registered\n", type->name);
 542         goto out;
 543
 544  found:
 545         *t = (*t)->next;
 546         if (strlen(type->name) != max_tracer_type_len)
 547                 goto out;
 548
 549         max_tracer_type_len = 0;
 550         for (t = &trace_types; *t; t = &(*t)->next) {
 551                 len = strlen((*t)->name);
 552                 if (len > max_tracer_type_len)
 553                         max_tracer_type_len = len;
 554         }
 555  out:
 556         mutex_unlock(&trace_types_lock);
 557 }
 558
 559 void tracing_reset(struct trace_array *tr, int cpu)
 560 {
 561         ftrace_disable_cpu();
 562         ring_buffer_reset_cpu(tr->buffer, cpu);
 563         ftrace_enable_cpu();
 564 }
 565
 566 #define SAVED_CMDLINES 128
 567 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 568 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 569 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 570 static int cmdline_idx;
 571 static DEFINE_SPINLOCK(trace_cmdline_lock);
 572
 573 /* temporary disable recording */
 574 atomic_t trace_record_cmdline_disabled __read_mostly;
 575
 576 static void trace_init_cmdlines(void)
 577 {
 578         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
 579         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
 580         cmdline_idx = 0;
 581 }
 582
 583 void trace_stop_cmdline_recording(void);
 584
 585 static void trace_save_cmdline(struct task_struct *tsk)
 586 {
 587         unsigned map;
 588         unsigned idx;
 589
 590         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
 591                 return;
 592
 593         /*
 594          * It's not the end of the world if we don't get
 595          * the lock, but we also don't want to spin
 596          * nor do we want to disable interrupts,
 597          * so if we miss here, then better luck next time.
 598          */
 599         if (!spin_trylock(&trace_cmdline_lock))
 600                 return;
 601
 602         idx = map_pid_to_cmdline[tsk->pid];
 603         if (idx >= SAVED_CMDLINES) {
 604                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 605
 606                 map = map_cmdline_to_pid[idx];
 607                 if (map <= PID_MAX_DEFAULT)
 608                         map_pid_to_cmdline[map] = (unsigned)-1;
 609
 610                 map_pid_to_cmdline[tsk->pid] = idx;
 611
 612                 cmdline_idx = idx;
 613         }
 614
 615         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 616
 617         spin_unlock(&trace_cmdline_lock);
 618 }
 619
 620 static char *trace_find_cmdline(int pid)
 621 {
 622         char *cmdline = "<...>";
 623         unsigned map;
 624
 625         if (!pid)
 626                 return "<idle>";
 627
 628         if (pid > PID_MAX_DEFAULT)
 629                 goto out;
 630
 631         map = map_pid_to_cmdline[pid];
 632         if (map >= SAVED_CMDLINES)
 633                 goto out;
 634
 635         cmdline = saved_cmdlines[map];
 636
 637  out:
 638         return cmdline;
 639 }
 640
 641 void tracing_record_cmdline(struct task_struct *tsk)
 642 {
 643         if (atomic_read(&trace_record_cmdline_disabled))
 644                 return;
 645
 646         trace_save_cmdline(tsk);
 647 }
 648
 649 void
 650 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 651                              int pc)
 652 {
 653         struct task_struct *tsk = current;
 654
 655         entry->preempt_count            = pc & 0xff;
 656         entry->pid                      = (tsk) ? tsk->pid : 0;
 657         entry->flags =
 658                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
 659                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 660                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
 661                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 662 }
 663
 664 void
 665 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
 666                unsigned long ip, unsigned long parent_ip, unsigned long flags,
 667                int pc)
 668 {
 669         struct ring_buffer_event *event;
 670         struct ftrace_entry *entry;
 671         unsigned long irq_flags;
 672
 673         /* If we are reading the ring buffer, don't trace */
 674         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
 675                 return;
 676
 677         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 678                                          &irq_flags);
 679         if (!event)
 680                 return;
 681         entry   = ring_buffer_event_data(event);
 682         tracing_generic_entry_update(&entry->ent, flags, pc);
 683         entry->ent.type                 = TRACE_FN;
 684         entry->ip                       = ip;
 685         entry->parent_ip                = parent_ip;
 686         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 687 }
 688
 689 void
 690 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
 691        unsigned long ip, unsigned long parent_ip, unsigned long flags,
 692        int pc)
 693 {
 694         if (likely(!atomic_read(&data->disabled)))
 695                 trace_function(tr, data, ip, parent_ip, flags, pc);
 696 }
 697
 698 static void ftrace_trace_stack(struct trace_array *tr,
 699                                struct trace_array_cpu *data,
 700                                unsigned long flags,
 701                                int skip, int pc)
 702 {
 703         struct ring_buffer_event *event;
 704         struct stack_entry *entry;
 705         struct stack_trace trace;
 706         unsigned long irq_flags;
 707
 708         if (!(trace_flags & TRACE_ITER_STACKTRACE))
 709                 return;
 710
 711         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 712                                          &irq_flags);
 713         if (!event)
 714                 return;
 715         entry   = ring_buffer_event_data(event);
 716         tracing_generic_entry_update(&entry->ent, flags, pc);
 717         entry->ent.type         = TRACE_STACK;
 718
 719         memset(&entry->caller, 0, sizeof(entry->caller));
 720
 721         trace.nr_entries        = 0;
 722         trace.max_entries       = FTRACE_STACK_ENTRIES;
 723         trace.skip              = skip;
 724         trace.entries           = entry->caller;
 725
 726         save_stack_trace(&trace);
 727         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 728 }
 729
 730 void __trace_stack(struct trace_array *tr,
 731                    struct trace_array_cpu *data,
 732                    unsigned long flags,
 733                    int skip)
 734 {
 735         ftrace_trace_stack(tr, data, flags, skip, preempt_count());
 736 }
 737
 738 static void
 739 ftrace_trace_special(void *__tr, void *__data,
 740                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
 741                      int pc)
 742 {
 743         struct ring_buffer_event *event;
 744         struct trace_array_cpu *data = __data;
 745         struct trace_array *tr = __tr;
 746         struct special_entry *entry;
 747         unsigned long irq_flags;
 748
 749         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 750                                          &irq_flags);
 751         if (!event)
 752                 return;
 753         entry   = ring_buffer_event_data(event);
 754         tracing_generic_entry_update(&entry->ent, 0, pc);
 755         entry->ent.type                 = TRACE_SPECIAL;
 756         entry->arg1                     = arg1;
 757         entry->arg2                     = arg2;
 758         entry->arg3                     = arg3;
 759         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 760         ftrace_trace_stack(tr, data, irq_flags, 4, pc);
 761
 762         trace_wake_up();
 763 }
 764
 765 void
 766 __trace_special(void *__tr, void *__data,
 767                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
 768 {
 769         ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
 770 }
 771
 772 void
 773 tracing_sched_switch_trace(struct trace_array *tr,
 774                            struct trace_array_cpu *data,
 775                            struct task_struct *prev,
 776                            struct task_struct *next,
 777                            unsigned long flags, int pc)
 778 {
 779         struct ring_buffer_event *event;
 780         struct ctx_switch_entry *entry;
 781         unsigned long irq_flags;
 782
 783         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 784                                            &irq_flags);
 785         if (!event)
 786                 return;
 787         entry   = ring_buffer_event_data(event);
 788         tracing_generic_entry_update(&entry->ent, flags, pc);
 789         entry->ent.type                 = TRACE_CTX;
 790         entry->prev_pid                 = prev->pid;
 791         entry->prev_prio                = prev->prio;
 792         entry->prev_state               = prev->state;
 793         entry->next_pid                 = next->pid;
 794         entry->next_prio                = next->prio;
 795         entry->next_state               = next->state;
 796         entry->next_cpu = task_cpu(next);
 797         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 798         ftrace_trace_stack(tr, data, flags, 5, pc);
 799 }
 800
 801 void
 802 tracing_sched_wakeup_trace(struct trace_array *tr,
 803                            struct trace_array_cpu *data,
 804                            struct task_struct *wakee,
 805                            struct task_struct *curr,
 806                            unsigned long flags, int pc)
 807 {
 808         struct ring_buffer_event *event;
 809         struct ctx_switch_entry *entry;
 810         unsigned long irq_flags;
 811
 812         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 813                                            &irq_flags);
 814         if (!event)
 815                 return;
 816         entry   = ring_buffer_event_data(event);
 817         tracing_generic_entry_update(&entry->ent, flags, pc);
 818         entry->ent.type                 = TRACE_WAKE;
 819         entry->prev_pid                 = curr->pid;
 820         entry->prev_prio                = curr->prio;
 821         entry->prev_state               = curr->state;
 822         entry->next_pid                 = wakee->pid;
 823         entry->next_prio                = wakee->prio;
 824         entry->next_state               = wakee->state;
 825         entry->next_cpu                 = task_cpu(wakee);
 826         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 827         ftrace_trace_stack(tr, data, flags, 6, pc);
 828
 829         trace_wake_up();
 830 }
 831
 832 void
 833 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
 834 {
 835         struct trace_array *tr = &global_trace;
 836         struct trace_array_cpu *data;
 837         int cpu;
 838         int pc;
 839
 840         if (tracing_disabled || !tr->ctrl)
 841                 return;
 842
 843         pc = preempt_count();
 844         preempt_disable_notrace();
 845         cpu = raw_smp_processor_id();
 846         data = tr->data[cpu];
 847
 848         if (likely(!atomic_read(&data->disabled)))
 849                 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
 850
 851         preempt_enable_notrace();
 852 }
 853
 854 #ifdef CONFIG_FTRACE
 855 static void
 856 function_trace_call(unsigned long ip, unsigned long parent_ip)
 857 {
 858         struct trace_array *tr = &global_trace;
 859         struct trace_array_cpu *data;
 860         unsigned long flags;
 861         long disabled;
 862         int cpu, resched;
 863         int pc;
 864
 865         if (unlikely(!ftrace_function_enabled))
 866                 return;
 867
 868         if (skip_trace(ip))
 869                 return;
 870
 871         pc = preempt_count();
 872         resched = need_resched();
 873         preempt_disable_notrace();
 874         local_save_flags(flags);
 875         cpu = raw_smp_processor_id();
 876         data = tr->data[cpu];
 877         disabled = atomic_inc_return(&data->disabled);
 878
 879         if (likely(disabled == 1))
 880                 trace_function(tr, data, ip, parent_ip, flags, pc);
 881
 882         atomic_dec(&data->disabled);
 883         if (resched)
 884                 preempt_enable_no_resched_notrace();
 885         else
 886                 preempt_enable_notrace();
 887 }
 888
 889 static struct ftrace_ops trace_ops __read_mostly =
 890 {
 891         .func = function_trace_call,
 892 };
 893
 894 void tracing_start_function_trace(void)
 895 {
 896         ftrace_function_enabled = 0;
 897         register_ftrace_function(&trace_ops);
 898         if (tracer_enabled)
 899                 ftrace_function_enabled = 1;
 900 }
 901
 902 void tracing_stop_function_trace(void)
 903 {
 904         ftrace_function_enabled = 0;
 905         unregister_ftrace_function(&trace_ops);
 906 }
 907 #endif
 908
 909 enum trace_file_type {
 910         TRACE_FILE_LAT_FMT      = 1,
 911 };
 912
 913 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
 914 {
 915         /* Don't allow ftrace to trace into the ring buffers */
 916         ftrace_disable_cpu();
 917
 918         iter->idx++;
 919         if (iter->buffer_iter[iter->cpu])
 920                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
 921
 922         ftrace_enable_cpu();
 923 }
 924
 925 static struct trace_entry *
 926 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
 927 {
 928         struct ring_buffer_event *event;
 929         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
 930
 931         /* Don't allow ftrace to trace into the ring buffers */
 932         ftrace_disable_cpu();
 933
 934         if (buf_iter)
 935                 event = ring_buffer_iter_peek(buf_iter, ts);
 936         else
 937                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
 938
 939         ftrace_enable_cpu();
 940
 941         return event ? ring_buffer_event_data(event) : NULL;
 942 }
 943
 944 static struct trace_entry *
 945 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
 946 {
 947         struct ring_buffer *buffer = iter->tr->buffer;
 948         struct trace_entry *ent, *next = NULL;
 949         u64 next_ts = 0, ts;
 950         int next_cpu = -1;
 951         int cpu;
 952
 953         for_each_tracing_cpu(cpu) {
 954
 955                 if (ring_buffer_empty_cpu(buffer, cpu))
 956                         continue;
 957
 958                 ent = peek_next_entry(iter, cpu, &ts);
 959
 960                 /*
 961                  * Pick the entry with the smallest timestamp:
 962                  */
 963                 if (ent && (!next || ts < next_ts)) {
 964                         next = ent;
 965                         next_cpu = cpu;
 966                         next_ts = ts;
 967                 }
 968         }
 969
 970         if (ent_cpu)
 971                 *ent_cpu = next_cpu;
 972
 973         if (ent_ts)
 974                 *ent_ts = next_ts;
 975
 976         return next;
 977 }
 978
 979 /* Find the next real entry, without updating the iterator itself */
 980 static struct trace_entry *
 981 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
 982 {
 983         return __find_next_entry(iter, ent_cpu, ent_ts);
 984 }
 985
 986 /* Find the next real entry, and increment the iterator to the next entry */
 987 static void *find_next_entry_inc(struct trace_iterator *iter)
 988 {
 989         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
 990
 991         if (iter->ent)
 992                 trace_iterator_increment(iter, iter->cpu);
 993
 994         return iter->ent ? iter : NULL;
 995 }
 996
 997 static void trace_consume(struct trace_iterator *iter)
 998 {
 999         /* Don't allow ftrace to trace into the ring buffers */
1000         ftrace_disable_cpu();
1001         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1002         ftrace_enable_cpu();
1003 }
1004
1005 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1006 {
1007         struct trace_iterator *iter = m->private;
1008         int i = (int)*pos;
1009         void *ent;
1010
1011         (*pos)++;
1012
1013         /* can't go backwards */
1014         if (iter->idx > i)
1015                 return NULL;
1016
1017         if (iter->idx < 0)
1018                 ent = find_next_entry_inc(iter);
1019         else
1020                 ent = iter;
1021
1022         while (ent && iter->idx < i)
1023                 ent = find_next_entry_inc(iter);
1024
1025         iter->pos = *pos;
1026
1027         return ent;
1028 }
1029
1030 static void *s_start(struct seq_file *m, loff_t *pos)
1031 {
1032         struct trace_iterator *iter = m->private;
1033         void *p = NULL;
1034         loff_t l = 0;
1035         int cpu;
1036
1037         mutex_lock(&trace_types_lock);
1038
1039         if (!current_trace || current_trace != iter->trace) {
1040                 mutex_unlock(&trace_types_lock);
1041                 return NULL;
1042         }
1043
1044         atomic_inc(&trace_record_cmdline_disabled);
1045
1046         /* let the tracer grab locks here if needed */
1047         if (current_trace->start)
1048                 current_trace->start(iter);
1049
1050         if (*pos != iter->pos) {
1051                 iter->ent = NULL;
1052                 iter->cpu = 0;
1053                 iter->idx = -1;
1054
1055                 ftrace_disable_cpu();
1056
1057                 for_each_tracing_cpu(cpu) {
1058                         ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1059                 }
1060
1061                 ftrace_enable_cpu();
1062
1063                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1064                         ;
1065
1066         } else {
1067                 l = *pos - 1;
1068                 p = s_next(m, p, &l);
1069         }
1070
1071         return p;
1072 }
1073
1074 static void s_stop(struct seq_file *m, void *p)
1075 {
1076         struct trace_iterator *iter = m->private;
1077
1078         atomic_dec(&trace_record_cmdline_disabled);
1079
1080         /* let the tracer release locks here if needed */
1081         if (current_trace && current_trace == iter->trace && iter->trace->stop)
1082                 iter->trace->stop(iter);
1083
1084         mutex_unlock(&trace_types_lock);
1085 }
1086
1087 #define KRETPROBE_MSG "[unknown/kretprobe'd]"
1088
1089 #ifdef CONFIG_KRETPROBES
1090 static inline int kretprobed(unsigned long addr)
1091 {
1092         return addr == (unsigned long)kretprobe_trampoline;
1093 }
1094 #else
1095 static inline int kretprobed(unsigned long addr)
1096 {
1097         return 0;
1098 }
1099 #endif /* CONFIG_KRETPROBES */
1100
1101 static int
1102 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1103 {
1104 #ifdef CONFIG_KALLSYMS
1105         char str[KSYM_SYMBOL_LEN];
1106
1107         kallsyms_lookup(address, NULL, NULL, NULL, str);
1108
1109         return trace_seq_printf(s, fmt, str);
1110 #endif
1111         return 1;
1112 }
1113
1114 static int
1115 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1116                      unsigned long address)
1117 {
1118 #ifdef CONFIG_KALLSYMS
1119         char str[KSYM_SYMBOL_LEN];
1120
1121         sprint_symbol(str, address);
1122         return trace_seq_printf(s, fmt, str);
1123 #endif
1124         return 1;
1125 }
1126
1127 #ifndef CONFIG_64BIT
1128 # define IP_FMT "%08lx"
1129 #else
1130 # define IP_FMT "%016lx"
1131 #endif
1132
1133 static int
1134 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1135 {
1136         int ret;
1137
1138         if (!ip)
1139                 return trace_seq_printf(s, "0");
1140
1141         if (sym_flags & TRACE_ITER_SYM_OFFSET)
1142                 ret = seq_print_sym_offset(s, "%s", ip);
1143         else
1144                 ret = seq_print_sym_short(s, "%s", ip);
1145
1146         if (!ret)
1147                 return 0;
1148
1149         if (sym_flags & TRACE_ITER_SYM_ADDR)
1150                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1151         return ret;
1152 }
1153
1154 static void print_lat_help_header(struct seq_file *m)
1155 {
1156         seq_puts(m, "#                  _------=> CPU#            \n");
1157         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1158         seq_puts(m, "#                | / _----=> need-resched    \n");
1159         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1160         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1161         seq_puts(m, "#                |||| /                      \n");
1162         seq_puts(m, "#                |||||     delay             \n");
1163         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1164         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1165 }
1166
1167 static void print_func_help_header(struct seq_file *m)
1168 {
1169         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1170         seq_puts(m, "#              | |       |          |         |\n");
1171 }
1172
1173
1174 static void
1175 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1176 {
1177         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1178         struct trace_array *tr = iter->tr;
1179         struct trace_array_cpu *data = tr->data[tr->cpu];
1180         struct tracer *type = current_trace;
1181         unsigned long total;
1182         unsigned long entries;
1183         const char *name = "preemption";
1184
1185         if (type)
1186                 name = type->name;
1187
1188         entries = ring_buffer_entries(iter->tr->buffer);
1189         total = entries +
1190                 ring_buffer_overruns(iter->tr->buffer);
1191
1192         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1193                    name, UTS_RELEASE);
1194         seq_puts(m, "-----------------------------------"
1195                  "---------------------------------\n");
1196         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1197                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1198                    nsecs_to_usecs(data->saved_latency),
1199                    entries,
1200                    total,
1201                    tr->cpu,
1202 #if defined(CONFIG_PREEMPT_NONE)
1203                    "server",
1204 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1205                    "desktop",
1206 #elif defined(CONFIG_PREEMPT)
1207                    "preempt",
1208 #else
1209                    "unknown",
1210 #endif
1211                    /* These are reserved for later use */
1212                    0, 0, 0, 0);
1213 #ifdef CONFIG_SMP
1214         seq_printf(m, " #P:%d)\n", num_online_cpus());
1215 #else
1216         seq_puts(m, ")\n");
1217 #endif
1218         seq_puts(m, "    -----------------\n");
1219         seq_printf(m, "    | task: %.16s-%d "
1220                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1221                    data->comm, data->pid, data->uid, data->nice,
1222                    data->policy, data->rt_priority);
1223         seq_puts(m, "    -----------------\n");
1224
1225         if (data->critical_start) {
1226                 seq_puts(m, " => started at: ");
1227                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1228                 trace_print_seq(m, &iter->seq);
1229                 seq_puts(m, "\n => ended at:   ");
1230                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1231                 trace_print_seq(m, &iter->seq);
1232                 seq_puts(m, "\n");
1233         }
1234
1235         seq_puts(m, "\n");
1236 }
1237
1238 static void
1239 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1240 {
1241         int hardirq, softirq;
1242         char *comm;
1243
1244         comm = trace_find_cmdline(entry->pid);
1245
1246         trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1247         trace_seq_printf(s, "%3d", cpu);
1248         trace_seq_printf(s, "%c%c",
1249                         (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1250                         ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1251
1252         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1253         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1254         if (hardirq && softirq) {
1255                 trace_seq_putc(s, 'H');
1256         } else {
1257                 if (hardirq) {
1258                         trace_seq_putc(s, 'h');
1259                 } else {
1260                         if (softirq)
1261                                 trace_seq_putc(s, 's');
1262                         else
1263                                 trace_seq_putc(s, '.');
1264                 }
1265         }
1266
1267         if (entry->preempt_count)
1268                 trace_seq_printf(s, "%x", entry->preempt_count);
1269         else
1270                 trace_seq_puts(s, ".");
1271 }
1272
1273 unsigned long preempt_mark_thresh = 100;
1274
1275 static void
1276 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1277                     unsigned long rel_usecs)
1278 {
1279         trace_seq_printf(s, " %4lldus", abs_usecs);
1280         if (rel_usecs > preempt_mark_thresh)
1281                 trace_seq_puts(s, "!: ");
1282         else if (rel_usecs > 1)
1283                 trace_seq_puts(s, "+: ");
1284         else
1285                 trace_seq_puts(s, " : ");
1286 }
1287
1288 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1289
1290 /*
1291  * The message is supposed to contain an ending newline.
1292  * If the printing stops prematurely, try to add a newline of our own.
1293  */
1294 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1295 {
1296         struct trace_entry *ent;
1297         struct trace_field_cont *cont;
1298         bool ok = true;
1299
1300         ent = peek_next_entry(iter, iter->cpu, NULL);
1301         if (!ent || ent->type != TRACE_CONT) {
1302                 trace_seq_putc(s, '\n');
1303                 return;
1304         }
1305
1306         do {
1307                 cont = (struct trace_field_cont *)ent;
1308                 if (ok)
1309                         ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1310
1311                 ftrace_disable_cpu();
1312
1313                 if (iter->buffer_iter[iter->cpu])
1314                         ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1315                 else
1316                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1317
1318                 ftrace_enable_cpu();
1319
1320                 ent = peek_next_entry(iter, iter->cpu, NULL);
1321         } while (ent && ent->type == TRACE_CONT);
1322
1323         if (!ok)
1324                 trace_seq_putc(s, '\n');
1325 }
1326
1327 static enum print_line_t
1328 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1329 {
1330         struct trace_seq *s = &iter->seq;
1331         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1332         struct trace_entry *next_entry;
1333         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1334         struct trace_entry *entry = iter->ent;
1335         unsigned long abs_usecs;
1336         unsigned long rel_usecs;
1337         u64 next_ts;
1338         char *comm;
1339         int S, T;
1340         int i;
1341         unsigned state;
1342
1343         if (entry->type == TRACE_CONT)
1344                 return TRACE_TYPE_HANDLED;
1345
1346         next_entry = find_next_entry(iter, NULL, &next_ts);
1347         if (!next_entry)
1348                 next_ts = iter->ts;
1349         rel_usecs = ns2usecs(next_ts - iter->ts);
1350         abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1351
1352         if (verbose) {
1353                 comm = trace_find_cmdline(entry->pid);
1354                 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1355                                  " %ld.%03ldms (+%ld.%03ldms): ",
1356                                  comm,
1357                                  entry->pid, cpu, entry->flags,
1358                                  entry->preempt_count, trace_idx,
1359                                  ns2usecs(iter->ts),
1360                                  abs_usecs/1000,
1361                                  abs_usecs % 1000, rel_usecs/1000,
1362                                  rel_usecs % 1000);
1363         } else {
1364                 lat_print_generic(s, entry, cpu);
1365                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1366         }
1367         switch (entry->type) {
1368         case TRACE_FN: {
1369                 struct ftrace_entry *field;
1370
1371                 trace_assign_type(field, entry);
1372
1373                 seq_print_ip_sym(s, field->ip, sym_flags);
1374                 trace_seq_puts(s, " (");
1375                 if (kretprobed(field->parent_ip))
1376                         trace_seq_puts(s, KRETPROBE_MSG);
1377                 else
1378                         seq_print_ip_sym(s, field->parent_ip, sym_flags);
1379                 trace_seq_puts(s, ")\n");
1380                 break;
1381         }
1382         case TRACE_CTX:
1383         case TRACE_WAKE: {
1384                 struct ctx_switch_entry *field;
1385
1386                 trace_assign_type(field, entry);
1387
1388                 T = field->next_state < sizeof(state_to_char) ?
1389                         state_to_char[field->next_state] : 'X';
1390
1391                 state = field->prev_state ?
1392                         __ffs(field->prev_state) + 1 : 0;
1393                 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1394                 comm = trace_find_cmdline(field->next_pid);
1395                 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1396                                  field->prev_pid,
1397                                  field->prev_prio,
1398                                  S, entry->type == TRACE_CTX ? "==>" : "  +",
1399                                  field->next_cpu,
1400                                  field->next_pid,
1401                                  field->next_prio,
1402                                  T, comm);
1403                 break;
1404         }
1405         case TRACE_SPECIAL: {
1406                 struct special_entry *field;
1407
1408                 trace_assign_type(field, entry);
1409
1410                 trace_seq_printf(s, "# %ld %ld %ld\n",
1411                                  field->arg1,
1412                                  field->arg2,
1413                                  field->arg3);
1414                 break;
1415         }
1416         case TRACE_STACK: {
1417                 struct stack_entry *field;
1418
1419                 trace_assign_type(field, entry);
1420
1421                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1422                         if (i)
1423                                 trace_seq_puts(s, " <= ");
1424                         seq_print_ip_sym(s, field->caller[i], sym_flags);
1425                 }
1426                 trace_seq_puts(s, "\n");
1427                 break;
1428         }
1429         case TRACE_PRINT: {
1430                 struct print_entry *field;
1431
1432                 trace_assign_type(field, entry);
1433
1434                 seq_print_ip_sym(s, field->ip, sym_flags);
1435                 trace_seq_printf(s, ": %s", field->buf);
1436                 if (entry->flags & TRACE_FLAG_CONT)
1437                         trace_seq_print_cont(s, iter);
1438                 break;
1439         }
1440         default:
1441                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1442         }
1443         return TRACE_TYPE_HANDLED;
1444 }
1445
1446 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1447 {
1448         struct trace_seq *s = &iter->seq;
1449         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1450         struct trace_entry *entry;
1451         unsigned long usec_rem;
1452         unsigned long long t;
1453         unsigned long secs;
1454         char *comm;
1455         int ret;
1456         int S, T;
1457         int i;
1458
1459         entry = iter->ent;
1460
1461         if (entry->type == TRACE_CONT)
1462                 return TRACE_TYPE_HANDLED;
1463
1464         comm = trace_find_cmdline(iter->ent->pid);
1465
1466         t = ns2usecs(iter->ts);
1467         usec_rem = do_div(t, 1000000ULL);
1468         secs = (unsigned long)t;
1469
1470         ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1471         if (!ret)
1472                 return TRACE_TYPE_PARTIAL_LINE;
1473         ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1474         if (!ret)
1475                 return TRACE_TYPE_PARTIAL_LINE;
1476         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1477         if (!ret)
1478                 return TRACE_TYPE_PARTIAL_LINE;
1479
1480         switch (entry->type) {
1481         case TRACE_FN: {
1482                 struct ftrace_entry *field;
1483
1484                 trace_assign_type(field, entry);
1485
1486                 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1487                 if (!ret)
1488                         return TRACE_TYPE_PARTIAL_LINE;
1489                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1490                                                 field->parent_ip) {
1491                         ret = trace_seq_printf(s, " <-");
1492                         if (!ret)
1493                                 return TRACE_TYPE_PARTIAL_LINE;
1494                         if (kretprobed(field->parent_ip))
1495                                 ret = trace_seq_puts(s, KRETPROBE_MSG);
1496                         else
1497                                 ret = seq_print_ip_sym(s,
1498                                                        field->parent_ip,
1499                                                        sym_flags);
1500                         if (!ret)
1501                                 return TRACE_TYPE_PARTIAL_LINE;
1502                 }
1503                 ret = trace_seq_printf(s, "\n");
1504                 if (!ret)
1505                         return TRACE_TYPE_PARTIAL_LINE;
1506                 break;
1507         }
1508         case TRACE_CTX:
1509         case TRACE_WAKE: {
1510                 struct ctx_switch_entry *field;
1511
1512                 trace_assign_type(field, entry);
1513
1514                 S = field->prev_state < sizeof(state_to_char) ?
1515                         state_to_char[field->prev_state] : 'X';
1516                 T = field->next_state < sizeof(state_to_char) ?
1517                         state_to_char[field->next_state] : 'X';
1518                 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1519                                        field->prev_pid,
1520                                        field->prev_prio,
1521                                        S,
1522                                        entry->type == TRACE_CTX ? "==>" : "  +",
1523                                        field->next_cpu,
1524                                        field->next_pid,
1525                                        field->next_prio,
1526                                        T);
1527                 if (!ret)
1528                         return TRACE_TYPE_PARTIAL_LINE;
1529                 break;
1530         }
1531         case TRACE_SPECIAL: {
1532                 struct special_entry *field;
1533
1534                 trace_assign_type(field, entry);
1535
1536                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1537                                  field->arg1,
1538                                  field->arg2,
1539                                  field->arg3);
1540                 if (!ret)
1541                         return TRACE_TYPE_PARTIAL_LINE;
1542                 break;
1543         }
1544         case TRACE_STACK: {
1545                 struct stack_entry *field;
1546
1547                 trace_assign_type(field, entry);
1548
1549                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1550                         if (i) {
1551                                 ret = trace_seq_puts(s, " <= ");
1552                                 if (!ret)
1553                                         return TRACE_TYPE_PARTIAL_LINE;
1554                         }
1555                         ret = seq_print_ip_sym(s, field->caller[i],
1556                                                sym_flags);
1557                         if (!ret)
1558                                 return TRACE_TYPE_PARTIAL_LINE;
1559                 }
1560                 ret = trace_seq_puts(s, "\n");
1561                 if (!ret)
1562                         return TRACE_TYPE_PARTIAL_LINE;
1563                 break;
1564         }
1565         case TRACE_PRINT: {
1566                 struct print_entry *field;
1567
1568                 trace_assign_type(field, entry);
1569
1570                 seq_print_ip_sym(s, field->ip, sym_flags);
1571                 trace_seq_printf(s, ": %s", field->buf);
1572                 if (entry->flags & TRACE_FLAG_CONT)
1573                         trace_seq_print_cont(s, iter);
1574                 break;
1575         }
1576         }
1577         return TRACE_TYPE_HANDLED;
1578 }
1579
1580 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1581 {
1582         struct trace_seq *s = &iter->seq;
1583         struct trace_entry *entry;
1584         int ret;
1585         int S, T;
1586
1587         entry = iter->ent;
1588
1589         if (entry->type == TRACE_CONT)
1590                 return TRACE_TYPE_HANDLED;
1591
1592         ret = trace_seq_printf(s, "%d %d %llu ",
1593                 entry->pid, iter->cpu, iter->ts);
1594         if (!ret)
1595                 return TRACE_TYPE_PARTIAL_LINE;
1596
1597         switch (entry->type) {
1598         case TRACE_FN: {
1599                 struct ftrace_entry *field;
1600
1601                 trace_assign_type(field, entry);
1602
1603                 ret = trace_seq_printf(s, "%x %x\n",
1604                                         field->ip,
1605                                         field->parent_ip);
1606                 if (!ret)
1607                         return TRACE_TYPE_PARTIAL_LINE;
1608                 break;
1609         }
1610         case TRACE_CTX:
1611         case TRACE_WAKE: {
1612                 struct ctx_switch_entry *field;
1613
1614                 trace_assign_type(field, entry);
1615
1616                 S = field->prev_state < sizeof(state_to_char) ?
1617                         state_to_char[field->prev_state] : 'X';
1618                 T = field->next_state < sizeof(state_to_char) ?
1619                         state_to_char[field->next_state] : 'X';
1620                 if (entry->type == TRACE_WAKE)
1621                         S = '+';
1622                 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1623                                        field->prev_pid,
1624                                        field->prev_prio,
1625                                        S,
1626                                        field->next_cpu,
1627                                        field->next_pid,
1628                                        field->next_prio,
1629                                        T);
1630                 if (!ret)
1631                         return TRACE_TYPE_PARTIAL_LINE;
1632                 break;
1633         }
1634         case TRACE_SPECIAL:
1635         case TRACE_STACK: {
1636                 struct special_entry *field;
1637
1638                 trace_assign_type(field, entry);
1639
1640                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1641                                  field->arg1,
1642                                  field->arg2,
1643                                  field->arg3);
1644                 if (!ret)
1645                         return TRACE_TYPE_PARTIAL_LINE;
1646                 break;
1647         }
1648         case TRACE_PRINT: {
1649                 struct print_entry *field;
1650
1651                 trace_assign_type(field, entry);
1652
1653                 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1654                 if (entry->flags & TRACE_FLAG_CONT)
1655                         trace_seq_print_cont(s, iter);
1656                 break;
1657         }
1658         }
1659         return TRACE_TYPE_HANDLED;
1660 }
1661
1662 #define SEQ_PUT_FIELD_RET(s, x)                         \
1663 do {                                                    \
1664         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1665                 return 0;                               \
1666 } while (0)
1667
1668 #define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
1669 do {                                                    \
1670         BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES);     \
1671         if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
1672                 return 0;                               \
1673 } while (0)
1674
1675 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1676 {
1677         struct trace_seq *s = &iter->seq;
1678         unsigned char newline = '\n';
1679         struct trace_entry *entry;
1680         int S, T;
1681
1682         entry = iter->ent;
1683
1684         if (entry->type == TRACE_CONT)
1685                 return TRACE_TYPE_HANDLED;
1686
1687         SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1688         SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1689         SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1690
1691         switch (entry->type) {
1692         case TRACE_FN: {
1693                 struct ftrace_entry *field;
1694
1695                 trace_assign_type(field, entry);
1696
1697                 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1698                 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1699                 break;
1700         }
1701         case TRACE_CTX:
1702         case TRACE_WAKE: {
1703                 struct ctx_switch_entry *field;
1704
1705                 trace_assign_type(field, entry);
1706
1707                 S = field->prev_state < sizeof(state_to_char) ?
1708                         state_to_char[field->prev_state] : 'X';
1709                 T = field->next_state < sizeof(state_to_char) ?
1710                         state_to_char[field->next_state] : 'X';
1711                 if (entry->type == TRACE_WAKE)
1712                         S = '+';
1713                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1714                 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1715                 SEQ_PUT_HEX_FIELD_RET(s, S);
1716                 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1717                 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1718                 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1719                 SEQ_PUT_HEX_FIELD_RET(s, T);
1720                 break;
1721         }
1722         case TRACE_SPECIAL:
1723         case TRACE_STACK: {
1724                 struct special_entry *field;
1725
1726                 trace_assign_type(field, entry);
1727
1728                 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1729                 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1730                 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1731                 break;
1732         }
1733         }
1734         SEQ_PUT_FIELD_RET(s, newline);
1735
1736         return TRACE_TYPE_HANDLED;
1737 }
1738
1739 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1740 {
1741         struct trace_seq *s = &iter->seq;
1742         struct trace_entry *entry;
1743
1744         entry = iter->ent;
1745
1746         if (entry->type == TRACE_CONT)
1747                 return TRACE_TYPE_HANDLED;
1748
1749         SEQ_PUT_FIELD_RET(s, entry->pid);
1750         SEQ_PUT_FIELD_RET(s, iter->cpu);
1751         SEQ_PUT_FIELD_RET(s, iter->ts);
1752
1753         switch (entry->type) {
1754         case TRACE_FN: {
1755                 struct ftrace_entry *field;
1756
1757                 trace_assign_type(field, entry);
1758
1759                 SEQ_PUT_FIELD_RET(s, field->ip);
1760                 SEQ_PUT_FIELD_RET(s, field->parent_ip);
1761                 break;
1762         }
1763         case TRACE_CTX: {
1764                 struct ctx_switch_entry *field;
1765
1766                 trace_assign_type(field, entry);
1767
1768                 SEQ_PUT_FIELD_RET(s, field->prev_pid);
1769                 SEQ_PUT_FIELD_RET(s, field->prev_prio);
1770                 SEQ_PUT_FIELD_RET(s, field->prev_state);
1771                 SEQ_PUT_FIELD_RET(s, field->next_pid);
1772                 SEQ_PUT_FIELD_RET(s, field->next_prio);
1773                 SEQ_PUT_FIELD_RET(s, field->next_state);
1774                 break;
1775         }
1776         case TRACE_SPECIAL:
1777         case TRACE_STACK: {
1778                 struct special_entry *field;
1779
1780                 trace_assign_type(field, entry);
1781
1782                 SEQ_PUT_FIELD_RET(s, field->arg1);
1783                 SEQ_PUT_FIELD_RET(s, field->arg2);
1784                 SEQ_PUT_FIELD_RET(s, field->arg3);
1785                 break;
1786         }
1787         }
1788         return 1;
1789 }
1790
1791 static int trace_empty(struct trace_iterator *iter)
1792 {
1793         int cpu;
1794
1795         for_each_tracing_cpu(cpu) {
1796                 if (iter->buffer_iter[cpu]) {
1797                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1798                                 return 0;
1799                 } else {
1800                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1801                                 return 0;
1802                 }
1803         }
1804
1805         return 1;
1806 }
1807
1808 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1809 {
1810         enum print_line_t ret;
1811
1812         if (iter->trace && iter->trace->print_line) {
1813                 ret = iter->trace->print_line(iter);
1814                 if (ret != TRACE_TYPE_UNHANDLED)
1815                         return ret;
1816         }
1817
1818         if (trace_flags & TRACE_ITER_BIN)
1819                 return print_bin_fmt(iter);
1820
1821         if (trace_flags & TRACE_ITER_HEX)
1822                 return print_hex_fmt(iter);
1823
1824         if (trace_flags & TRACE_ITER_RAW)
1825                 return print_raw_fmt(iter);
1826
1827         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1828                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1829
1830         return print_trace_fmt(iter);
1831 }
1832
1833 static int s_show(struct seq_file *m, void *v)
1834 {
1835         struct trace_iterator *iter = v;
1836
1837         if (iter->ent == NULL) {
1838                 if (iter->tr) {
1839                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1840                         seq_puts(m, "#\n");
1841                 }
1842                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1843                         /* print nothing if the buffers are empty */
1844                         if (trace_empty(iter))
1845                                 return 0;
1846                         print_trace_header(m, iter);
1847                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1848                                 print_lat_help_header(m);
1849                 } else {
1850                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1851                                 print_func_help_header(m);
1852                 }
1853         } else {
1854                 print_trace_line(iter);
1855                 trace_print_seq(m, &iter->seq);
1856         }
1857
1858         return 0;
1859 }
1860
1861 static struct seq_operations tracer_seq_ops = {
1862         .start          = s_start,
1863         .next           = s_next,
1864         .stop           = s_stop,
1865         .show           = s_show,
1866 };
1867
1868 static struct trace_iterator *
1869 __tracing_open(struct inode *inode, struct file *file, int *ret)
1870 {
1871         struct trace_iterator *iter;
1872         struct seq_file *m;
1873         int cpu;
1874
1875         if (tracing_disabled) {
1876                 *ret = -ENODEV;
1877                 return NULL;
1878         }
1879
1880         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1881         if (!iter) {
1882                 *ret = -ENOMEM;
1883                 goto out;
1884         }
1885
1886         mutex_lock(&trace_types_lock);
1887         if (current_trace && current_trace->print_max)
1888                 iter->tr = &max_tr;
1889         else
1890                 iter->tr = inode->i_private;
1891         iter->trace = current_trace;
1892         iter->pos = -1;
1893
1894         for_each_tracing_cpu(cpu) {
1895
1896                 iter->buffer_iter[cpu] =
1897                         ring_buffer_read_start(iter->tr->buffer, cpu);
1898
1899                 if (!iter->buffer_iter[cpu])
1900                         goto fail_buffer;
1901         }
1902
1903         /* TODO stop tracer */
1904         *ret = seq_open(file, &tracer_seq_ops);
1905         if (*ret)
1906                 goto fail_buffer;
1907
1908         m = file->private_data;
1909         m->private = iter;
1910
1911         /* stop the trace while dumping */
1912         if (iter->tr->ctrl) {
1913                 tracer_enabled = 0;
1914                 ftrace_function_enabled = 0;
1915         }
1916
1917         if (iter->trace && iter->trace->open)
1918                         iter->trace->open(iter);
1919
1920         mutex_unlock(&trace_types_lock);
1921
1922  out:
1923         return iter;
1924
1925  fail_buffer:
1926         for_each_tracing_cpu(cpu) {
1927                 if (iter->buffer_iter[cpu])
1928                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1929         }
1930         mutex_unlock(&trace_types_lock);
1931
1932         return ERR_PTR(-ENOMEM);
1933 }
1934
1935 int tracing_open_generic(struct inode *inode, struct file *filp)
1936 {
1937         if (tracing_disabled)
1938                 return -ENODEV;
1939
1940         filp->private_data = inode->i_private;
1941         return 0;
1942 }
1943
1944 int tracing_release(struct inode *inode, struct file *file)
1945 {
1946         struct seq_file *m = (struct seq_file *)file->private_data;
1947         struct trace_iterator *iter = m->private;
1948         int cpu;
1949
1950         mutex_lock(&trace_types_lock);
1951         for_each_tracing_cpu(cpu) {
1952                 if (iter->buffer_iter[cpu])
1953                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1954         }
1955
1956         if (iter->trace && iter->trace->close)
1957                 iter->trace->close(iter);
1958
1959         /* reenable tracing if it was previously enabled */
1960         if (iter->tr->ctrl) {
1961                 tracer_enabled = 1;
1962                 /*
1963                  * It is safe to enable function tracing even if it
1964                  * isn't used
1965                  */
1966                 ftrace_function_enabled = 1;
1967         }
1968         mutex_unlock(&trace_types_lock);
1969
1970         seq_release(inode, file);
1971         kfree(iter);
1972         return 0;
1973 }
1974
1975 static int tracing_open(struct inode *inode, struct file *file)
1976 {
1977         int ret;
1978
1979         __tracing_open(inode, file, &ret);
1980
1981         return ret;
1982 }
1983
1984 static int tracing_lt_open(struct inode *inode, struct file *file)
1985 {
1986         struct trace_iterator *iter;
1987         int ret;
1988
1989         iter = __tracing_open(inode, file, &ret);
1990
1991         if (!ret)
1992                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1993
1994         return ret;
1995 }
1996
1997
1998 static void *
1999 t_next(struct seq_file *m, void *v, loff_t *pos)
2000 {
2001         struct tracer *t = m->private;
2002
2003         (*pos)++;
2004
2005         if (t)
2006                 t = t->next;
2007
2008         m->private = t;
2009
2010         return t;
2011 }
2012
2013 static void *t_start(struct seq_file *m, loff_t *pos)
2014 {
2015         struct tracer *t = m->private;
2016         loff_t l = 0;
2017
2018         mutex_lock(&trace_types_lock);
2019         for (; t && l < *pos; t = t_next(m, t, &l))
2020                 ;
2021
2022         return t;
2023 }
2024
2025 static void t_stop(struct seq_file *m, void *p)
2026 {
2027         mutex_unlock(&trace_types_lock);
2028 }
2029
2030 static int t_show(struct seq_file *m, void *v)
2031 {
2032         struct tracer *t = v;
2033
2034         if (!t)
2035                 return 0;
2036
2037         seq_printf(m, "%s", t->name);
2038         if (t->next)
2039                 seq_putc(m, ' ');
2040         else
2041                 seq_putc(m, '\n');
2042
2043         return 0;
2044 }
2045
2046 static struct seq_operations show_traces_seq_ops = {
2047         .start          = t_start,
2048         .next           = t_next,
2049         .stop           = t_stop,
2050         .show           = t_show,
2051 };
2052
2053 static int show_traces_open(struct inode *inode, struct file *file)
2054 {
2055         int ret;
2056
2057         if (tracing_disabled)
2058                 return -ENODEV;
2059
2060         ret = seq_open(file, &show_traces_seq_ops);
2061         if (!ret) {
2062                 struct seq_file *m = file->private_data;
2063                 m->private = trace_types;
2064         }
2065
2066         return ret;
2067 }
2068
2069 static struct file_operations tracing_fops = {
2070         .open           = tracing_open,
2071         .read           = seq_read,
2072         .llseek         = seq_lseek,
2073         .release        = tracing_release,
2074 };
2075
2076 static struct file_operations tracing_lt_fops = {
2077         .open           = tracing_lt_open,
2078         .read           = seq_read,
2079         .llseek         = seq_lseek,
2080         .release        = tracing_release,
2081 };
2082
2083 static struct file_operations show_traces_fops = {
2084         .open           = show_traces_open,
2085         .read           = seq_read,
2086         .release        = seq_release,
2087 };
2088
2089 /*
2090  * Only trace on a CPU if the bitmask is set:
2091  */
2092 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2093
2094 /*
2095  * When tracing/tracing_cpu_mask is modified then this holds
2096  * the new bitmask we are about to install:
2097  */
2098 static cpumask_t tracing_cpumask_new;
2099
2100 /*
2101  * The tracer itself will not take this lock, but still we want
2102  * to provide a consistent cpumask to user-space:
2103  */
2104 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2105
2106 /*
2107  * Temporary storage for the character representation of the
2108  * CPU bitmask (and one more byte for the newline):
2109  */
2110 static char mask_str[NR_CPUS + 1];
2111
2112 static ssize_t
2113 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2114                      size_t count, loff_t *ppos)
2115 {
2116         int len;
2117
2118         mutex_lock(&tracing_cpumask_update_lock);
2119
2120         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2121         if (count - len < 2) {
2122                 count = -EINVAL;
2123                 goto out_err;
2124         }
2125         len += sprintf(mask_str + len, "\n");
2126         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2127
2128 out_err:
2129         mutex_unlock(&tracing_cpumask_update_lock);
2130
2131         return count;
2132 }
2133
2134 static ssize_t
2135 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2136                       size_t count, loff_t *ppos)
2137 {
2138         int err, cpu;
2139
2140         mutex_lock(&tracing_cpumask_update_lock);
2141         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2142         if (err)
2143                 goto err_unlock;
2144
2145         raw_local_irq_disable();
2146         __raw_spin_lock(&ftrace_max_lock);
2147         for_each_tracing_cpu(cpu) {
2148                 /*
2149                  * Increase/decrease the disabled counter if we are
2150                  * about to flip a bit in the cpumask:
2151                  */
2152                 if (cpu_isset(cpu, tracing_cpumask) &&
2153                                 !cpu_isset(cpu, tracing_cpumask_new)) {
2154                         atomic_inc(&global_trace.data[cpu]->disabled);
2155                 }
2156                 if (!cpu_isset(cpu, tracing_cpumask) &&
2157                                 cpu_isset(cpu, tracing_cpumask_new)) {
2158                         atomic_dec(&global_trace.data[cpu]->disabled);
2159                 }
2160         }
2161         __raw_spin_unlock(&ftrace_max_lock);
2162         raw_local_irq_enable();
2163
2164         tracing_cpumask = tracing_cpumask_new;
2165
2166         mutex_unlock(&tracing_cpumask_update_lock);
2167
2168         return count;
2169
2170 err_unlock:
2171         mutex_unlock(&tracing_cpumask_update_lock);
2172
2173         return err;
2174 }
2175
2176 static struct file_operations tracing_cpumask_fops = {
2177         .open           = tracing_open_generic,
2178         .read           = tracing_cpumask_read,
2179         .write          = tracing_cpumask_write,
2180 };
2181
2182 static ssize_t
2183 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2184                        size_t cnt, loff_t *ppos)
2185 {
2186         char *buf;
2187         int r = 0;
2188         int len = 0;
2189         int i;
2190
2191         /* calulate max size */
2192         for (i = 0; trace_options[i]; i++) {
2193                 len += strlen(trace_options[i]);
2194                 len += 3; /* "no" and space */
2195         }
2196
2197         /* +2 for \n and \0 */
2198         buf = kmalloc(len + 2, GFP_KERNEL);
2199         if (!buf)
2200                 return -ENOMEM;
2201
2202         for (i = 0; trace_options[i]; i++) {
2203                 if (trace_flags & (1 << i))
2204                         r += sprintf(buf + r, "%s ", trace_options[i]);
2205                 else
2206                         r += sprintf(buf + r, "no%s ", trace_options[i]);
2207         }
2208
2209         r += sprintf(buf + r, "\n");
2210         WARN_ON(r >= len + 2);
2211
2212         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2213
2214         kfree(buf);
2215
2216         return r;
2217 }
2218
2219 static ssize_t
2220 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2221                         size_t cnt, loff_t *ppos)
2222 {
2223         char buf[64];
2224         char *cmp = buf;
2225         int neg = 0;
2226         int i;
2227
2228         if (cnt >= sizeof(buf))
2229                 return -EINVAL;
2230
2231         if (copy_from_user(&buf, ubuf, cnt))
2232                 return -EFAULT;
2233
2234         buf[cnt] = 0;
2235
2236         if (strncmp(buf, "no", 2) == 0) {
2237                 neg = 1;
2238                 cmp += 2;
2239         }
2240
2241         for (i = 0; trace_options[i]; i++) {
2242                 int len = strlen(trace_options[i]);
2243
2244                 if (strncmp(cmp, trace_options[i], len) == 0) {
2245                         if (neg)
2246                                 trace_flags &= ~(1 << i);
2247                         else
2248                                 trace_flags |= (1 << i);
2249                         break;
2250                 }
2251         }
2252         /*
2253          * If no option could be set, return an error:
2254          */
2255         if (!trace_options[i])
2256                 return -EINVAL;
2257
2258         filp->f_pos += cnt;
2259
2260         return cnt;
2261 }
2262
2263 static struct file_operations tracing_iter_fops = {
2264         .open           = tracing_open_generic,
2265         .read           = tracing_iter_ctrl_read,
2266         .write          = tracing_iter_ctrl_write,
2267 };
2268
2269 static const char readme_msg[] =
2270         "tracing mini-HOWTO:\n\n"
2271         "# mkdir /debug\n"
2272         "# mount -t debugfs nodev /debug\n\n"
2273         "# cat /debug/tracing/available_tracers\n"
2274         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2275         "# cat /debug/tracing/current_tracer\n"
2276         "none\n"
2277         "# echo sched_switch > /debug/tracing/current_tracer\n"
2278         "# cat /debug/tracing/current_tracer\n"
2279         "sched_switch\n"
2280         "# cat /debug/tracing/iter_ctrl\n"
2281         "noprint-parent nosym-offset nosym-addr noverbose\n"
2282         "# echo print-parent > /debug/tracing/iter_ctrl\n"
2283         "# echo 1 > /debug/tracing/tracing_enabled\n"
2284         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2285         "echo 0 > /debug/tracing/tracing_enabled\n"
2286 ;
2287
2288 static ssize_t
2289 tracing_readme_read(struct file *filp, char __user *ubuf,
2290                        size_t cnt, loff_t *ppos)
2291 {
2292         return simple_read_from_buffer(ubuf, cnt, ppos,
2293                                         readme_msg, strlen(readme_msg));
2294 }
2295
2296 static struct file_operations tracing_readme_fops = {
2297         .open           = tracing_open_generic,
2298         .read           = tracing_readme_read,
2299 };
2300
2301 static ssize_t
2302 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2303                   size_t cnt, loff_t *ppos)
2304 {
2305         struct trace_array *tr = filp->private_data;
2306         char buf[64];
2307         int r;
2308
2309         r = sprintf(buf, "%ld\n", tr->ctrl);
2310         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2311 }
2312
2313 static ssize_t
2314 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2315                    size_t cnt, loff_t *ppos)
2316 {
2317         struct trace_array *tr = filp->private_data;
2318         char buf[64];
2319         long val;
2320         int ret;
2321
2322         if (cnt >= sizeof(buf))
2323                 return -EINVAL;
2324
2325         if (copy_from_user(&buf, ubuf, cnt))
2326                 return -EFAULT;
2327
2328         buf[cnt] = 0;
2329
2330         ret = strict_strtoul(buf, 10, &val);
2331         if (ret < 0)
2332                 return ret;
2333
2334         val = !!val;
2335
2336         mutex_lock(&trace_types_lock);
2337         if (tr->ctrl ^ val) {
2338                 if (val)
2339                         tracer_enabled = 1;
2340                 else
2341                         tracer_enabled = 0;
2342
2343                 tr->ctrl = val;
2344
2345                 if (current_trace && current_trace->ctrl_update)
2346                         current_trace->ctrl_update(tr);
2347         }
2348         mutex_unlock(&trace_types_lock);
2349
2350         filp->f_pos += cnt;
2351
2352         return cnt;
2353 }
2354
2355 static ssize_t
2356 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2357                        size_t cnt, loff_t *ppos)
2358 {
2359         char buf[max_tracer_type_len+2];
2360         int r;
2361
2362         mutex_lock(&trace_types_lock);
2363         if (current_trace)
2364                 r = sprintf(buf, "%s\n", current_trace->name);
2365         else
2366                 r = sprintf(buf, "\n");
2367         mutex_unlock(&trace_types_lock);
2368
2369         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2370 }
2371
2372 static ssize_t
2373 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2374                         size_t cnt, loff_t *ppos)
2375 {
2376         struct trace_array *tr = &global_trace;
2377         struct tracer *t;
2378         char buf[max_tracer_type_len+1];
2379         int i;
2380         size_t ret;
2381
2382         if (cnt > max_tracer_type_len)
2383                 cnt = max_tracer_type_len;
2384         ret = cnt;
2385
2386         if (copy_from_user(&buf, ubuf, cnt))
2387                 return -EFAULT;
2388
2389         buf[cnt] = 0;
2390
2391         /* strip ending whitespace. */
2392         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2393                 buf[i] = 0;
2394
2395         mutex_lock(&trace_types_lock);
2396         for (t = trace_types; t; t = t->next) {
2397                 if (strcmp(t->name, buf) == 0)
2398                         break;
2399         }
2400         if (!t) {
2401                 ret = -EINVAL;
2402                 goto out;
2403         }
2404         if (t == current_trace)
2405                 goto out;
2406
2407         if (current_trace && current_trace->reset)
2408                 current_trace->reset(tr);
2409
2410         current_trace = t;
2411         if (t->init)
2412                 t->init(tr);
2413
2414  out:
2415         mutex_unlock(&trace_types_lock);
2416
2417         if (ret == cnt)
2418                 filp->f_pos += cnt;
2419
2420         return ret;
2421 }
2422
2423 static ssize_t
2424 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2425                      size_t cnt, loff_t *ppos)
2426 {
2427         unsigned long *ptr = filp->private_data;
2428         char buf[64];
2429         int r;
2430
2431         r = snprintf(buf, sizeof(buf), "%ld\n",
2432                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2433         if (r > sizeof(buf))
2434                 r = sizeof(buf);
2435         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2436 }
2437
2438 static ssize_t
2439 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2440                       size_t cnt, loff_t *ppos)
2441 {
2442         long *ptr = filp->private_data;
2443         char buf[64];
2444         long val;
2445         int ret;
2446
2447         if (cnt >= sizeof(buf))
2448                 return -EINVAL;
2449
2450         if (copy_from_user(&buf, ubuf, cnt))
2451                 return -EFAULT;
2452
2453         buf[cnt] = 0;
2454
2455         ret = strict_strtoul(buf, 10, &val);
2456         if (ret < 0)
2457                 return ret;
2458
2459         *ptr = val * 1000;
2460
2461         return cnt;
2462 }
2463
2464 static atomic_t tracing_reader;
2465
2466 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2467 {
2468         struct trace_iterator *iter;
2469
2470         if (tracing_disabled)
2471                 return -ENODEV;
2472
2473         /* We only allow for reader of the pipe */
2474         if (atomic_inc_return(&tracing_reader) != 1) {
2475                 atomic_dec(&tracing_reader);
2476                 return -EBUSY;
2477         }
2478
2479         /* create a buffer to store the information to pass to userspace */
2480         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2481         if (!iter)
2482                 return -ENOMEM;
2483
2484         mutex_lock(&trace_types_lock);
2485         iter->tr = &global_trace;
2486         iter->trace = current_trace;
2487         filp->private_data = iter;
2488
2489         if (iter->trace->pipe_open)
2490                 iter->trace->pipe_open(iter);
2491         mutex_unlock(&trace_types_lock);
2492
2493         return 0;
2494 }
2495
2496 static int tracing_release_pipe(struct inode *inode, struct file *file)
2497 {
2498         struct trace_iterator *iter = file->private_data;
2499
2500         kfree(iter);
2501         atomic_dec(&tracing_reader);
2502
2503         return 0;
2504 }
2505
2506 static unsigned int
2507 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2508 {
2509         struct trace_iterator *iter = filp->private_data;
2510
2511         if (trace_flags & TRACE_ITER_BLOCK) {
2512                 /*
2513                  * Always select as readable when in blocking mode
2514                  */
2515                 return POLLIN | POLLRDNORM;
2516         } else {
2517                 if (!trace_empty(iter))
2518                         return POLLIN | POLLRDNORM;
2519                 poll_wait(filp, &trace_wait, poll_table);
2520                 if (!trace_empty(iter))
2521                         return POLLIN | POLLRDNORM;
2522
2523                 return 0;
2524         }
2525 }
2526
2527 /*
2528  * Consumer reader.
2529  */
2530 static ssize_t
2531 tracing_read_pipe(struct file *filp, char __user *ubuf,
2532                   size_t cnt, loff_t *ppos)
2533 {
2534         struct trace_iterator *iter = filp->private_data;
2535         ssize_t sret;
2536
2537         /* return any leftover data */
2538         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2539         if (sret != -EBUSY)
2540                 return sret;
2541
2542         trace_seq_reset(&iter->seq);
2543
2544         mutex_lock(&trace_types_lock);
2545         if (iter->trace->read) {
2546                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2547                 if (sret)
2548                         goto out;
2549         }
2550
2551 waitagain:
2552         sret = 0;
2553         while (trace_empty(iter)) {
2554
2555                 if ((filp->f_flags & O_NONBLOCK)) {
2556                         sret = -EAGAIN;
2557                         goto out;
2558                 }
2559
2560                 /*
2561                  * This is a make-shift waitqueue. The reason we don't use
2562                  * an actual wait queue is because:
2563                  *  1) we only ever have one waiter
2564                  *  2) the tracing, traces all functions, we don't want
2565                  *     the overhead of calling wake_up and friends
2566                  *     (and tracing them too)
2567                  *     Anyway, this is really very primitive wakeup.
2568                  */
2569                 set_current_state(TASK_INTERRUPTIBLE);
2570                 iter->tr->waiter = current;
2571
2572                 mutex_unlock(&trace_types_lock);
2573
2574                 /* sleep for 100 msecs, and try again. */
2575                 schedule_timeout(HZ/10);
2576
2577                 mutex_lock(&trace_types_lock);
2578
2579                 iter->tr->waiter = NULL;
2580
2581                 if (signal_pending(current)) {
2582                         sret = -EINTR;
2583                         goto out;
2584                 }
2585
2586                 if (iter->trace != current_trace)
2587                         goto out;
2588
2589                 /*
2590                  * We block until we read something and tracing is disabled.
2591                  * We still block if tracing is disabled, but we have never
2592                  * read anything. This allows a user to cat this file, and
2593                  * then enable tracing. But after we have read something,
2594                  * we give an EOF when tracing is again disabled.
2595                  *
2596                  * iter->pos will be 0 if we haven't read anything.
2597                  */
2598                 if (!tracer_enabled && iter->pos)
2599                         break;
2600
2601                 continue;
2602         }
2603
2604         /* stop when tracing is finished */
2605         if (trace_empty(iter))
2606                 goto out;
2607
2608         if (cnt >= PAGE_SIZE)
2609                 cnt = PAGE_SIZE - 1;
2610
2611         /* reset all but tr, trace, and overruns */
2612         memset(&iter->seq, 0,
2613                sizeof(struct trace_iterator) -
2614                offsetof(struct trace_iterator, seq));
2615         iter->pos = -1;
2616
2617         while (find_next_entry_inc(iter) != NULL) {
2618                 enum print_line_t ret;
2619                 int len = iter->seq.len;
2620
2621                 ret = print_trace_line(iter);
2622                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2623                         /* don't print partial lines */
2624                         iter->seq.len = len;
2625                         break;
2626                 }
2627
2628                 trace_consume(iter);
2629
2630                 if (iter->seq.len >= cnt)
2631                         break;
2632         }
2633
2634         /* Now copy what we have to the user */
2635         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2636         if (iter->seq.readpos >= iter->seq.len)
2637                 trace_seq_reset(&iter->seq);
2638
2639         /*
2640          * If there was nothing to send to user, inspite of consuming trace
2641          * entries, go back to wait for more entries.
2642          */
2643         if (sret == -EBUSY)
2644                 goto waitagain;
2645
2646 out:
2647         mutex_unlock(&trace_types_lock);
2648
2649         return sret;
2650 }
2651
2652 static ssize_t
2653 tracing_entries_read(struct file *filp, char __user *ubuf,
2654                      size_t cnt, loff_t *ppos)
2655 {
2656         struct trace_array *tr = filp->private_data;
2657         char buf[64];
2658         int r;
2659
2660         r = sprintf(buf, "%lu\n", tr->entries);
2661         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2662 }
2663
2664 static ssize_t
2665 tracing_entries_write(struct file *filp, const char __user *ubuf,
2666                       size_t cnt, loff_t *ppos)
2667 {
2668         unsigned long val;
2669         char buf[64];
2670         int ret;
2671         struct trace_array *tr = filp->private_data;
2672
2673         if (cnt >= sizeof(buf))
2674                 return -EINVAL;
2675
2676         if (copy_from_user(&buf, ubuf, cnt))
2677                 return -EFAULT;
2678
2679         buf[cnt] = 0;
2680
2681         ret = strict_strtoul(buf, 10, &val);
2682         if (ret < 0)
2683                 return ret;
2684
2685         /* must have at least 1 entry */
2686         if (!val)
2687                 return -EINVAL;
2688
2689         mutex_lock(&trace_types_lock);
2690
2691         if (tr->ctrl) {
2692                 cnt = -EBUSY;
2693                 pr_info("ftrace: please disable tracing"
2694                         " before modifying buffer size\n");
2695                 goto out;
2696         }
2697
2698         if (val != global_trace.entries) {
2699                 ret = ring_buffer_resize(global_trace.buffer, val);
2700                 if (ret < 0) {
2701                         cnt = ret;
2702                         goto out;
2703                 }
2704
2705                 ret = ring_buffer_resize(max_tr.buffer, val);
2706                 if (ret < 0) {
2707                         int r;
2708                         cnt = ret;
2709                         r = ring_buffer_resize(global_trace.buffer,
2710                                                global_trace.entries);
2711                         if (r < 0) {
2712                                 /* AARGH! We are left with different
2713                                  * size max buffer!!!! */
2714                                 WARN_ON(1);
2715                                 tracing_disabled = 1;
2716                         }
2717                         goto out;
2718                 }
2719
2720                 global_trace.entries = val;
2721         }
2722
2723         filp->f_pos += cnt;
2724
2725         /* If check pages failed, return ENOMEM */
2726         if (tracing_disabled)
2727                 cnt = -ENOMEM;
2728  out:
2729         max_tr.entries = global_trace.entries;
2730         mutex_unlock(&trace_types_lock);
2731
2732         return cnt;
2733 }
2734
2735 static int mark_printk(const char *fmt, ...)
2736 {
2737         int ret;
2738         va_list args;
2739         va_start(args, fmt);
2740         ret = trace_vprintk(0, fmt, args);
2741         va_end(args);
2742         return ret;
2743 }
2744
2745 static ssize_t
2746 tracing_mark_write(struct file *filp, const char __user *ubuf,
2747                                         size_t cnt, loff_t *fpos)
2748 {
2749         char *buf;
2750         char *end;
2751         struct trace_array *tr = &global_trace;
2752
2753         if (!tr->ctrl || tracing_disabled)
2754                 return -EINVAL;
2755
2756         if (cnt > TRACE_BUF_SIZE)
2757                 cnt = TRACE_BUF_SIZE;
2758
2759         buf = kmalloc(cnt + 1, GFP_KERNEL);
2760         if (buf == NULL)
2761                 return -ENOMEM;
2762
2763         if (copy_from_user(buf, ubuf, cnt)) {
2764                 kfree(buf);
2765                 return -EFAULT;
2766         }
2767
2768         /* Cut from the first nil or newline. */
2769         buf[cnt] = '\0';
2770         end = strchr(buf, '\n');
2771         if (end)
2772                 *end = '\0';
2773
2774         cnt = mark_printk("%s\n", buf);
2775         kfree(buf);
2776         *fpos += cnt;
2777
2778         return cnt;
2779 }
2780
2781 static struct file_operations tracing_max_lat_fops = {
2782         .open           = tracing_open_generic,
2783         .read           = tracing_max_lat_read,
2784         .write          = tracing_max_lat_write,
2785 };
2786
2787 static struct file_operations tracing_ctrl_fops = {
2788         .open           = tracing_open_generic,
2789         .read           = tracing_ctrl_read,
2790         .write          = tracing_ctrl_write,
2791 };
2792
2793 static struct file_operations set_tracer_fops = {
2794         .open           = tracing_open_generic,
2795         .read           = tracing_set_trace_read,
2796         .write          = tracing_set_trace_write,
2797 };
2798
2799 static struct file_operations tracing_pipe_fops = {
2800         .open           = tracing_open_pipe,
2801         .poll           = tracing_poll_pipe,
2802         .read           = tracing_read_pipe,
2803         .release        = tracing_release_pipe,
2804 };
2805
2806 static struct file_operations tracing_entries_fops = {
2807         .open           = tracing_open_generic,
2808         .read           = tracing_entries_read,
2809         .write          = tracing_entries_write,
2810 };
2811
2812 static struct file_operations tracing_mark_fops = {
2813         .open           = tracing_open_generic,
2814         .write          = tracing_mark_write,
2815 };
2816
2817 #ifdef CONFIG_DYNAMIC_FTRACE
2818
2819 static ssize_t
2820 tracing_read_long(struct file *filp, char __user *ubuf,
2821                   size_t cnt, loff_t *ppos)
2822 {
2823         unsigned long *p = filp->private_data;
2824         char buf[64];
2825         int r;
2826
2827         r = sprintf(buf, "%ld\n", *p);
2828
2829         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2830 }
2831
2832 static struct file_operations tracing_read_long_fops = {
2833         .open           = tracing_open_generic,
2834         .read           = tracing_read_long,
2835 };
2836 #endif
2837
2838 static struct dentry *d_tracer;
2839
2840 struct dentry *tracing_init_dentry(void)
2841 {
2842         static int once;
2843
2844         if (d_tracer)
2845                 return d_tracer;
2846
2847         d_tracer = debugfs_create_dir("tracing", NULL);
2848
2849         if (!d_tracer && !once) {
2850                 once = 1;
2851                 pr_warning("Could not create debugfs directory 'tracing'\n");
2852                 return NULL;
2853         }
2854
2855         return d_tracer;
2856 }
2857
2858 #ifdef CONFIG_FTRACE_SELFTEST
2859 /* Let selftest have access to static functions in this file */
2860 #include "trace_selftest.c"
2861 #endif
2862
2863 static __init int tracer_init_debugfs(void)
2864 {
2865         struct dentry *d_tracer;
2866         struct dentry *entry;
2867
2868         d_tracer = tracing_init_dentry();
2869
2870         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2871                                     &global_trace, &tracing_ctrl_fops);
2872         if (!entry)
2873                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2874
2875         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2876                                     NULL, &tracing_iter_fops);
2877         if (!entry)
2878                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2879
2880         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2881                                     NULL, &tracing_cpumask_fops);
2882         if (!entry)
2883                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2884
2885         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2886                                     &global_trace, &tracing_lt_fops);
2887         if (!entry)
2888                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2889
2890         entry = debugfs_create_file("trace", 0444, d_tracer,
2891                                     &global_trace, &tracing_fops);
2892         if (!entry)
2893                 pr_warning("Could not create debugfs 'trace' entry\n");
2894
2895         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2896                                     &global_trace, &show_traces_fops);
2897         if (!entry)
2898                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
2899
2900         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2901                                     &global_trace, &set_tracer_fops);
2902         if (!entry)
2903                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
2904
2905         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2906                                     &tracing_max_latency,
2907                                     &tracing_max_lat_fops);
2908         if (!entry)
2909                 pr_warning("Could not create debugfs "
2910                            "'tracing_max_latency' entry\n");
2911
2912         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2913                                     &tracing_thresh, &tracing_max_lat_fops);
2914         if (!entry)
2915                 pr_warning("Could not create debugfs "
2916                            "'tracing_thresh' entry\n");
2917         entry = debugfs_create_file("README", 0644, d_tracer,
2918                                     NULL, &tracing_readme_fops);
2919         if (!entry)
2920                 pr_warning("Could not create debugfs 'README' entry\n");
2921
2922         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2923                                     NULL, &tracing_pipe_fops);
2924         if (!entry)
2925                 pr_warning("Could not create debugfs "
2926                            "'trace_pipe' entry\n");
2927
2928         entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2929                                     &global_trace, &tracing_entries_fops);
2930         if (!entry)
2931                 pr_warning("Could not create debugfs "
2932                            "'trace_entries' entry\n");
2933
2934         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
2935                                     NULL, &tracing_mark_fops);
2936         if (!entry)
2937                 pr_warning("Could not create debugfs "
2938                            "'trace_marker' entry\n");
2939
2940 #ifdef CONFIG_DYNAMIC_FTRACE
2941         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2942                                     &ftrace_update_tot_cnt,
2943                                     &tracing_read_long_fops);
2944         if (!entry)
2945                 pr_warning("Could not create debugfs "
2946                            "'dyn_ftrace_total_info' entry\n");
2947 #endif
2948 #ifdef CONFIG_SYSPROF_TRACER
2949         init_tracer_sysprof_debugfs(d_tracer);
2950 #endif
2951         return 0;
2952 }
2953
2954 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2955 {
2956         static DEFINE_SPINLOCK(trace_buf_lock);
2957         static char trace_buf[TRACE_BUF_SIZE];
2958
2959         struct ring_buffer_event *event;
2960         struct trace_array *tr = &global_trace;
2961         struct trace_array_cpu *data;
2962         struct print_entry *entry;
2963         unsigned long flags, irq_flags;
2964         int cpu, len = 0, size, pc;
2965
2966         if (!tr->ctrl || tracing_disabled)
2967                 return 0;
2968
2969         pc = preempt_count();
2970         preempt_disable_notrace();
2971         cpu = raw_smp_processor_id();
2972         data = tr->data[cpu];
2973
2974         if (unlikely(atomic_read(&data->disabled)))
2975                 goto out;
2976
2977         spin_lock_irqsave(&trace_buf_lock, flags);
2978         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
2979
2980         len = min(len, TRACE_BUF_SIZE-1);
2981         trace_buf[len] = 0;
2982
2983         size = sizeof(*entry) + len + 1;
2984         event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
2985         if (!event)
2986                 goto out_unlock;
2987         entry = ring_buffer_event_data(event);
2988         tracing_generic_entry_update(&entry->ent, flags, pc);
2989         entry->ent.type                 = TRACE_PRINT;
2990         entry->ip                       = ip;
2991
2992         memcpy(&entry->buf, trace_buf, len);
2993         entry->buf[len] = 0;
2994         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
2995
2996  out_unlock:
2997         spin_unlock_irqrestore(&trace_buf_lock, flags);
2998
2999  out:
3000         preempt_enable_notrace();
3001
3002         return len;
3003 }
3004 EXPORT_SYMBOL_GPL(trace_vprintk);
3005
3006 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3007 {
3008         int ret;
3009         va_list ap;
3010
3011         if (!(trace_flags & TRACE_ITER_PRINTK))
3012                 return 0;
3013
3014         va_start(ap, fmt);
3015         ret = trace_vprintk(ip, fmt, ap);
3016         va_end(ap);
3017         return ret;
3018 }
3019 EXPORT_SYMBOL_GPL(__ftrace_printk);
3020
3021 static int trace_panic_handler(struct notifier_block *this,
3022                                unsigned long event, void *unused)
3023 {
3024         ftrace_dump();
3025         return NOTIFY_OK;
3026 }
3027
3028 static struct notifier_block trace_panic_notifier = {
3029         .notifier_call  = trace_panic_handler,
3030         .next           = NULL,
3031         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
3032 };
3033
3034 static int trace_die_handler(struct notifier_block *self,
3035                              unsigned long val,
3036                              void *data)
3037 {
3038         switch (val) {
3039         case DIE_OOPS:
3040                 ftrace_dump();
3041                 break;
3042         default:
3043                 break;
3044         }
3045         return NOTIFY_OK;
3046 }
3047
3048 static struct notifier_block trace_die_notifier = {
3049         .notifier_call = trace_die_handler,
3050         .priority = 200
3051 };
3052
3053 /*
3054  * printk is set to max of 1024, we really don't need it that big.
3055  * Nothing should be printing 1000 characters anyway.
3056  */
3057 #define TRACE_MAX_PRINT         1000
3058
3059 /*
3060  * Define here KERN_TRACE so that we have one place to modify
3061  * it if we decide to change what log level the ftrace dump
3062  * should be at.
3063  */
3064 #define KERN_TRACE              KERN_INFO
3065
3066 static void
3067 trace_printk_seq(struct trace_seq *s)
3068 {
3069         /* Probably should print a warning here. */
3070         if (s->len >= 1000)
3071                 s->len = 1000;
3072
3073         /* should be zero ended, but we are paranoid. */
3074         s->buffer[s->len] = 0;
3075
3076         printk(KERN_TRACE "%s", s->buffer);
3077
3078         trace_seq_reset(s);
3079 }
3080
3081
3082 void ftrace_dump(void)
3083 {
3084         static DEFINE_SPINLOCK(ftrace_dump_lock);
3085         /* use static because iter can be a bit big for the stack */
3086         static struct trace_iterator iter;
3087         static cpumask_t mask;
3088         static int dump_ran;
3089         unsigned long flags;
3090         int cnt = 0, cpu;
3091
3092         /* only one dump */
3093         spin_lock_irqsave(&ftrace_dump_lock, flags);
3094         if (dump_ran)
3095                 goto out;
3096
3097         dump_ran = 1;
3098
3099         /* No turning back! */
3100         ftrace_kill_atomic();
3101
3102         for_each_tracing_cpu(cpu) {
3103                 atomic_inc(&global_trace.data[cpu]->disabled);
3104         }
3105
3106         printk(KERN_TRACE "Dumping ftrace buffer:\n");
3107
3108         iter.tr = &global_trace;
3109         iter.trace = current_trace;
3110
3111         /*
3112          * We need to stop all tracing on all CPUS to read the
3113          * the next buffer. This is a bit expensive, but is
3114          * not done often. We fill all what we can read,
3115          * and then release the locks again.
3116          */
3117
3118         cpus_clear(mask);
3119
3120         while (!trace_empty(&iter)) {
3121
3122                 if (!cnt)
3123                         printk(KERN_TRACE "---------------------------------\n");
3124
3125                 cnt++;
3126
3127                 /* reset all but tr, trace, and overruns */
3128                 memset(&iter.seq, 0,
3129                        sizeof(struct trace_iterator) -
3130                        offsetof(struct trace_iterator, seq));
3131                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3132                 iter.pos = -1;
3133
3134                 if (find_next_entry_inc(&iter) != NULL) {
3135                         print_trace_line(&iter);
3136                         trace_consume(&iter);
3137                 }
3138
3139                 trace_printk_seq(&iter.seq);
3140         }
3141
3142         if (!cnt)
3143                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3144         else
3145                 printk(KERN_TRACE "---------------------------------\n");
3146
3147  out:
3148         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3149 }
3150
3151 __init static int tracer_alloc_buffers(void)
3152 {
3153         struct trace_array_cpu *data;
3154         int i;
3155
3156         /* TODO: make the number of buffers hot pluggable with CPUS */
3157         tracing_buffer_mask = cpu_possible_map;
3158
3159         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3160                                                    TRACE_BUFFER_FLAGS);
3161         if (!global_trace.buffer) {
3162                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3163                 WARN_ON(1);
3164                 return 0;
3165         }
3166         global_trace.entries = ring_buffer_size(global_trace.buffer);
3167
3168 #ifdef CONFIG_TRACER_MAX_TRACE
3169         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3170                                              TRACE_BUFFER_FLAGS);
3171         if (!max_tr.buffer) {
3172                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3173                 WARN_ON(1);
3174                 ring_buffer_free(global_trace.buffer);
3175                 return 0;
3176         }
3177         max_tr.entries = ring_buffer_size(max_tr.buffer);
3178         WARN_ON(max_tr.entries != global_trace.entries);
3179 #endif
3180
3181         /* Allocate the first page for all buffers */
3182         for_each_tracing_cpu(i) {
3183                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3184                 max_tr.data[i] = &per_cpu(max_data, i);
3185         }
3186
3187         trace_init_cmdlines();
3188
3189         register_tracer(&nop_trace);
3190 #ifdef CONFIG_BOOT_TRACER
3191         register_tracer(&boot_tracer);
3192         current_trace = &boot_tracer;
3193         current_trace->init(&global_trace);
3194 #else
3195         current_trace = &nop_trace;
3196 #endif
3197
3198         /* All seems OK, enable tracing */
3199         global_trace.ctrl = tracer_enabled;
3200         tracing_disabled = 0;
3201
3202         atomic_notifier_chain_register(&panic_notifier_list,
3203                                        &trace_panic_notifier);
3204
3205         register_die_notifier(&trace_die_notifier);
3206
3207         return 0;
3208 }
3209 early_initcall(tracer_alloc_buffers);
3210 fs_initcall(tracer_init_debugfs);