Merge branch 'tip/tracing/ftrace' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6] / kernel / trace / ring_buffer.c
1 /*
2  * Generic ring buffer
3  *
4  * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
5  */
6 #include <linux/ring_buffer.h>
7 #include <linux/trace_clock.h>
8 #include <linux/ftrace_irq.h>
9 #include <linux/spinlock.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <linux/mutex.h>
16 #include <linux/init.h>
17 #include <linux/hash.h>
18 #include <linux/list.h>
19 #include <linux/cpu.h>
20 #include <linux/fs.h>
21
22 #include "trace.h"
23
24 /*
25  * The ring buffer is made up of a list of pages. A separate list of pages is
26  * allocated for each CPU. A writer may only write to a buffer that is
27  * associated with the CPU it is currently executing on.  A reader may read
28  * from any per cpu buffer.
29  *
30  * The reader is special. For each per cpu buffer, the reader has its own
31  * reader page. When a reader has read the entire reader page, this reader
32  * page is swapped with another page in the ring buffer.
33  *
34  * Now, as long as the writer is off the reader page, the reader can do what
35  * ever it wants with that page. The writer will never write to that page
36  * again (as long as it is out of the ring buffer).
37  *
38  * Here's some silly ASCII art.
39  *
40  *   +------+
41  *   |reader|          RING BUFFER
42  *   |page  |
43  *   +------+        +---+   +---+   +---+
44  *                   |   |-->|   |-->|   |
45  *                   +---+   +---+   +---+
46  *                     ^               |
47  *                     |               |
48  *                     +---------------+
49  *
50  *
51  *   +------+
52  *   |reader|          RING BUFFER
53  *   |page  |------------------v
54  *   +------+        +---+   +---+   +---+
55  *                   |   |-->|   |-->|   |
56  *                   +---+   +---+   +---+
57  *                     ^               |
58  *                     |               |
59  *                     +---------------+
60  *
61  *
62  *   +------+
63  *   |reader|          RING BUFFER
64  *   |page  |------------------v
65  *   +------+        +---+   +---+   +---+
66  *      ^            |   |-->|   |-->|   |
67  *      |            +---+   +---+   +---+
68  *      |                              |
69  *      |                              |
70  *      +------------------------------+
71  *
72  *
73  *   +------+
74  *   |buffer|          RING BUFFER
75  *   |page  |------------------v
76  *   +------+        +---+   +---+   +---+
77  *      ^            |   |   |   |-->|   |
78  *      |   New      +---+   +---+   +---+
79  *      |  Reader------^               |
80  *      |   page                       |
81  *      +------------------------------+
82  *
83  *
84  * After we make this swap, the reader can hand this page off to the splice
85  * code and be done with it. It can even allocate a new page if it needs to
86  * and swap that into the ring buffer.
87  *
88  * We will be using cmpxchg soon to make all this lockless.
89  *
90  */
91
92 /*
93  * A fast way to enable or disable all ring buffers is to
94  * call tracing_on or tracing_off. Turning off the ring buffers
95  * prevents all ring buffers from being recorded to.
96  * Turning this switch on, makes it OK to write to the
97  * ring buffer, if the ring buffer is enabled itself.
98  *
99  * There's three layers that must be on in order to write
100  * to the ring buffer.
101  *
102  * 1) This global flag must be set.
103  * 2) The ring buffer must be enabled for recording.
104  * 3) The per cpu buffer must be enabled for recording.
105  *
106  * In case of an anomaly, this global flag has a bit set that
107  * will permantly disable all ring buffers.
108  */
109
110 /*
111  * Global flag to disable all recording to ring buffers
112  *  This has two bits: ON, DISABLED
113  *
114  *  ON   DISABLED
115  * ---- ----------
116  *   0      0        : ring buffers are off
117  *   1      0        : ring buffers are on
118  *   X      1        : ring buffers are permanently disabled
119  */
120
121 enum {
122         RB_BUFFERS_ON_BIT       = 0,
123         RB_BUFFERS_DISABLED_BIT = 1,
124 };
125
126 enum {
127         RB_BUFFERS_ON           = 1 << RB_BUFFERS_ON_BIT,
128         RB_BUFFERS_DISABLED     = 1 << RB_BUFFERS_DISABLED_BIT,
129 };
130
131 static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
132
133 #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
134
135 /**
136  * tracing_on - enable all tracing buffers
137  *
138  * This function enables all tracing buffers that may have been
139  * disabled with tracing_off.
140  */
141 void tracing_on(void)
142 {
143         set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
144 }
145 EXPORT_SYMBOL_GPL(tracing_on);
146
147 /**
148  * tracing_off - turn off all tracing buffers
149  *
150  * This function stops all tracing buffers from recording data.
151  * It does not disable any overhead the tracers themselves may
152  * be causing. This function simply causes all recording to
153  * the ring buffers to fail.
154  */
155 void tracing_off(void)
156 {
157         clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
158 }
159 EXPORT_SYMBOL_GPL(tracing_off);
160
161 /**
162  * tracing_off_permanent - permanently disable ring buffers
163  *
164  * This function, once called, will disable all ring buffers
165  * permanently.
166  */
167 void tracing_off_permanent(void)
168 {
169         set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
170 }
171
172 /**
173  * tracing_is_on - show state of ring buffers enabled
174  */
175 int tracing_is_on(void)
176 {
177         return ring_buffer_flags == RB_BUFFERS_ON;
178 }
179 EXPORT_SYMBOL_GPL(tracing_is_on);
180
181 #include "trace.h"
182
183 #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
184 #define RB_ALIGNMENT            4U
185 #define RB_MAX_SMALL_DATA       28
186
187 enum {
188         RB_LEN_TIME_EXTEND = 8,
189         RB_LEN_TIME_STAMP = 16,
190 };
191
192 /* inline for ring buffer fast paths */
193 static unsigned
194 rb_event_length(struct ring_buffer_event *event)
195 {
196         unsigned length;
197
198         switch (event->type) {
199         case RINGBUF_TYPE_PADDING:
200                 /* undefined */
201                 return -1;
202
203         case RINGBUF_TYPE_TIME_EXTEND:
204                 return RB_LEN_TIME_EXTEND;
205
206         case RINGBUF_TYPE_TIME_STAMP:
207                 return RB_LEN_TIME_STAMP;
208
209         case RINGBUF_TYPE_DATA:
210                 if (event->len)
211                         length = event->len * RB_ALIGNMENT;
212                 else
213                         length = event->array[0];
214                 return length + RB_EVNT_HDR_SIZE;
215         default:
216                 BUG();
217         }
218         /* not hit */
219         return 0;
220 }
221
222 /**
223  * ring_buffer_event_length - return the length of the event
224  * @event: the event to get the length of
225  */
226 unsigned ring_buffer_event_length(struct ring_buffer_event *event)
227 {
228         unsigned length = rb_event_length(event);
229         if (event->type != RINGBUF_TYPE_DATA)
230                 return length;
231         length -= RB_EVNT_HDR_SIZE;
232         if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
233                 length -= sizeof(event->array[0]);
234         return length;
235 }
236 EXPORT_SYMBOL_GPL(ring_buffer_event_length);
237
238 /* inline for ring buffer fast paths */
239 static void *
240 rb_event_data(struct ring_buffer_event *event)
241 {
242         BUG_ON(event->type != RINGBUF_TYPE_DATA);
243         /* If length is in len field, then array[0] has the data */
244         if (event->len)
245                 return (void *)&event->array[0];
246         /* Otherwise length is in array[0] and array[1] has the data */
247         return (void *)&event->array[1];
248 }
249
250 /**
251  * ring_buffer_event_data - return the data of the event
252  * @event: the event to get the data from
253  */
254 void *ring_buffer_event_data(struct ring_buffer_event *event)
255 {
256         return rb_event_data(event);
257 }
258 EXPORT_SYMBOL_GPL(ring_buffer_event_data);
259
260 #define for_each_buffer_cpu(buffer, cpu)                \
261         for_each_cpu(cpu, buffer->cpumask)
262
263 #define TS_SHIFT        27
264 #define TS_MASK         ((1ULL << TS_SHIFT) - 1)
265 #define TS_DELTA_TEST   (~TS_MASK)
266
267 struct buffer_data_page {
268         u64              time_stamp;    /* page time stamp */
269         local_t          commit;        /* write committed index */
270         unsigned char    data[];        /* data of buffer page */
271 };
272
273 struct buffer_page {
274         local_t          write;         /* index for next write */
275         unsigned         read;          /* index for next read */
276         struct list_head list;          /* list of free pages */
277         struct buffer_data_page *page;  /* Actual data page */
278 };
279
280 static void rb_init_page(struct buffer_data_page *bpage)
281 {
282         local_set(&bpage->commit, 0);
283 }
284
285 /**
286  * ring_buffer_page_len - the size of data on the page.
287  * @page: The page to read
288  *
289  * Returns the amount of data on the page, including buffer page header.
290  */
291 size_t ring_buffer_page_len(void *page)
292 {
293         return local_read(&((struct buffer_data_page *)page)->commit)
294                 + BUF_PAGE_HDR_SIZE;
295 }
296
297 /*
298  * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
299  * this issue out.
300  */
301 static void free_buffer_page(struct buffer_page *bpage)
302 {
303         free_page((unsigned long)bpage->page);
304         kfree(bpage);
305 }
306
307 /*
308  * We need to fit the time_stamp delta into 27 bits.
309  */
310 static inline int test_time_stamp(u64 delta)
311 {
312         if (delta & TS_DELTA_TEST)
313                 return 1;
314         return 0;
315 }
316
317 #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
318
319 /*
320  * head_page == tail_page && head == tail then buffer is empty.
321  */
322 struct ring_buffer_per_cpu {
323         int                             cpu;
324         struct ring_buffer              *buffer;
325         spinlock_t                      reader_lock; /* serialize readers */
326         raw_spinlock_t                  lock;
327         struct lock_class_key           lock_key;
328         struct list_head                pages;
329         struct buffer_page              *head_page;     /* read from head */
330         struct buffer_page              *tail_page;     /* write to tail */
331         struct buffer_page              *commit_page;   /* committed pages */
332         struct buffer_page              *reader_page;
333         unsigned long                   overrun;
334         unsigned long                   entries;
335         u64                             write_stamp;
336         u64                             read_stamp;
337         atomic_t                        record_disabled;
338 };
339
340 struct ring_buffer {
341         unsigned                        pages;
342         unsigned                        flags;
343         int                             cpus;
344         atomic_t                        record_disabled;
345         cpumask_var_t                   cpumask;
346
347         struct mutex                    mutex;
348
349         struct ring_buffer_per_cpu      **buffers;
350
351 #ifdef CONFIG_HOTPLUG_CPU
352         struct notifier_block           cpu_notify;
353 #endif
354         u64                             (*clock)(void);
355 };
356
357 struct ring_buffer_iter {
358         struct ring_buffer_per_cpu      *cpu_buffer;
359         unsigned long                   head;
360         struct buffer_page              *head_page;
361         u64                             read_stamp;
362 };
363
364 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
365 #define RB_WARN_ON(buffer, cond)                                \
366         ({                                                      \
367                 int _____ret = unlikely(cond);                  \
368                 if (_____ret) {                                 \
369                         atomic_inc(&buffer->record_disabled);   \
370                         WARN_ON(1);                             \
371                 }                                               \
372                 _____ret;                                       \
373         })
374
375 /* Up this if you want to test the TIME_EXTENTS and normalization */
376 #define DEBUG_SHIFT 0
377
378 u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
379 {
380         u64 time;
381
382         preempt_disable_notrace();
383         /* shift to debug/test normalization and TIME_EXTENTS */
384         time = buffer->clock() << DEBUG_SHIFT;
385         preempt_enable_no_resched_notrace();
386
387         return time;
388 }
389 EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
390
391 void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
392                                       int cpu, u64 *ts)
393 {
394         /* Just stupid testing the normalize function and deltas */
395         *ts >>= DEBUG_SHIFT;
396 }
397 EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
398
399 /**
400  * check_pages - integrity check of buffer pages
401  * @cpu_buffer: CPU buffer with pages to test
402  *
403  * As a safety measure we check to make sure the data pages have not
404  * been corrupted.
405  */
406 static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
407 {
408         struct list_head *head = &cpu_buffer->pages;
409         struct buffer_page *bpage, *tmp;
410
411         if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
412                 return -1;
413         if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
414                 return -1;
415
416         list_for_each_entry_safe(bpage, tmp, head, list) {
417                 if (RB_WARN_ON(cpu_buffer,
418                                bpage->list.next->prev != &bpage->list))
419                         return -1;
420                 if (RB_WARN_ON(cpu_buffer,
421                                bpage->list.prev->next != &bpage->list))
422                         return -1;
423         }
424
425         return 0;
426 }
427
428 static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
429                              unsigned nr_pages)
430 {
431         struct list_head *head = &cpu_buffer->pages;
432         struct buffer_page *bpage, *tmp;
433         unsigned long addr;
434         LIST_HEAD(pages);
435         unsigned i;
436
437         for (i = 0; i < nr_pages; i++) {
438                 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
439                                     GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
440                 if (!bpage)
441                         goto free_pages;
442                 list_add(&bpage->list, &pages);
443
444                 addr = __get_free_page(GFP_KERNEL);
445                 if (!addr)
446                         goto free_pages;
447                 bpage->page = (void *)addr;
448                 rb_init_page(bpage->page);
449         }
450
451         list_splice(&pages, head);
452
453         rb_check_pages(cpu_buffer);
454
455         return 0;
456
457  free_pages:
458         list_for_each_entry_safe(bpage, tmp, &pages, list) {
459                 list_del_init(&bpage->list);
460                 free_buffer_page(bpage);
461         }
462         return -ENOMEM;
463 }
464
465 static struct ring_buffer_per_cpu *
466 rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
467 {
468         struct ring_buffer_per_cpu *cpu_buffer;
469         struct buffer_page *bpage;
470         unsigned long addr;
471         int ret;
472
473         cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
474                                   GFP_KERNEL, cpu_to_node(cpu));
475         if (!cpu_buffer)
476                 return NULL;
477
478         cpu_buffer->cpu = cpu;
479         cpu_buffer->buffer = buffer;
480         spin_lock_init(&cpu_buffer->reader_lock);
481         cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
482         INIT_LIST_HEAD(&cpu_buffer->pages);
483
484         bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
485                             GFP_KERNEL, cpu_to_node(cpu));
486         if (!bpage)
487                 goto fail_free_buffer;
488
489         cpu_buffer->reader_page = bpage;
490         addr = __get_free_page(GFP_KERNEL);
491         if (!addr)
492                 goto fail_free_reader;
493         bpage->page = (void *)addr;
494         rb_init_page(bpage->page);
495
496         INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
497
498         ret = rb_allocate_pages(cpu_buffer, buffer->pages);
499         if (ret < 0)
500                 goto fail_free_reader;
501
502         cpu_buffer->head_page
503                 = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
504         cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
505
506         return cpu_buffer;
507
508  fail_free_reader:
509         free_buffer_page(cpu_buffer->reader_page);
510
511  fail_free_buffer:
512         kfree(cpu_buffer);
513         return NULL;
514 }
515
516 static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
517 {
518         struct list_head *head = &cpu_buffer->pages;
519         struct buffer_page *bpage, *tmp;
520
521         list_del_init(&cpu_buffer->reader_page->list);
522         free_buffer_page(cpu_buffer->reader_page);
523
524         list_for_each_entry_safe(bpage, tmp, head, list) {
525                 list_del_init(&bpage->list);
526                 free_buffer_page(bpage);
527         }
528         kfree(cpu_buffer);
529 }
530
531 /*
532  * Causes compile errors if the struct buffer_page gets bigger
533  * than the struct page.
534  */
535 extern int ring_buffer_page_too_big(void);
536
537 #ifdef CONFIG_HOTPLUG_CPU
538 static int __cpuinit rb_cpu_notify(struct notifier_block *self,
539                                    unsigned long action, void *hcpu);
540 #endif
541
542 /**
543  * ring_buffer_alloc - allocate a new ring_buffer
544  * @size: the size in bytes per cpu that is needed.
545  * @flags: attributes to set for the ring buffer.
546  *
547  * Currently the only flag that is available is the RB_FL_OVERWRITE
548  * flag. This flag means that the buffer will overwrite old data
549  * when the buffer wraps. If this flag is not set, the buffer will
550  * drop data when the tail hits the head.
551  */
552 struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
553 {
554         struct ring_buffer *buffer;
555         int bsize;
556         int cpu;
557
558         /* Paranoid! Optimizes out when all is well */
559         if (sizeof(struct buffer_page) > sizeof(struct page))
560                 ring_buffer_page_too_big();
561
562
563         /* keep it in its own cache line */
564         buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
565                          GFP_KERNEL);
566         if (!buffer)
567                 return NULL;
568
569         if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
570                 goto fail_free_buffer;
571
572         buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
573         buffer->flags = flags;
574         buffer->clock = trace_clock_local;
575
576         /* need at least two pages */
577         if (buffer->pages == 1)
578                 buffer->pages++;
579
580         get_online_cpus();
581         cpumask_copy(buffer->cpumask, cpu_online_mask);
582         buffer->cpus = nr_cpu_ids;
583
584         bsize = sizeof(void *) * nr_cpu_ids;
585         buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
586                                   GFP_KERNEL);
587         if (!buffer->buffers)
588                 goto fail_free_cpumask;
589
590         for_each_buffer_cpu(buffer, cpu) {
591                 buffer->buffers[cpu] =
592                         rb_allocate_cpu_buffer(buffer, cpu);
593                 if (!buffer->buffers[cpu])
594                         goto fail_free_buffers;
595         }
596
597 #ifdef CONFIG_HOTPLUG_CPU
598         buffer->cpu_notify.notifier_call = rb_cpu_notify;
599         buffer->cpu_notify.priority = 0;
600         register_cpu_notifier(&buffer->cpu_notify);
601 #endif
602
603         put_online_cpus();
604         mutex_init(&buffer->mutex);
605
606         return buffer;
607
608  fail_free_buffers:
609         for_each_buffer_cpu(buffer, cpu) {
610                 if (buffer->buffers[cpu])
611                         rb_free_cpu_buffer(buffer->buffers[cpu]);
612         }
613         kfree(buffer->buffers);
614
615  fail_free_cpumask:
616         free_cpumask_var(buffer->cpumask);
617         put_online_cpus();
618
619  fail_free_buffer:
620         kfree(buffer);
621         return NULL;
622 }
623 EXPORT_SYMBOL_GPL(ring_buffer_alloc);
624
625 /**
626  * ring_buffer_free - free a ring buffer.
627  * @buffer: the buffer to free.
628  */
629 void
630 ring_buffer_free(struct ring_buffer *buffer)
631 {
632         int cpu;
633
634         get_online_cpus();
635
636 #ifdef CONFIG_HOTPLUG_CPU
637         unregister_cpu_notifier(&buffer->cpu_notify);
638 #endif
639
640         for_each_buffer_cpu(buffer, cpu)
641                 rb_free_cpu_buffer(buffer->buffers[cpu]);
642
643         put_online_cpus();
644
645         free_cpumask_var(buffer->cpumask);
646
647         kfree(buffer);
648 }
649 EXPORT_SYMBOL_GPL(ring_buffer_free);
650
651 void ring_buffer_set_clock(struct ring_buffer *buffer,
652                            u64 (*clock)(void))
653 {
654         buffer->clock = clock;
655 }
656
657 static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
658
659 static void
660 rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
661 {
662         struct buffer_page *bpage;
663         struct list_head *p;
664         unsigned i;
665
666         atomic_inc(&cpu_buffer->record_disabled);
667         synchronize_sched();
668
669         for (i = 0; i < nr_pages; i++) {
670                 if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
671                         return;
672                 p = cpu_buffer->pages.next;
673                 bpage = list_entry(p, struct buffer_page, list);
674                 list_del_init(&bpage->list);
675                 free_buffer_page(bpage);
676         }
677         if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
678                 return;
679
680         rb_reset_cpu(cpu_buffer);
681
682         rb_check_pages(cpu_buffer);
683
684         atomic_dec(&cpu_buffer->record_disabled);
685
686 }
687
688 static void
689 rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
690                 struct list_head *pages, unsigned nr_pages)
691 {
692         struct buffer_page *bpage;
693         struct list_head *p;
694         unsigned i;
695
696         atomic_inc(&cpu_buffer->record_disabled);
697         synchronize_sched();
698
699         for (i = 0; i < nr_pages; i++) {
700                 if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
701                         return;
702                 p = pages->next;
703                 bpage = list_entry(p, struct buffer_page, list);
704                 list_del_init(&bpage->list);
705                 list_add_tail(&bpage->list, &cpu_buffer->pages);
706         }
707         rb_reset_cpu(cpu_buffer);
708
709         rb_check_pages(cpu_buffer);
710
711         atomic_dec(&cpu_buffer->record_disabled);
712 }
713
714 /**
715  * ring_buffer_resize - resize the ring buffer
716  * @buffer: the buffer to resize.
717  * @size: the new size.
718  *
719  * The tracer is responsible for making sure that the buffer is
720  * not being used while changing the size.
721  * Note: We may be able to change the above requirement by using
722  *  RCU synchronizations.
723  *
724  * Minimum size is 2 * BUF_PAGE_SIZE.
725  *
726  * Returns -1 on failure.
727  */
728 int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
729 {
730         struct ring_buffer_per_cpu *cpu_buffer;
731         unsigned nr_pages, rm_pages, new_pages;
732         struct buffer_page *bpage, *tmp;
733         unsigned long buffer_size;
734         unsigned long addr;
735         LIST_HEAD(pages);
736         int i, cpu;
737
738         /*
739          * Always succeed at resizing a non-existent buffer:
740          */
741         if (!buffer)
742                 return size;
743
744         size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
745         size *= BUF_PAGE_SIZE;
746         buffer_size = buffer->pages * BUF_PAGE_SIZE;
747
748         /* we need a minimum of two pages */
749         if (size < BUF_PAGE_SIZE * 2)
750                 size = BUF_PAGE_SIZE * 2;
751
752         if (size == buffer_size)
753                 return size;
754
755         mutex_lock(&buffer->mutex);
756         get_online_cpus();
757
758         nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
759
760         if (size < buffer_size) {
761
762                 /* easy case, just free pages */
763                 if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
764                         goto out_fail;
765
766                 rm_pages = buffer->pages - nr_pages;
767
768                 for_each_buffer_cpu(buffer, cpu) {
769                         cpu_buffer = buffer->buffers[cpu];
770                         rb_remove_pages(cpu_buffer, rm_pages);
771                 }
772                 goto out;
773         }
774
775         /*
776          * This is a bit more difficult. We only want to add pages
777          * when we can allocate enough for all CPUs. We do this
778          * by allocating all the pages and storing them on a local
779          * link list. If we succeed in our allocation, then we
780          * add these pages to the cpu_buffers. Otherwise we just free
781          * them all and return -ENOMEM;
782          */
783         if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
784                 goto out_fail;
785
786         new_pages = nr_pages - buffer->pages;
787
788         for_each_buffer_cpu(buffer, cpu) {
789                 for (i = 0; i < new_pages; i++) {
790                         bpage = kzalloc_node(ALIGN(sizeof(*bpage),
791                                                   cache_line_size()),
792                                             GFP_KERNEL, cpu_to_node(cpu));
793                         if (!bpage)
794                                 goto free_pages;
795                         list_add(&bpage->list, &pages);
796                         addr = __get_free_page(GFP_KERNEL);
797                         if (!addr)
798                                 goto free_pages;
799                         bpage->page = (void *)addr;
800                         rb_init_page(bpage->page);
801                 }
802         }
803
804         for_each_buffer_cpu(buffer, cpu) {
805                 cpu_buffer = buffer->buffers[cpu];
806                 rb_insert_pages(cpu_buffer, &pages, new_pages);
807         }
808
809         if (RB_WARN_ON(buffer, !list_empty(&pages)))
810                 goto out_fail;
811
812  out:
813         buffer->pages = nr_pages;
814         put_online_cpus();
815         mutex_unlock(&buffer->mutex);
816
817         return size;
818
819  free_pages:
820         list_for_each_entry_safe(bpage, tmp, &pages, list) {
821                 list_del_init(&bpage->list);
822                 free_buffer_page(bpage);
823         }
824         put_online_cpus();
825         mutex_unlock(&buffer->mutex);
826         return -ENOMEM;
827
828         /*
829          * Something went totally wrong, and we are too paranoid
830          * to even clean up the mess.
831          */
832  out_fail:
833         put_online_cpus();
834         mutex_unlock(&buffer->mutex);
835         return -1;
836 }
837 EXPORT_SYMBOL_GPL(ring_buffer_resize);
838
839 static inline int rb_null_event(struct ring_buffer_event *event)
840 {
841         return event->type == RINGBUF_TYPE_PADDING;
842 }
843
844 static inline void *
845 __rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
846 {
847         return bpage->data + index;
848 }
849
850 static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
851 {
852         return bpage->page->data + index;
853 }
854
855 static inline struct ring_buffer_event *
856 rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
857 {
858         return __rb_page_index(cpu_buffer->reader_page,
859                                cpu_buffer->reader_page->read);
860 }
861
862 static inline struct ring_buffer_event *
863 rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
864 {
865         return __rb_page_index(cpu_buffer->head_page,
866                                cpu_buffer->head_page->read);
867 }
868
869 static inline struct ring_buffer_event *
870 rb_iter_head_event(struct ring_buffer_iter *iter)
871 {
872         return __rb_page_index(iter->head_page, iter->head);
873 }
874
875 static inline unsigned rb_page_write(struct buffer_page *bpage)
876 {
877         return local_read(&bpage->write);
878 }
879
880 static inline unsigned rb_page_commit(struct buffer_page *bpage)
881 {
882         return local_read(&bpage->page->commit);
883 }
884
885 /* Size is determined by what has been commited */
886 static inline unsigned rb_page_size(struct buffer_page *bpage)
887 {
888         return rb_page_commit(bpage);
889 }
890
891 static inline unsigned
892 rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
893 {
894         return rb_page_commit(cpu_buffer->commit_page);
895 }
896
897 static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
898 {
899         return rb_page_commit(cpu_buffer->head_page);
900 }
901
902 /*
903  * When the tail hits the head and the buffer is in overwrite mode,
904  * the head jumps to the next page and all content on the previous
905  * page is discarded. But before doing so, we update the overrun
906  * variable of the buffer.
907  */
908 static void rb_update_overflow(struct ring_buffer_per_cpu *cpu_buffer)
909 {
910         struct ring_buffer_event *event;
911         unsigned long head;
912
913         for (head = 0; head < rb_head_size(cpu_buffer);
914              head += rb_event_length(event)) {
915
916                 event = __rb_page_index(cpu_buffer->head_page, head);
917                 if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
918                         return;
919                 /* Only count data entries */
920                 if (event->type != RINGBUF_TYPE_DATA)
921                         continue;
922                 cpu_buffer->overrun++;
923                 cpu_buffer->entries--;
924         }
925 }
926
927 static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
928                                struct buffer_page **bpage)
929 {
930         struct list_head *p = (*bpage)->list.next;
931
932         if (p == &cpu_buffer->pages)
933                 p = p->next;
934
935         *bpage = list_entry(p, struct buffer_page, list);
936 }
937
938 static inline unsigned
939 rb_event_index(struct ring_buffer_event *event)
940 {
941         unsigned long addr = (unsigned long)event;
942
943         return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
944 }
945
946 static int
947 rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
948              struct ring_buffer_event *event)
949 {
950         unsigned long addr = (unsigned long)event;
951         unsigned long index;
952
953         index = rb_event_index(event);
954         addr &= PAGE_MASK;
955
956         return cpu_buffer->commit_page->page == (void *)addr &&
957                 rb_commit_index(cpu_buffer) == index;
958 }
959
960 static void
961 rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
962                     struct ring_buffer_event *event)
963 {
964         unsigned long addr = (unsigned long)event;
965         unsigned long index;
966
967         index = rb_event_index(event);
968         addr &= PAGE_MASK;
969
970         while (cpu_buffer->commit_page->page != (void *)addr) {
971                 if (RB_WARN_ON(cpu_buffer,
972                           cpu_buffer->commit_page == cpu_buffer->tail_page))
973                         return;
974                 cpu_buffer->commit_page->page->commit =
975                         cpu_buffer->commit_page->write;
976                 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
977                 cpu_buffer->write_stamp =
978                         cpu_buffer->commit_page->page->time_stamp;
979         }
980
981         /* Now set the commit to the event's index */
982         local_set(&cpu_buffer->commit_page->page->commit, index);
983 }
984
985 static void
986 rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
987 {
988         /*
989          * We only race with interrupts and NMIs on this CPU.
990          * If we own the commit event, then we can commit
991          * all others that interrupted us, since the interruptions
992          * are in stack format (they finish before they come
993          * back to us). This allows us to do a simple loop to
994          * assign the commit to the tail.
995          */
996  again:
997         while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
998                 cpu_buffer->commit_page->page->commit =
999                         cpu_buffer->commit_page->write;
1000                 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
1001                 cpu_buffer->write_stamp =
1002                         cpu_buffer->commit_page->page->time_stamp;
1003                 /* add barrier to keep gcc from optimizing too much */
1004                 barrier();
1005         }
1006         while (rb_commit_index(cpu_buffer) !=
1007                rb_page_write(cpu_buffer->commit_page)) {
1008                 cpu_buffer->commit_page->page->commit =
1009                         cpu_buffer->commit_page->write;
1010                 barrier();
1011         }
1012
1013         /* again, keep gcc from optimizing */
1014         barrier();
1015
1016         /*
1017          * If an interrupt came in just after the first while loop
1018          * and pushed the tail page forward, we will be left with
1019          * a dangling commit that will never go forward.
1020          */
1021         if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
1022                 goto again;
1023 }
1024
1025 static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1026 {
1027         cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
1028         cpu_buffer->reader_page->read = 0;
1029 }
1030
1031 static void rb_inc_iter(struct ring_buffer_iter *iter)
1032 {
1033         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
1034
1035         /*
1036          * The iterator could be on the reader page (it starts there).
1037          * But the head could have moved, since the reader was
1038          * found. Check for this case and assign the iterator
1039          * to the head page instead of next.
1040          */
1041         if (iter->head_page == cpu_buffer->reader_page)
1042                 iter->head_page = cpu_buffer->head_page;
1043         else
1044                 rb_inc_page(cpu_buffer, &iter->head_page);
1045
1046         iter->read_stamp = iter->head_page->page->time_stamp;
1047         iter->head = 0;
1048 }
1049
1050 /**
1051  * ring_buffer_update_event - update event type and data
1052  * @event: the even to update
1053  * @type: the type of event
1054  * @length: the size of the event field in the ring buffer
1055  *
1056  * Update the type and data fields of the event. The length
1057  * is the actual size that is written to the ring buffer,
1058  * and with this, we can determine what to place into the
1059  * data field.
1060  */
1061 static void
1062 rb_update_event(struct ring_buffer_event *event,
1063                          unsigned type, unsigned length)
1064 {
1065         event->type = type;
1066
1067         switch (type) {
1068
1069         case RINGBUF_TYPE_PADDING:
1070                 break;
1071
1072         case RINGBUF_TYPE_TIME_EXTEND:
1073                 event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
1074                 break;
1075
1076         case RINGBUF_TYPE_TIME_STAMP:
1077                 event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
1078                 break;
1079
1080         case RINGBUF_TYPE_DATA:
1081                 length -= RB_EVNT_HDR_SIZE;
1082                 if (length > RB_MAX_SMALL_DATA) {
1083                         event->len = 0;
1084                         event->array[0] = length;
1085                 } else
1086                         event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
1087                 break;
1088         default:
1089                 BUG();
1090         }
1091 }
1092
1093 static unsigned rb_calculate_event_length(unsigned length)
1094 {
1095         struct ring_buffer_event event; /* Used only for sizeof array */
1096
1097         /* zero length can cause confusions */
1098         if (!length)
1099                 length = 1;
1100
1101         if (length > RB_MAX_SMALL_DATA)
1102                 length += sizeof(event.array[0]);
1103
1104         length += RB_EVNT_HDR_SIZE;
1105         length = ALIGN(length, RB_ALIGNMENT);
1106
1107         return length;
1108 }
1109
1110 static struct ring_buffer_event *
1111 __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1112                   unsigned type, unsigned long length, u64 *ts)
1113 {
1114         struct buffer_page *tail_page, *head_page, *reader_page, *commit_page;
1115         unsigned long tail, write;
1116         struct ring_buffer *buffer = cpu_buffer->buffer;
1117         struct ring_buffer_event *event;
1118         unsigned long flags;
1119         bool lock_taken = false;
1120
1121         commit_page = cpu_buffer->commit_page;
1122         /* we just need to protect against interrupts */
1123         barrier();
1124         tail_page = cpu_buffer->tail_page;
1125         write = local_add_return(length, &tail_page->write);
1126         tail = write - length;
1127
1128         /* See if we shot pass the end of this buffer page */
1129         if (write > BUF_PAGE_SIZE) {
1130                 struct buffer_page *next_page = tail_page;
1131
1132                 local_irq_save(flags);
1133                 /*
1134                  * Since the write to the buffer is still not
1135                  * fully lockless, we must be careful with NMIs.
1136                  * The locks in the writers are taken when a write
1137                  * crosses to a new page. The locks protect against
1138                  * races with the readers (this will soon be fixed
1139                  * with a lockless solution).
1140                  *
1141                  * Because we can not protect against NMIs, and we
1142                  * want to keep traces reentrant, we need to manage
1143                  * what happens when we are in an NMI.
1144                  *
1145                  * NMIs can happen after we take the lock.
1146                  * If we are in an NMI, only take the lock
1147                  * if it is not already taken. Otherwise
1148                  * simply fail.
1149                  */
1150                 if (unlikely(in_nmi())) {
1151                         if (!__raw_spin_trylock(&cpu_buffer->lock))
1152                                 goto out_reset;
1153                 } else
1154                         __raw_spin_lock(&cpu_buffer->lock);
1155
1156                 lock_taken = true;
1157
1158                 rb_inc_page(cpu_buffer, &next_page);
1159
1160                 head_page = cpu_buffer->head_page;
1161                 reader_page = cpu_buffer->reader_page;
1162
1163                 /* we grabbed the lock before incrementing */
1164                 if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
1165                         goto out_reset;
1166
1167                 /*
1168                  * If for some reason, we had an interrupt storm that made
1169                  * it all the way around the buffer, bail, and warn
1170                  * about it.
1171                  */
1172                 if (unlikely(next_page == commit_page)) {
1173                         WARN_ON_ONCE(1);
1174                         goto out_reset;
1175                 }
1176
1177                 if (next_page == head_page) {
1178                         if (!(buffer->flags & RB_FL_OVERWRITE))
1179                                 goto out_reset;
1180
1181                         /* tail_page has not moved yet? */
1182                         if (tail_page == cpu_buffer->tail_page) {
1183                                 /* count overflows */
1184                                 rb_update_overflow(cpu_buffer);
1185
1186                                 rb_inc_page(cpu_buffer, &head_page);
1187                                 cpu_buffer->head_page = head_page;
1188                                 cpu_buffer->head_page->read = 0;
1189                         }
1190                 }
1191
1192                 /*
1193                  * If the tail page is still the same as what we think
1194                  * it is, then it is up to us to update the tail
1195                  * pointer.
1196                  */
1197                 if (tail_page == cpu_buffer->tail_page) {
1198                         local_set(&next_page->write, 0);
1199                         local_set(&next_page->page->commit, 0);
1200                         cpu_buffer->tail_page = next_page;
1201
1202                         /* reread the time stamp */
1203                         *ts = ring_buffer_time_stamp(buffer, cpu_buffer->cpu);
1204                         cpu_buffer->tail_page->page->time_stamp = *ts;
1205                 }
1206
1207                 /*
1208                  * The actual tail page has moved forward.
1209                  */
1210                 if (tail < BUF_PAGE_SIZE) {
1211                         /* Mark the rest of the page with padding */
1212                         event = __rb_page_index(tail_page, tail);
1213                         event->type = RINGBUF_TYPE_PADDING;
1214                 }
1215
1216                 if (tail <= BUF_PAGE_SIZE)
1217                         /* Set the write back to the previous setting */
1218                         local_set(&tail_page->write, tail);
1219
1220                 /*
1221                  * If this was a commit entry that failed,
1222                  * increment that too
1223                  */
1224                 if (tail_page == cpu_buffer->commit_page &&
1225                     tail == rb_commit_index(cpu_buffer)) {
1226                         rb_set_commit_to_write(cpu_buffer);
1227                 }
1228
1229                 __raw_spin_unlock(&cpu_buffer->lock);
1230                 local_irq_restore(flags);
1231
1232                 /* fail and let the caller try again */
1233                 return ERR_PTR(-EAGAIN);
1234         }
1235
1236         /* We reserved something on the buffer */
1237
1238         if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
1239                 return NULL;
1240
1241         event = __rb_page_index(tail_page, tail);
1242         rb_update_event(event, type, length);
1243
1244         /*
1245          * If this is a commit and the tail is zero, then update
1246          * this page's time stamp.
1247          */
1248         if (!tail && rb_is_commit(cpu_buffer, event))
1249                 cpu_buffer->commit_page->page->time_stamp = *ts;
1250
1251         return event;
1252
1253  out_reset:
1254         /* reset write */
1255         if (tail <= BUF_PAGE_SIZE)
1256                 local_set(&tail_page->write, tail);
1257
1258         if (likely(lock_taken))
1259                 __raw_spin_unlock(&cpu_buffer->lock);
1260         local_irq_restore(flags);
1261         return NULL;
1262 }
1263
1264 static int
1265 rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
1266                   u64 *ts, u64 *delta)
1267 {
1268         struct ring_buffer_event *event;
1269         static int once;
1270         int ret;
1271
1272         if (unlikely(*delta > (1ULL << 59) && !once++)) {
1273                 printk(KERN_WARNING "Delta way too big! %llu"
1274                        " ts=%llu write stamp = %llu\n",
1275                        (unsigned long long)*delta,
1276                        (unsigned long long)*ts,
1277                        (unsigned long long)cpu_buffer->write_stamp);
1278                 WARN_ON(1);
1279         }
1280
1281         /*
1282          * The delta is too big, we to add a
1283          * new timestamp.
1284          */
1285         event = __rb_reserve_next(cpu_buffer,
1286                                   RINGBUF_TYPE_TIME_EXTEND,
1287                                   RB_LEN_TIME_EXTEND,
1288                                   ts);
1289         if (!event)
1290                 return -EBUSY;
1291
1292         if (PTR_ERR(event) == -EAGAIN)
1293                 return -EAGAIN;
1294
1295         /* Only a commited time event can update the write stamp */
1296         if (rb_is_commit(cpu_buffer, event)) {
1297                 /*
1298                  * If this is the first on the page, then we need to
1299                  * update the page itself, and just put in a zero.
1300                  */
1301                 if (rb_event_index(event)) {
1302                         event->time_delta = *delta & TS_MASK;
1303                         event->array[0] = *delta >> TS_SHIFT;
1304                 } else {
1305                         cpu_buffer->commit_page->page->time_stamp = *ts;
1306                         event->time_delta = 0;
1307                         event->array[0] = 0;
1308                 }
1309                 cpu_buffer->write_stamp = *ts;
1310                 /* let the caller know this was the commit */
1311                 ret = 1;
1312         } else {
1313                 /* Darn, this is just wasted space */
1314                 event->time_delta = 0;
1315                 event->array[0] = 0;
1316                 ret = 0;
1317         }
1318
1319         *delta = 0;
1320
1321         return ret;
1322 }
1323
1324 static struct ring_buffer_event *
1325 rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
1326                       unsigned type, unsigned long length)
1327 {
1328         struct ring_buffer_event *event;
1329         u64 ts, delta;
1330         int commit = 0;
1331         int nr_loops = 0;
1332
1333  again:
1334         /*
1335          * We allow for interrupts to reenter here and do a trace.
1336          * If one does, it will cause this original code to loop
1337          * back here. Even with heavy interrupts happening, this
1338          * should only happen a few times in a row. If this happens
1339          * 1000 times in a row, there must be either an interrupt
1340          * storm or we have something buggy.
1341          * Bail!
1342          */
1343         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
1344                 return NULL;
1345
1346         ts = ring_buffer_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu);
1347
1348         /*
1349          * Only the first commit can update the timestamp.
1350          * Yes there is a race here. If an interrupt comes in
1351          * just after the conditional and it traces too, then it
1352          * will also check the deltas. More than one timestamp may
1353          * also be made. But only the entry that did the actual
1354          * commit will be something other than zero.
1355          */
1356         if (cpu_buffer->tail_page == cpu_buffer->commit_page &&
1357             rb_page_write(cpu_buffer->tail_page) ==
1358             rb_commit_index(cpu_buffer)) {
1359
1360                 delta = ts - cpu_buffer->write_stamp;
1361
1362                 /* make sure this delta is calculated here */
1363                 barrier();
1364
1365                 /* Did the write stamp get updated already? */
1366                 if (unlikely(ts < cpu_buffer->write_stamp))
1367                         delta = 0;
1368
1369                 if (test_time_stamp(delta)) {
1370
1371                         commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);
1372
1373                         if (commit == -EBUSY)
1374                                 return NULL;
1375
1376                         if (commit == -EAGAIN)
1377                                 goto again;
1378
1379                         RB_WARN_ON(cpu_buffer, commit < 0);
1380                 }
1381         } else
1382                 /* Non commits have zero deltas */
1383                 delta = 0;
1384
1385         event = __rb_reserve_next(cpu_buffer, type, length, &ts);
1386         if (PTR_ERR(event) == -EAGAIN)
1387                 goto again;
1388
1389         if (!event) {
1390                 if (unlikely(commit))
1391                         /*
1392                          * Ouch! We needed a timestamp and it was commited. But
1393                          * we didn't get our event reserved.
1394                          */
1395                         rb_set_commit_to_write(cpu_buffer);
1396                 return NULL;
1397         }
1398
1399         /*
1400          * If the timestamp was commited, make the commit our entry
1401          * now so that we will update it when needed.
1402          */
1403         if (commit)
1404                 rb_set_commit_event(cpu_buffer, event);
1405         else if (!rb_is_commit(cpu_buffer, event))
1406                 delta = 0;
1407
1408         event->time_delta = delta;
1409
1410         return event;
1411 }
1412
1413 static DEFINE_PER_CPU(int, rb_need_resched);
1414
1415 /**
1416  * ring_buffer_lock_reserve - reserve a part of the buffer
1417  * @buffer: the ring buffer to reserve from
1418  * @length: the length of the data to reserve (excluding event header)
1419  *
1420  * Returns a reseverd event on the ring buffer to copy directly to.
1421  * The user of this interface will need to get the body to write into
1422  * and can use the ring_buffer_event_data() interface.
1423  *
1424  * The length is the length of the data needed, not the event length
1425  * which also includes the event header.
1426  *
1427  * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
1428  * If NULL is returned, then nothing has been allocated or locked.
1429  */
1430 struct ring_buffer_event *
1431 ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
1432 {
1433         struct ring_buffer_per_cpu *cpu_buffer;
1434         struct ring_buffer_event *event;
1435         int cpu, resched;
1436
1437         if (ring_buffer_flags != RB_BUFFERS_ON)
1438                 return NULL;
1439
1440         if (atomic_read(&buffer->record_disabled))
1441                 return NULL;
1442
1443         /* If we are tracing schedule, we don't want to recurse */
1444         resched = ftrace_preempt_disable();
1445
1446         cpu = raw_smp_processor_id();
1447
1448         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1449                 goto out;
1450
1451         cpu_buffer = buffer->buffers[cpu];
1452
1453         if (atomic_read(&cpu_buffer->record_disabled))
1454                 goto out;
1455
1456         length = rb_calculate_event_length(length);
1457         if (length > BUF_PAGE_SIZE)
1458                 goto out;
1459
1460         event = rb_reserve_next_event(cpu_buffer, RINGBUF_TYPE_DATA, length);
1461         if (!event)
1462                 goto out;
1463
1464         /*
1465          * Need to store resched state on this cpu.
1466          * Only the first needs to.
1467          */
1468
1469         if (preempt_count() == 1)
1470                 per_cpu(rb_need_resched, cpu) = resched;
1471
1472         return event;
1473
1474  out:
1475         ftrace_preempt_enable(resched);
1476         return NULL;
1477 }
1478 EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
1479
1480 static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
1481                       struct ring_buffer_event *event)
1482 {
1483         cpu_buffer->entries++;
1484
1485         /* Only process further if we own the commit */
1486         if (!rb_is_commit(cpu_buffer, event))
1487                 return;
1488
1489         cpu_buffer->write_stamp += event->time_delta;
1490
1491         rb_set_commit_to_write(cpu_buffer);
1492 }
1493
1494 /**
1495  * ring_buffer_unlock_commit - commit a reserved
1496  * @buffer: The buffer to commit to
1497  * @event: The event pointer to commit.
1498  *
1499  * This commits the data to the ring buffer, and releases any locks held.
1500  *
1501  * Must be paired with ring_buffer_lock_reserve.
1502  */
1503 int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1504                               struct ring_buffer_event *event)
1505 {
1506         struct ring_buffer_per_cpu *cpu_buffer;
1507         int cpu = raw_smp_processor_id();
1508
1509         cpu_buffer = buffer->buffers[cpu];
1510
1511         rb_commit(cpu_buffer, event);
1512
1513         /*
1514          * Only the last preempt count needs to restore preemption.
1515          */
1516         if (preempt_count() == 1)
1517                 ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
1518         else
1519                 preempt_enable_no_resched_notrace();
1520
1521         return 0;
1522 }
1523 EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
1524
1525 /**
1526  * ring_buffer_write - write data to the buffer without reserving
1527  * @buffer: The ring buffer to write to.
1528  * @length: The length of the data being written (excluding the event header)
1529  * @data: The data to write to the buffer.
1530  *
1531  * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
1532  * one function. If you already have the data to write to the buffer, it
1533  * may be easier to simply call this function.
1534  *
1535  * Note, like ring_buffer_lock_reserve, the length is the length of the data
1536  * and not the length of the event which would hold the header.
1537  */
1538 int ring_buffer_write(struct ring_buffer *buffer,
1539                         unsigned long length,
1540                         void *data)
1541 {
1542         struct ring_buffer_per_cpu *cpu_buffer;
1543         struct ring_buffer_event *event;
1544         unsigned long event_length;
1545         void *body;
1546         int ret = -EBUSY;
1547         int cpu, resched;
1548
1549         if (ring_buffer_flags != RB_BUFFERS_ON)
1550                 return -EBUSY;
1551
1552         if (atomic_read(&buffer->record_disabled))
1553                 return -EBUSY;
1554
1555         resched = ftrace_preempt_disable();
1556
1557         cpu = raw_smp_processor_id();
1558
1559         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1560                 goto out;
1561
1562         cpu_buffer = buffer->buffers[cpu];
1563
1564         if (atomic_read(&cpu_buffer->record_disabled))
1565                 goto out;
1566
1567         event_length = rb_calculate_event_length(length);
1568         event = rb_reserve_next_event(cpu_buffer,
1569                                       RINGBUF_TYPE_DATA, event_length);
1570         if (!event)
1571                 goto out;
1572
1573         body = rb_event_data(event);
1574
1575         memcpy(body, data, length);
1576
1577         rb_commit(cpu_buffer, event);
1578
1579         ret = 0;
1580  out:
1581         ftrace_preempt_enable(resched);
1582
1583         return ret;
1584 }
1585 EXPORT_SYMBOL_GPL(ring_buffer_write);
1586
1587 static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1588 {
1589         struct buffer_page *reader = cpu_buffer->reader_page;
1590         struct buffer_page *head = cpu_buffer->head_page;
1591         struct buffer_page *commit = cpu_buffer->commit_page;
1592
1593         return reader->read == rb_page_commit(reader) &&
1594                 (commit == reader ||
1595                  (commit == head &&
1596                   head->read == rb_page_commit(commit)));
1597 }
1598
1599 /**
1600  * ring_buffer_record_disable - stop all writes into the buffer
1601  * @buffer: The ring buffer to stop writes to.
1602  *
1603  * This prevents all writes to the buffer. Any attempt to write
1604  * to the buffer after this will fail and return NULL.
1605  *
1606  * The caller should call synchronize_sched() after this.
1607  */
1608 void ring_buffer_record_disable(struct ring_buffer *buffer)
1609 {
1610         atomic_inc(&buffer->record_disabled);
1611 }
1612 EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
1613
1614 /**
1615  * ring_buffer_record_enable - enable writes to the buffer
1616  * @buffer: The ring buffer to enable writes
1617  *
1618  * Note, multiple disables will need the same number of enables
1619  * to truely enable the writing (much like preempt_disable).
1620  */
1621 void ring_buffer_record_enable(struct ring_buffer *buffer)
1622 {
1623         atomic_dec(&buffer->record_disabled);
1624 }
1625 EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
1626
1627 /**
1628  * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
1629  * @buffer: The ring buffer to stop writes to.
1630  * @cpu: The CPU buffer to stop
1631  *
1632  * This prevents all writes to the buffer. Any attempt to write
1633  * to the buffer after this will fail and return NULL.
1634  *
1635  * The caller should call synchronize_sched() after this.
1636  */
1637 void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
1638 {
1639         struct ring_buffer_per_cpu *cpu_buffer;
1640
1641         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1642                 return;
1643
1644         cpu_buffer = buffer->buffers[cpu];
1645         atomic_inc(&cpu_buffer->record_disabled);
1646 }
1647 EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
1648
1649 /**
1650  * ring_buffer_record_enable_cpu - enable writes to the buffer
1651  * @buffer: The ring buffer to enable writes
1652  * @cpu: The CPU to enable.
1653  *
1654  * Note, multiple disables will need the same number of enables
1655  * to truely enable the writing (much like preempt_disable).
1656  */
1657 void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
1658 {
1659         struct ring_buffer_per_cpu *cpu_buffer;
1660
1661         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1662                 return;
1663
1664         cpu_buffer = buffer->buffers[cpu];
1665         atomic_dec(&cpu_buffer->record_disabled);
1666 }
1667 EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
1668
1669 /**
1670  * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
1671  * @buffer: The ring buffer
1672  * @cpu: The per CPU buffer to get the entries from.
1673  */
1674 unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
1675 {
1676         struct ring_buffer_per_cpu *cpu_buffer;
1677         unsigned long ret;
1678
1679         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1680                 return 0;
1681
1682         cpu_buffer = buffer->buffers[cpu];
1683         ret = cpu_buffer->entries;
1684
1685         return ret;
1686 }
1687 EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
1688
1689 /**
1690  * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
1691  * @buffer: The ring buffer
1692  * @cpu: The per CPU buffer to get the number of overruns from
1693  */
1694 unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
1695 {
1696         struct ring_buffer_per_cpu *cpu_buffer;
1697         unsigned long ret;
1698
1699         if (!cpumask_test_cpu(cpu, buffer->cpumask))
1700                 return 0;
1701
1702         cpu_buffer = buffer->buffers[cpu];
1703         ret = cpu_buffer->overrun;
1704
1705         return ret;
1706 }
1707 EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
1708
1709 /**
1710  * ring_buffer_entries - get the number of entries in a buffer
1711  * @buffer: The ring buffer
1712  *
1713  * Returns the total number of entries in the ring buffer
1714  * (all CPU entries)
1715  */
1716 unsigned long ring_buffer_entries(struct ring_buffer *buffer)
1717 {
1718         struct ring_buffer_per_cpu *cpu_buffer;
1719         unsigned long entries = 0;
1720         int cpu;
1721
1722         /* if you care about this being correct, lock the buffer */
1723         for_each_buffer_cpu(buffer, cpu) {
1724                 cpu_buffer = buffer->buffers[cpu];
1725                 entries += cpu_buffer->entries;
1726         }
1727
1728         return entries;
1729 }
1730 EXPORT_SYMBOL_GPL(ring_buffer_entries);
1731
1732 /**
1733  * ring_buffer_overrun_cpu - get the number of overruns in buffer
1734  * @buffer: The ring buffer
1735  *
1736  * Returns the total number of overruns in the ring buffer
1737  * (all CPU entries)
1738  */
1739 unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
1740 {
1741         struct ring_buffer_per_cpu *cpu_buffer;
1742         unsigned long overruns = 0;
1743         int cpu;
1744
1745         /* if you care about this being correct, lock the buffer */
1746         for_each_buffer_cpu(buffer, cpu) {
1747                 cpu_buffer = buffer->buffers[cpu];
1748                 overruns += cpu_buffer->overrun;
1749         }
1750
1751         return overruns;
1752 }
1753 EXPORT_SYMBOL_GPL(ring_buffer_overruns);
1754
1755 static void rb_iter_reset(struct ring_buffer_iter *iter)
1756 {
1757         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
1758
1759         /* Iterator usage is expected to have record disabled */
1760         if (list_empty(&cpu_buffer->reader_page->list)) {
1761                 iter->head_page = cpu_buffer->head_page;
1762                 iter->head = cpu_buffer->head_page->read;
1763         } else {
1764                 iter->head_page = cpu_buffer->reader_page;
1765                 iter->head = cpu_buffer->reader_page->read;
1766         }
1767         if (iter->head)
1768                 iter->read_stamp = cpu_buffer->read_stamp;
1769         else
1770                 iter->read_stamp = iter->head_page->page->time_stamp;
1771 }
1772
1773 /**
1774  * ring_buffer_iter_reset - reset an iterator
1775  * @iter: The iterator to reset
1776  *
1777  * Resets the iterator, so that it will start from the beginning
1778  * again.
1779  */
1780 void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
1781 {
1782         struct ring_buffer_per_cpu *cpu_buffer;
1783         unsigned long flags;
1784
1785         if (!iter)
1786                 return;
1787
1788         cpu_buffer = iter->cpu_buffer;
1789
1790         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
1791         rb_iter_reset(iter);
1792         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1793 }
1794 EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
1795
1796 /**
1797  * ring_buffer_iter_empty - check if an iterator has no more to read
1798  * @iter: The iterator to check
1799  */
1800 int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
1801 {
1802         struct ring_buffer_per_cpu *cpu_buffer;
1803
1804         cpu_buffer = iter->cpu_buffer;
1805
1806         return iter->head_page == cpu_buffer->commit_page &&
1807                 iter->head == rb_commit_index(cpu_buffer);
1808 }
1809 EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
1810
1811 static void
1812 rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
1813                      struct ring_buffer_event *event)
1814 {
1815         u64 delta;
1816
1817         switch (event->type) {
1818         case RINGBUF_TYPE_PADDING:
1819                 return;
1820
1821         case RINGBUF_TYPE_TIME_EXTEND:
1822                 delta = event->array[0];
1823                 delta <<= TS_SHIFT;
1824                 delta += event->time_delta;
1825                 cpu_buffer->read_stamp += delta;
1826                 return;
1827
1828         case RINGBUF_TYPE_TIME_STAMP:
1829                 /* FIXME: not implemented */
1830                 return;
1831
1832         case RINGBUF_TYPE_DATA:
1833                 cpu_buffer->read_stamp += event->time_delta;
1834                 return;
1835
1836         default:
1837                 BUG();
1838         }
1839         return;
1840 }
1841
1842 static void
1843 rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
1844                           struct ring_buffer_event *event)
1845 {
1846         u64 delta;
1847
1848         switch (event->type) {
1849         case RINGBUF_TYPE_PADDING:
1850                 return;
1851
1852         case RINGBUF_TYPE_TIME_EXTEND:
1853                 delta = event->array[0];
1854                 delta <<= TS_SHIFT;
1855                 delta += event->time_delta;
1856                 iter->read_stamp += delta;
1857                 return;
1858
1859         case RINGBUF_TYPE_TIME_STAMP:
1860                 /* FIXME: not implemented */
1861                 return;
1862
1863         case RINGBUF_TYPE_DATA:
1864                 iter->read_stamp += event->time_delta;
1865                 return;
1866
1867         default:
1868                 BUG();
1869         }
1870         return;
1871 }
1872
1873 static struct buffer_page *
1874 rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1875 {
1876         struct buffer_page *reader = NULL;
1877         unsigned long flags;
1878         int nr_loops = 0;
1879
1880         local_irq_save(flags);
1881         __raw_spin_lock(&cpu_buffer->lock);
1882
1883  again:
1884         /*
1885          * This should normally only loop twice. But because the
1886          * start of the reader inserts an empty page, it causes
1887          * a case where we will loop three times. There should be no
1888          * reason to loop four times (that I know of).
1889          */
1890         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
1891                 reader = NULL;
1892                 goto out;
1893         }
1894
1895         reader = cpu_buffer->reader_page;
1896
1897         /* If there's more to read, return this page */
1898         if (cpu_buffer->reader_page->read < rb_page_size(reader))
1899                 goto out;
1900
1901         /* Never should we have an index greater than the size */
1902         if (RB_WARN_ON(cpu_buffer,
1903                        cpu_buffer->reader_page->read > rb_page_size(reader)))
1904                 goto out;
1905
1906         /* check if we caught up to the tail */
1907         reader = NULL;
1908         if (cpu_buffer->commit_page == cpu_buffer->reader_page)
1909                 goto out;
1910
1911         /*
1912          * Splice the empty reader page into the list around the head.
1913          * Reset the reader page to size zero.
1914          */
1915
1916         reader = cpu_buffer->head_page;
1917         cpu_buffer->reader_page->list.next = reader->list.next;
1918         cpu_buffer->reader_page->list.prev = reader->list.prev;
1919
1920         local_set(&cpu_buffer->reader_page->write, 0);
1921         local_set(&cpu_buffer->reader_page->page->commit, 0);
1922
1923         /* Make the reader page now replace the head */
1924         reader->list.prev->next = &cpu_buffer->reader_page->list;
1925         reader->list.next->prev = &cpu_buffer->reader_page->list;
1926
1927         /*
1928          * If the tail is on the reader, then we must set the head
1929          * to the inserted page, otherwise we set it one before.
1930          */
1931         cpu_buffer->head_page = cpu_buffer->reader_page;
1932
1933         if (cpu_buffer->commit_page != reader)
1934                 rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
1935
1936         /* Finally update the reader page to the new head */
1937         cpu_buffer->reader_page = reader;
1938         rb_reset_reader_page(cpu_buffer);
1939
1940         goto again;
1941
1942  out:
1943         __raw_spin_unlock(&cpu_buffer->lock);
1944         local_irq_restore(flags);
1945
1946         return reader;
1947 }
1948
1949 static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
1950 {
1951         struct ring_buffer_event *event;
1952         struct buffer_page *reader;
1953         unsigned length;
1954
1955         reader = rb_get_reader_page(cpu_buffer);
1956
1957         /* This function should not be called when buffer is empty */
1958         if (RB_WARN_ON(cpu_buffer, !reader))
1959                 return;
1960
1961         event = rb_reader_event(cpu_buffer);
1962
1963         if (event->type == RINGBUF_TYPE_DATA)
1964                 cpu_buffer->entries--;
1965
1966         rb_update_read_stamp(cpu_buffer, event);
1967
1968         length = rb_event_length(event);
1969         cpu_buffer->reader_page->read += length;
1970 }
1971
1972 static void rb_advance_iter(struct ring_buffer_iter *iter)
1973 {
1974         struct ring_buffer *buffer;
1975         struct ring_buffer_per_cpu *cpu_buffer;
1976         struct ring_buffer_event *event;
1977         unsigned length;
1978
1979         cpu_buffer = iter->cpu_buffer;
1980         buffer = cpu_buffer->buffer;
1981
1982         /*
1983          * Check if we are at the end of the buffer.
1984          */
1985         if (iter->head >= rb_page_size(iter->head_page)) {
1986                 if (RB_WARN_ON(buffer,
1987                                iter->head_page == cpu_buffer->commit_page))
1988                         return;
1989                 rb_inc_iter(iter);
1990                 return;
1991         }
1992
1993         event = rb_iter_head_event(iter);
1994
1995         length = rb_event_length(event);
1996
1997         /*
1998          * This should not be called to advance the header if we are
1999          * at the tail of the buffer.
2000          */
2001         if (RB_WARN_ON(cpu_buffer,
2002                        (iter->head_page == cpu_buffer->commit_page) &&
2003                        (iter->head + length > rb_commit_index(cpu_buffer))))
2004                 return;
2005
2006         rb_update_iter_read_stamp(iter, event);
2007
2008         iter->head += length;
2009
2010         /* check for end of page padding */
2011         if ((iter->head >= rb_page_size(iter->head_page)) &&
2012             (iter->head_page != cpu_buffer->commit_page))
2013                 rb_advance_iter(iter);
2014 }
2015
2016 static struct ring_buffer_event *
2017 rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
2018 {
2019         struct ring_buffer_per_cpu *cpu_buffer;
2020         struct ring_buffer_event *event;
2021         struct buffer_page *reader;
2022         int nr_loops = 0;
2023
2024         cpu_buffer = buffer->buffers[cpu];
2025
2026  again:
2027         /*
2028          * We repeat when a timestamp is encountered. It is possible
2029          * to get multiple timestamps from an interrupt entering just
2030          * as one timestamp is about to be written. The max times
2031          * that this can happen is the number of nested interrupts we
2032          * can have.  Nesting 10 deep of interrupts is clearly
2033          * an anomaly.
2034          */
2035         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
2036                 return NULL;
2037
2038         reader = rb_get_reader_page(cpu_buffer);
2039         if (!reader)
2040                 return NULL;
2041
2042         event = rb_reader_event(cpu_buffer);
2043
2044         switch (event->type) {
2045         case RINGBUF_TYPE_PADDING:
2046                 RB_WARN_ON(cpu_buffer, 1);
2047                 rb_advance_reader(cpu_buffer);
2048                 return NULL;
2049
2050         case RINGBUF_TYPE_TIME_EXTEND:
2051                 /* Internal data, OK to advance */
2052                 rb_advance_reader(cpu_buffer);
2053                 goto again;
2054
2055         case RINGBUF_TYPE_TIME_STAMP:
2056                 /* FIXME: not implemented */
2057                 rb_advance_reader(cpu_buffer);
2058                 goto again;
2059
2060         case RINGBUF_TYPE_DATA:
2061                 if (ts) {
2062                         *ts = cpu_buffer->read_stamp + event->time_delta;
2063                         ring_buffer_normalize_time_stamp(buffer,
2064                                                          cpu_buffer->cpu, ts);
2065                 }
2066                 return event;
2067
2068         default:
2069                 BUG();
2070         }
2071
2072         return NULL;
2073 }
2074 EXPORT_SYMBOL_GPL(ring_buffer_peek);
2075
2076 static struct ring_buffer_event *
2077 rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
2078 {
2079         struct ring_buffer *buffer;
2080         struct ring_buffer_per_cpu *cpu_buffer;
2081         struct ring_buffer_event *event;
2082         int nr_loops = 0;
2083
2084         if (ring_buffer_iter_empty(iter))
2085                 return NULL;
2086
2087         cpu_buffer = iter->cpu_buffer;
2088         buffer = cpu_buffer->buffer;
2089
2090  again:
2091         /*
2092          * We repeat when a timestamp is encountered. It is possible
2093          * to get multiple timestamps from an interrupt entering just
2094          * as one timestamp is about to be written. The max times
2095          * that this can happen is the number of nested interrupts we
2096          * can have. Nesting 10 deep of interrupts is clearly
2097          * an anomaly.
2098          */
2099         if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
2100                 return NULL;
2101
2102         if (rb_per_cpu_empty(cpu_buffer))
2103                 return NULL;
2104
2105         event = rb_iter_head_event(iter);
2106
2107         switch (event->type) {
2108         case RINGBUF_TYPE_PADDING:
2109                 rb_inc_iter(iter);
2110                 goto again;
2111
2112         case RINGBUF_TYPE_TIME_EXTEND:
2113                 /* Internal data, OK to advance */
2114                 rb_advance_iter(iter);
2115                 goto again;
2116
2117         case RINGBUF_TYPE_TIME_STAMP:
2118                 /* FIXME: not implemented */
2119                 rb_advance_iter(iter);
2120                 goto again;
2121
2122         case RINGBUF_TYPE_DATA:
2123                 if (ts) {
2124                         *ts = iter->read_stamp + event->time_delta;
2125                         ring_buffer_normalize_time_stamp(buffer,
2126                                                          cpu_buffer->cpu, ts);
2127                 }
2128                 return event;
2129
2130         default:
2131                 BUG();
2132         }
2133
2134         return NULL;
2135 }
2136 EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
2137
2138 /**
2139  * ring_buffer_peek - peek at the next event to be read
2140  * @buffer: The ring buffer to read
2141  * @cpu: The cpu to peak at
2142  * @ts: The timestamp counter of this event.
2143  *
2144  * This will return the event that will be read next, but does
2145  * not consume the data.
2146  */
2147 struct ring_buffer_event *
2148 ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
2149 {
2150         struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2151         struct ring_buffer_event *event;
2152         unsigned long flags;
2153
2154         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2155                 return NULL;
2156
2157         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2158         event = rb_buffer_peek(buffer, cpu, ts);
2159         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2160
2161         return event;
2162 }
2163
2164 /**
2165  * ring_buffer_iter_peek - peek at the next event to be read
2166  * @iter: The ring buffer iterator
2167  * @ts: The timestamp counter of this event.
2168  *
2169  * This will return the event that will be read next, but does
2170  * not increment the iterator.
2171  */
2172 struct ring_buffer_event *
2173 ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
2174 {
2175         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
2176         struct ring_buffer_event *event;
2177         unsigned long flags;
2178
2179         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2180         event = rb_iter_peek(iter, ts);
2181         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2182
2183         return event;
2184 }
2185
2186 /**
2187  * ring_buffer_consume - return an event and consume it
2188  * @buffer: The ring buffer to get the next event from
2189  *
2190  * Returns the next event in the ring buffer, and that event is consumed.
2191  * Meaning, that sequential reads will keep returning a different event,
2192  * and eventually empty the ring buffer if the producer is slower.
2193  */
2194 struct ring_buffer_event *
2195 ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
2196 {
2197         struct ring_buffer_per_cpu *cpu_buffer;
2198         struct ring_buffer_event *event = NULL;
2199         unsigned long flags;
2200
2201         /* might be called in atomic */
2202         preempt_disable();
2203
2204         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2205                 goto out;
2206
2207         cpu_buffer = buffer->buffers[cpu];
2208         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2209
2210         event = rb_buffer_peek(buffer, cpu, ts);
2211         if (!event)
2212                 goto out_unlock;
2213
2214         rb_advance_reader(cpu_buffer);
2215
2216  out_unlock:
2217         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2218
2219  out:
2220         preempt_enable();
2221
2222         return event;
2223 }
2224 EXPORT_SYMBOL_GPL(ring_buffer_consume);
2225
2226 /**
2227  * ring_buffer_read_start - start a non consuming read of the buffer
2228  * @buffer: The ring buffer to read from
2229  * @cpu: The cpu buffer to iterate over
2230  *
2231  * This starts up an iteration through the buffer. It also disables
2232  * the recording to the buffer until the reading is finished.
2233  * This prevents the reading from being corrupted. This is not
2234  * a consuming read, so a producer is not expected.
2235  *
2236  * Must be paired with ring_buffer_finish.
2237  */
2238 struct ring_buffer_iter *
2239 ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
2240 {
2241         struct ring_buffer_per_cpu *cpu_buffer;
2242         struct ring_buffer_iter *iter;
2243         unsigned long flags;
2244
2245         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2246                 return NULL;
2247
2248         iter = kmalloc(sizeof(*iter), GFP_KERNEL);
2249         if (!iter)
2250                 return NULL;
2251
2252         cpu_buffer = buffer->buffers[cpu];
2253
2254         iter->cpu_buffer = cpu_buffer;
2255
2256         atomic_inc(&cpu_buffer->record_disabled);
2257         synchronize_sched();
2258
2259         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2260         __raw_spin_lock(&cpu_buffer->lock);
2261         rb_iter_reset(iter);
2262         __raw_spin_unlock(&cpu_buffer->lock);
2263         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2264
2265         return iter;
2266 }
2267 EXPORT_SYMBOL_GPL(ring_buffer_read_start);
2268
2269 /**
2270  * ring_buffer_finish - finish reading the iterator of the buffer
2271  * @iter: The iterator retrieved by ring_buffer_start
2272  *
2273  * This re-enables the recording to the buffer, and frees the
2274  * iterator.
2275  */
2276 void
2277 ring_buffer_read_finish(struct ring_buffer_iter *iter)
2278 {
2279         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
2280
2281         atomic_dec(&cpu_buffer->record_disabled);
2282         kfree(iter);
2283 }
2284 EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
2285
2286 /**
2287  * ring_buffer_read - read the next item in the ring buffer by the iterator
2288  * @iter: The ring buffer iterator
2289  * @ts: The time stamp of the event read.
2290  *
2291  * This reads the next event in the ring buffer and increments the iterator.
2292  */
2293 struct ring_buffer_event *
2294 ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
2295 {
2296         struct ring_buffer_event *event;
2297         struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
2298         unsigned long flags;
2299
2300         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2301         event = rb_iter_peek(iter, ts);
2302         if (!event)
2303                 goto out;
2304
2305         rb_advance_iter(iter);
2306  out:
2307         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2308
2309         return event;
2310 }
2311 EXPORT_SYMBOL_GPL(ring_buffer_read);
2312
2313 /**
2314  * ring_buffer_size - return the size of the ring buffer (in bytes)
2315  * @buffer: The ring buffer.
2316  */
2317 unsigned long ring_buffer_size(struct ring_buffer *buffer)
2318 {
2319         return BUF_PAGE_SIZE * buffer->pages;
2320 }
2321 EXPORT_SYMBOL_GPL(ring_buffer_size);
2322
2323 static void
2324 rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
2325 {
2326         cpu_buffer->head_page
2327                 = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
2328         local_set(&cpu_buffer->head_page->write, 0);
2329         local_set(&cpu_buffer->head_page->page->commit, 0);
2330
2331         cpu_buffer->head_page->read = 0;
2332
2333         cpu_buffer->tail_page = cpu_buffer->head_page;
2334         cpu_buffer->commit_page = cpu_buffer->head_page;
2335
2336         INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
2337         local_set(&cpu_buffer->reader_page->write, 0);
2338         local_set(&cpu_buffer->reader_page->page->commit, 0);
2339         cpu_buffer->reader_page->read = 0;
2340
2341         cpu_buffer->overrun = 0;
2342         cpu_buffer->entries = 0;
2343
2344         cpu_buffer->write_stamp = 0;
2345         cpu_buffer->read_stamp = 0;
2346 }
2347
2348 /**
2349  * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
2350  * @buffer: The ring buffer to reset a per cpu buffer of
2351  * @cpu: The CPU buffer to be reset
2352  */
2353 void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
2354 {
2355         struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2356         unsigned long flags;
2357
2358         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2359                 return;
2360
2361         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2362
2363         __raw_spin_lock(&cpu_buffer->lock);
2364
2365         rb_reset_cpu(cpu_buffer);
2366
2367         __raw_spin_unlock(&cpu_buffer->lock);
2368
2369         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2370 }
2371 EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
2372
2373 /**
2374  * ring_buffer_reset - reset a ring buffer
2375  * @buffer: The ring buffer to reset all cpu buffers
2376  */
2377 void ring_buffer_reset(struct ring_buffer *buffer)
2378 {
2379         int cpu;
2380
2381         for_each_buffer_cpu(buffer, cpu)
2382                 ring_buffer_reset_cpu(buffer, cpu);
2383 }
2384 EXPORT_SYMBOL_GPL(ring_buffer_reset);
2385
2386 /**
2387  * rind_buffer_empty - is the ring buffer empty?
2388  * @buffer: The ring buffer to test
2389  */
2390 int ring_buffer_empty(struct ring_buffer *buffer)
2391 {
2392         struct ring_buffer_per_cpu *cpu_buffer;
2393         int cpu;
2394
2395         /* yes this is racy, but if you don't like the race, lock the buffer */
2396         for_each_buffer_cpu(buffer, cpu) {
2397                 cpu_buffer = buffer->buffers[cpu];
2398                 if (!rb_per_cpu_empty(cpu_buffer))
2399                         return 0;
2400         }
2401
2402         return 1;
2403 }
2404 EXPORT_SYMBOL_GPL(ring_buffer_empty);
2405
2406 /**
2407  * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
2408  * @buffer: The ring buffer
2409  * @cpu: The CPU buffer to test
2410  */
2411 int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
2412 {
2413         struct ring_buffer_per_cpu *cpu_buffer;
2414         int ret;
2415
2416         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2417                 return 1;
2418
2419         cpu_buffer = buffer->buffers[cpu];
2420         ret = rb_per_cpu_empty(cpu_buffer);
2421
2422
2423         return ret;
2424 }
2425 EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
2426
2427 /**
2428  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
2429  * @buffer_a: One buffer to swap with
2430  * @buffer_b: The other buffer to swap with
2431  *
2432  * This function is useful for tracers that want to take a "snapshot"
2433  * of a CPU buffer and has another back up buffer lying around.
2434  * it is expected that the tracer handles the cpu buffer not being
2435  * used at the moment.
2436  */
2437 int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2438                          struct ring_buffer *buffer_b, int cpu)
2439 {
2440         struct ring_buffer_per_cpu *cpu_buffer_a;
2441         struct ring_buffer_per_cpu *cpu_buffer_b;
2442         int ret = -EINVAL;
2443
2444         if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
2445             !cpumask_test_cpu(cpu, buffer_b->cpumask))
2446                 goto out;
2447
2448         /* At least make sure the two buffers are somewhat the same */
2449         if (buffer_a->pages != buffer_b->pages)
2450                 goto out;
2451
2452         ret = -EAGAIN;
2453
2454         if (ring_buffer_flags != RB_BUFFERS_ON)
2455                 goto out;
2456
2457         if (atomic_read(&buffer_a->record_disabled))
2458                 goto out;
2459
2460         if (atomic_read(&buffer_b->record_disabled))
2461                 goto out;
2462
2463         cpu_buffer_a = buffer_a->buffers[cpu];
2464         cpu_buffer_b = buffer_b->buffers[cpu];
2465
2466         if (atomic_read(&cpu_buffer_a->record_disabled))
2467                 goto out;
2468
2469         if (atomic_read(&cpu_buffer_b->record_disabled))
2470                 goto out;
2471
2472         /*
2473          * We can't do a synchronize_sched here because this
2474          * function can be called in atomic context.
2475          * Normally this will be called from the same CPU as cpu.
2476          * If not it's up to the caller to protect this.
2477          */
2478         atomic_inc(&cpu_buffer_a->record_disabled);
2479         atomic_inc(&cpu_buffer_b->record_disabled);
2480
2481         buffer_a->buffers[cpu] = cpu_buffer_b;
2482         buffer_b->buffers[cpu] = cpu_buffer_a;
2483
2484         cpu_buffer_b->buffer = buffer_a;
2485         cpu_buffer_a->buffer = buffer_b;
2486
2487         atomic_dec(&cpu_buffer_a->record_disabled);
2488         atomic_dec(&cpu_buffer_b->record_disabled);
2489
2490         ret = 0;
2491 out:
2492         return ret;
2493 }
2494 EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
2495
2496 static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2497                               struct buffer_data_page *bpage,
2498                               unsigned int offset)
2499 {
2500         struct ring_buffer_event *event;
2501         unsigned long head;
2502
2503         __raw_spin_lock(&cpu_buffer->lock);
2504         for (head = offset; head < local_read(&bpage->commit);
2505              head += rb_event_length(event)) {
2506
2507                 event = __rb_data_page_index(bpage, head);
2508                 if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
2509                         return;
2510                 /* Only count data entries */
2511                 if (event->type != RINGBUF_TYPE_DATA)
2512                         continue;
2513                 cpu_buffer->entries--;
2514         }
2515         __raw_spin_unlock(&cpu_buffer->lock);
2516 }
2517
2518 /**
2519  * ring_buffer_alloc_read_page - allocate a page to read from buffer
2520  * @buffer: the buffer to allocate for.
2521  *
2522  * This function is used in conjunction with ring_buffer_read_page.
2523  * When reading a full page from the ring buffer, these functions
2524  * can be used to speed up the process. The calling function should
2525  * allocate a few pages first with this function. Then when it
2526  * needs to get pages from the ring buffer, it passes the result
2527  * of this function into ring_buffer_read_page, which will swap
2528  * the page that was allocated, with the read page of the buffer.
2529  *
2530  * Returns:
2531  *  The page allocated, or NULL on error.
2532  */
2533 void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
2534 {
2535         struct buffer_data_page *bpage;
2536         unsigned long addr;
2537
2538         addr = __get_free_page(GFP_KERNEL);
2539         if (!addr)
2540                 return NULL;
2541
2542         bpage = (void *)addr;
2543
2544         rb_init_page(bpage);
2545
2546         return bpage;
2547 }
2548
2549 /**
2550  * ring_buffer_free_read_page - free an allocated read page
2551  * @buffer: the buffer the page was allocate for
2552  * @data: the page to free
2553  *
2554  * Free a page allocated from ring_buffer_alloc_read_page.
2555  */
2556 void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2557 {
2558         free_page((unsigned long)data);
2559 }
2560
2561 /**
2562  * ring_buffer_read_page - extract a page from the ring buffer
2563  * @buffer: buffer to extract from
2564  * @data_page: the page to use allocated from ring_buffer_alloc_read_page
2565  * @len: amount to extract
2566  * @cpu: the cpu of the buffer to extract
2567  * @full: should the extraction only happen when the page is full.
2568  *
2569  * This function will pull out a page from the ring buffer and consume it.
2570  * @data_page must be the address of the variable that was returned
2571  * from ring_buffer_alloc_read_page. This is because the page might be used
2572  * to swap with a page in the ring buffer.
2573  *
2574  * for example:
2575  *      rpage = ring_buffer_alloc_read_page(buffer);
2576  *      if (!rpage)
2577  *              return error;
2578  *      ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
2579  *      if (ret >= 0)
2580  *              process_page(rpage, ret);
2581  *
2582  * When @full is set, the function will not return true unless
2583  * the writer is off the reader page.
2584  *
2585  * Note: it is up to the calling functions to handle sleeps and wakeups.
2586  *  The ring buffer can be used anywhere in the kernel and can not
2587  *  blindly call wake_up. The layer that uses the ring buffer must be
2588  *  responsible for that.
2589  *
2590  * Returns:
2591  *  >=0 if data has been transferred, returns the offset of consumed data.
2592  *  <0 if no data has been transferred.
2593  */
2594 int ring_buffer_read_page(struct ring_buffer *buffer,
2595                           void **data_page, size_t len, int cpu, int full)
2596 {
2597         struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2598         struct ring_buffer_event *event;
2599         struct buffer_data_page *bpage;
2600         struct buffer_page *reader;
2601         unsigned long flags;
2602         unsigned int commit;
2603         unsigned int read;
2604         u64 save_timestamp;
2605         int ret = -1;
2606
2607         if (!cpumask_test_cpu(cpu, buffer->cpumask))
2608                 goto out;
2609
2610         /*
2611          * If len is not big enough to hold the page header, then
2612          * we can not copy anything.
2613          */
2614         if (len <= BUF_PAGE_HDR_SIZE)
2615                 goto out;
2616
2617         len -= BUF_PAGE_HDR_SIZE;
2618
2619         if (!data_page)
2620                 goto out;
2621
2622         bpage = *data_page;
2623         if (!bpage)
2624                 goto out;
2625
2626         spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2627
2628         reader = rb_get_reader_page(cpu_buffer);
2629         if (!reader)
2630                 goto out_unlock;
2631
2632         event = rb_reader_event(cpu_buffer);
2633
2634         read = reader->read;
2635         commit = rb_page_commit(reader);
2636
2637         /*
2638          * If this page has been partially read or
2639          * if len is not big enough to read the rest of the page or
2640          * a writer is still on the page, then
2641          * we must copy the data from the page to the buffer.
2642          * Otherwise, we can simply swap the page with the one passed in.
2643          */
2644         if (read || (len < (commit - read)) ||
2645             cpu_buffer->reader_page == cpu_buffer->commit_page) {
2646                 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
2647                 unsigned int rpos = read;
2648                 unsigned int pos = 0;
2649                 unsigned int size;
2650
2651                 if (full)
2652                         goto out_unlock;
2653
2654                 if (len > (commit - read))
2655                         len = (commit - read);
2656
2657                 size = rb_event_length(event);
2658
2659                 if (len < size)
2660                         goto out_unlock;
2661
2662                 /* save the current timestamp, since the user will need it */
2663                 save_timestamp = cpu_buffer->read_stamp;
2664
2665                 /* Need to copy one event at a time */
2666                 do {
2667                         memcpy(bpage->data + pos, rpage->data + rpos, size);
2668
2669                         len -= size;
2670
2671                         rb_advance_reader(cpu_buffer);
2672                         rpos = reader->read;
2673                         pos += size;
2674
2675                         event = rb_reader_event(cpu_buffer);
2676                         size = rb_event_length(event);
2677                 } while (len > size);
2678
2679                 /* update bpage */
2680                 local_set(&bpage->commit, pos);
2681                 bpage->time_stamp = save_timestamp;
2682
2683                 /* we copied everything to the beginning */
2684                 read = 0;
2685         } else {
2686                 /* swap the pages */
2687                 rb_init_page(bpage);
2688                 bpage = reader->page;
2689                 reader->page = *data_page;
2690                 local_set(&reader->write, 0);
2691                 reader->read = 0;
2692                 *data_page = bpage;
2693
2694                 /* update the entry counter */
2695                 rb_remove_entries(cpu_buffer, bpage, read);
2696         }
2697         ret = read;
2698
2699  out_unlock:
2700         spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2701
2702  out:
2703         return ret;
2704 }
2705
2706 static ssize_t
2707 rb_simple_read(struct file *filp, char __user *ubuf,
2708                size_t cnt, loff_t *ppos)
2709 {
2710         unsigned long *p = filp->private_data;
2711         char buf[64];
2712         int r;
2713
2714         if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
2715                 r = sprintf(buf, "permanently disabled\n");
2716         else
2717                 r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
2718
2719         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2720 }
2721
2722 static ssize_t
2723 rb_simple_write(struct file *filp, const char __user *ubuf,
2724                 size_t cnt, loff_t *ppos)
2725 {
2726         unsigned long *p = filp->private_data;
2727         char buf[64];
2728         unsigned long val;
2729         int ret;
2730
2731         if (cnt >= sizeof(buf))
2732                 return -EINVAL;
2733
2734         if (copy_from_user(&buf, ubuf, cnt))
2735                 return -EFAULT;
2736
2737         buf[cnt] = 0;
2738
2739         ret = strict_strtoul(buf, 10, &val);
2740         if (ret < 0)
2741                 return ret;
2742
2743         if (val)
2744                 set_bit(RB_BUFFERS_ON_BIT, p);
2745         else
2746                 clear_bit(RB_BUFFERS_ON_BIT, p);
2747
2748         (*ppos)++;
2749
2750         return cnt;
2751 }
2752
2753 static const struct file_operations rb_simple_fops = {
2754         .open           = tracing_open_generic,
2755         .read           = rb_simple_read,
2756         .write          = rb_simple_write,
2757 };
2758
2759
2760 static __init int rb_init_debugfs(void)
2761 {
2762         struct dentry *d_tracer;
2763         struct dentry *entry;
2764
2765         d_tracer = tracing_init_dentry();
2766
2767         entry = debugfs_create_file("tracing_on", 0644, d_tracer,
2768                                     &ring_buffer_flags, &rb_simple_fops);
2769         if (!entry)
2770                 pr_warning("Could not create debugfs 'tracing_on' entry\n");
2771
2772         return 0;
2773 }
2774
2775 fs_initcall(rb_init_debugfs);
2776
2777 #ifdef CONFIG_HOTPLUG_CPU
2778 static int __cpuinit rb_cpu_notify(struct notifier_block *self,
2779                                    unsigned long action, void *hcpu)
2780 {
2781         struct ring_buffer *buffer =
2782                 container_of(self, struct ring_buffer, cpu_notify);
2783         long cpu = (long)hcpu;
2784
2785         switch (action) {
2786         case CPU_UP_PREPARE:
2787         case CPU_UP_PREPARE_FROZEN:
2788                 if (cpu_isset(cpu, *buffer->cpumask))
2789                         return NOTIFY_OK;
2790
2791                 buffer->buffers[cpu] =
2792                         rb_allocate_cpu_buffer(buffer, cpu);
2793                 if (!buffer->buffers[cpu]) {
2794                         WARN(1, "failed to allocate ring buffer on CPU %ld\n",
2795                              cpu);
2796                         return NOTIFY_OK;
2797                 }
2798                 smp_wmb();
2799                 cpu_set(cpu, *buffer->cpumask);
2800                 break;
2801         case CPU_DOWN_PREPARE:
2802         case CPU_DOWN_PREPARE_FROZEN:
2803                 /*
2804                  * Do nothing.
2805                  *  If we were to free the buffer, then the user would
2806                  *  lose any trace that was in the buffer.
2807                  */
2808                 break;
2809         default:
2810                 break;
2811         }
2812         return NOTIFY_OK;
2813 }
2814 #endif