Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/shaggy...
[linux-2.6] / drivers / oprofile / cpu_buffer.c
1 /**
2  * @file cpu_buffer.c
3  *
4  * @remark Copyright 2002 OProfile authors
5  * @remark Read the file COPYING
6  *
7  * @author John Levon <levon@movementarian.org>
8  * @author Barry Kasindorf <barry.kasindorf@amd.com>
9  *
10  * Each CPU has a local buffer that stores PC value/event
11  * pairs. We also log context switches when we notice them.
12  * Eventually each CPU's buffer is processed into the global
13  * event buffer by sync_buffer().
14  *
15  * We use a local buffer for two reasons: an NMI or similar
16  * interrupt cannot synchronise, and high sampling rates
17  * would lead to catastrophic global synchronisation if
18  * a global buffer was used.
19  */
20
21 #include <linux/sched.h>
22 #include <linux/oprofile.h>
23 #include <linux/vmalloc.h>
24 #include <linux/errno.h>
25
26 #include "event_buffer.h"
27 #include "cpu_buffer.h"
28 #include "buffer_sync.h"
29 #include "oprof.h"
30
31 #define OP_BUFFER_FLAGS 0
32
33 /*
34  * Read and write access is using spin locking. Thus, writing to the
35  * buffer by NMI handler (x86) could occur also during critical
36  * sections when reading the buffer. To avoid this, there are 2
37  * buffers for independent read and write access. Read access is in
38  * process context only, write access only in the NMI handler. If the
39  * read buffer runs empty, both buffers are swapped atomically. There
40  * is potentially a small window during swapping where the buffers are
41  * disabled and samples could be lost.
42  *
43  * Using 2 buffers is a little bit overhead, but the solution is clear
44  * and does not require changes in the ring buffer implementation. It
45  * can be changed to a single buffer solution when the ring buffer
46  * access is implemented as non-locking atomic code.
47  */
48 struct ring_buffer *op_ring_buffer_read;
49 struct ring_buffer *op_ring_buffer_write;
50 DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer);
51
52 static void wq_sync_buffer(struct work_struct *work);
53
54 #define DEFAULT_TIMER_EXPIRE (HZ / 10)
55 static int work_enabled;
56
57 void free_cpu_buffers(void)
58 {
59         if (op_ring_buffer_read)
60                 ring_buffer_free(op_ring_buffer_read);
61         op_ring_buffer_read = NULL;
62         if (op_ring_buffer_write)
63                 ring_buffer_free(op_ring_buffer_write);
64         op_ring_buffer_write = NULL;
65 }
66
67 unsigned long oprofile_get_cpu_buffer_size(void)
68 {
69         return fs_cpu_buffer_size;
70 }
71
72 void oprofile_cpu_buffer_inc_smpl_lost(void)
73 {
74         struct oprofile_cpu_buffer *cpu_buf
75                 = &__get_cpu_var(cpu_buffer);
76
77         cpu_buf->sample_lost_overflow++;
78 }
79
80 int alloc_cpu_buffers(void)
81 {
82         int i;
83
84         unsigned long buffer_size = fs_cpu_buffer_size;
85
86         op_ring_buffer_read = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS);
87         if (!op_ring_buffer_read)
88                 goto fail;
89         op_ring_buffer_write = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS);
90         if (!op_ring_buffer_write)
91                 goto fail;
92
93         for_each_possible_cpu(i) {
94                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
95
96                 b->last_task = NULL;
97                 b->last_is_kernel = -1;
98                 b->tracing = 0;
99                 b->buffer_size = buffer_size;
100                 b->tail_pos = 0;
101                 b->head_pos = 0;
102                 b->sample_received = 0;
103                 b->sample_lost_overflow = 0;
104                 b->backtrace_aborted = 0;
105                 b->sample_invalid_eip = 0;
106                 b->cpu = i;
107                 INIT_DELAYED_WORK(&b->work, wq_sync_buffer);
108         }
109         return 0;
110
111 fail:
112         free_cpu_buffers();
113         return -ENOMEM;
114 }
115
116 void start_cpu_work(void)
117 {
118         int i;
119
120         work_enabled = 1;
121
122         for_each_online_cpu(i) {
123                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
124
125                 /*
126                  * Spread the work by 1 jiffy per cpu so they dont all
127                  * fire at once.
128                  */
129                 schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i);
130         }
131 }
132
133 void end_cpu_work(void)
134 {
135         int i;
136
137         work_enabled = 0;
138
139         for_each_online_cpu(i) {
140                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
141
142                 cancel_delayed_work(&b->work);
143         }
144
145         flush_scheduled_work();
146 }
147
148 static inline int
149 add_sample(struct oprofile_cpu_buffer *cpu_buf,
150            unsigned long pc, unsigned long event)
151 {
152         struct op_entry entry;
153         int ret;
154
155         ret = cpu_buffer_write_entry(&entry);
156         if (ret)
157                 return ret;
158
159         entry.sample->eip = pc;
160         entry.sample->event = event;
161
162         ret = cpu_buffer_write_commit(&entry);
163         if (ret)
164                 return ret;
165
166         return 0;
167 }
168
169 static inline int
170 add_code(struct oprofile_cpu_buffer *buffer, unsigned long value)
171 {
172         return add_sample(buffer, ESCAPE_CODE, value);
173 }
174
175 /* This must be safe from any context. It's safe writing here
176  * because of the head/tail separation of the writer and reader
177  * of the CPU buffer.
178  *
179  * is_kernel is needed because on some architectures you cannot
180  * tell if you are in kernel or user space simply by looking at
181  * pc. We tag this in the buffer by generating kernel enter/exit
182  * events whenever is_kernel changes
183  */
184 static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
185                       int is_kernel, unsigned long event)
186 {
187         struct task_struct *task;
188
189         cpu_buf->sample_received++;
190
191         if (pc == ESCAPE_CODE) {
192                 cpu_buf->sample_invalid_eip++;
193                 return 0;
194         }
195
196         is_kernel = !!is_kernel;
197
198         task = current;
199
200         /* notice a switch from user->kernel or vice versa */
201         if (cpu_buf->last_is_kernel != is_kernel) {
202                 cpu_buf->last_is_kernel = is_kernel;
203                 if (add_code(cpu_buf, is_kernel))
204                         goto fail;
205         }
206
207         /* notice a task switch */
208         if (cpu_buf->last_task != task) {
209                 cpu_buf->last_task = task;
210                 if (add_code(cpu_buf, (unsigned long)task))
211                         goto fail;
212         }
213
214         if (add_sample(cpu_buf, pc, event))
215                 goto fail;
216
217         return 1;
218
219 fail:
220         cpu_buf->sample_lost_overflow++;
221         return 0;
222 }
223
224 static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
225 {
226         add_code(cpu_buf, CPU_TRACE_BEGIN);
227         cpu_buf->tracing = 1;
228         return 1;
229 }
230
231 static void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
232 {
233         cpu_buf->tracing = 0;
234 }
235
236 void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
237                                 unsigned long event, int is_kernel)
238 {
239         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
240
241         if (!backtrace_depth) {
242                 log_sample(cpu_buf, pc, is_kernel, event);
243                 return;
244         }
245
246         if (!oprofile_begin_trace(cpu_buf))
247                 return;
248
249         /*
250          * if log_sample() fail we can't backtrace since we lost the
251          * source of this event
252          */
253         if (log_sample(cpu_buf, pc, is_kernel, event))
254                 oprofile_ops.backtrace(regs, backtrace_depth);
255         oprofile_end_trace(cpu_buf);
256 }
257
258 void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
259 {
260         int is_kernel = !user_mode(regs);
261         unsigned long pc = profile_pc(regs);
262
263         oprofile_add_ext_sample(pc, regs, event, is_kernel);
264 }
265
266 #ifdef CONFIG_OPROFILE_IBS
267
268 #define MAX_IBS_SAMPLE_SIZE 14
269
270 void oprofile_add_ibs_sample(struct pt_regs * const regs,
271                              unsigned int * const ibs_sample, int ibs_code)
272 {
273         int is_kernel = !user_mode(regs);
274         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
275         struct task_struct *task;
276         int fail = 0;
277
278         cpu_buf->sample_received++;
279
280         /* notice a switch from user->kernel or vice versa */
281         if (cpu_buf->last_is_kernel != is_kernel) {
282                 if (add_code(cpu_buf, is_kernel))
283                         goto fail;
284                 cpu_buf->last_is_kernel = is_kernel;
285         }
286
287         /* notice a task switch */
288         if (!is_kernel) {
289                 task = current;
290                 if (cpu_buf->last_task != task) {
291                         if (add_code(cpu_buf, (unsigned long)task))
292                                 goto fail;
293                         cpu_buf->last_task = task;
294                 }
295         }
296
297         fail = fail || add_code(cpu_buf, ibs_code);
298         fail = fail || add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]);
299         fail = fail || add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]);
300         fail = fail || add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]);
301
302         if (ibs_code == IBS_OP_BEGIN) {
303                 fail = fail || add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]);
304                 fail = fail || add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]);
305                 fail = fail || add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]);
306         }
307
308         if (fail)
309                 goto fail;
310
311         if (backtrace_depth)
312                 oprofile_ops.backtrace(regs, backtrace_depth);
313
314         return;
315
316 fail:
317         cpu_buf->sample_lost_overflow++;
318         return;
319 }
320
321 #endif
322
323 void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
324 {
325         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
326         log_sample(cpu_buf, pc, is_kernel, event);
327 }
328
329 void oprofile_add_trace(unsigned long pc)
330 {
331         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
332
333         if (!cpu_buf->tracing)
334                 return;
335
336         /*
337          * broken frame can give an eip with the same value as an
338          * escape code, abort the trace if we get it
339          */
340         if (pc == ESCAPE_CODE)
341                 goto fail;
342
343         if (add_sample(cpu_buf, pc, 0))
344                 goto fail;
345
346         return;
347 fail:
348         cpu_buf->tracing = 0;
349         cpu_buf->backtrace_aborted++;
350         return;
351 }
352
353 /*
354  * This serves to avoid cpu buffer overflow, and makes sure
355  * the task mortuary progresses
356  *
357  * By using schedule_delayed_work_on and then schedule_delayed_work
358  * we guarantee this will stay on the correct cpu
359  */
360 static void wq_sync_buffer(struct work_struct *work)
361 {
362         struct oprofile_cpu_buffer *b =
363                 container_of(work, struct oprofile_cpu_buffer, work.work);
364         if (b->cpu != smp_processor_id()) {
365                 printk(KERN_DEBUG "WQ on CPU%d, prefer CPU%d\n",
366                        smp_processor_id(), b->cpu);
367
368                 if (!cpu_online(b->cpu)) {
369                         cancel_delayed_work(&b->work);
370                         return;
371                 }
372         }
373         sync_buffer(b->cpu);
374
375         /* don't re-add the work if we're shutting down */
376         if (work_enabled)
377                 schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE);
378 }