x86: clean up and simplify 64-bit split_large_page()
[linux-2.6] / kernel / profile.c
1 /*
2  *  linux/kernel/profile.c
3  *  Simple profiling. Manages a direct-mapped profile hit count buffer,
4  *  with configurable resolution, support for restricting the cpus on
5  *  which profiling is done, and switching between cpu time and
6  *  schedule() calls via kernel command line parameters passed at boot.
7  *
8  *  Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
9  *      Red Hat, July 2004
10  *  Consolidation of architecture support code for profiling,
11  *      William Irwin, Oracle, July 2004
12  *  Amortized hit count accounting via per-cpu open-addressed hashtables
13  *      to resolve timer interrupt livelocks, William Irwin, Oracle, 2004
14  */
15
16 #include <linux/module.h>
17 #include <linux/profile.h>
18 #include <linux/bootmem.h>
19 #include <linux/notifier.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/cpu.h>
23 #include <linux/profile.h>
24 #include <linux/highmem.h>
25 #include <linux/mutex.h>
26 #include <asm/sections.h>
27 #include <asm/semaphore.h>
28 #include <asm/irq_regs.h>
29 #include <asm/ptrace.h>
30
31 struct profile_hit {
32         u32 pc, hits;
33 };
34 #define PROFILE_GRPSHIFT        3
35 #define PROFILE_GRPSZ           (1 << PROFILE_GRPSHIFT)
36 #define NR_PROFILE_HIT          (PAGE_SIZE/sizeof(struct profile_hit))
37 #define NR_PROFILE_GRP          (NR_PROFILE_HIT/PROFILE_GRPSZ)
38
39 /* Oprofile timer tick hook */
40 static int (*timer_hook)(struct pt_regs *) __read_mostly;
41
42 static atomic_t *prof_buffer;
43 static unsigned long prof_len, prof_shift;
44
45 int prof_on __read_mostly;
46 EXPORT_SYMBOL_GPL(prof_on);
47
48 static cpumask_t prof_cpu_mask = CPU_MASK_ALL;
49 #ifdef CONFIG_SMP
50 static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
51 static DEFINE_PER_CPU(int, cpu_profile_flip);
52 static DEFINE_MUTEX(profile_flip_mutex);
53 #endif /* CONFIG_SMP */
54
55 static int __init profile_setup(char *str)
56 {
57         static char __initdata schedstr[] = "schedule";
58         static char __initdata sleepstr[] = "sleep";
59         static char __initdata kvmstr[] = "kvm";
60         int par;
61
62         if (!strncmp(str, sleepstr, strlen(sleepstr))) {
63 #ifdef CONFIG_SCHEDSTATS
64                 prof_on = SLEEP_PROFILING;
65                 if (str[strlen(sleepstr)] == ',')
66                         str += strlen(sleepstr) + 1;
67                 if (get_option(&str, &par))
68                         prof_shift = par;
69                 printk(KERN_INFO
70                         "kernel sleep profiling enabled (shift: %ld)\n",
71                         prof_shift);
72 #else
73                 printk(KERN_WARNING
74                         "kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
75 #endif /* CONFIG_SCHEDSTATS */
76         } else if (!strncmp(str, schedstr, strlen(schedstr))) {
77                 prof_on = SCHED_PROFILING;
78                 if (str[strlen(schedstr)] == ',')
79                         str += strlen(schedstr) + 1;
80                 if (get_option(&str, &par))
81                         prof_shift = par;
82                 printk(KERN_INFO
83                         "kernel schedule profiling enabled (shift: %ld)\n",
84                         prof_shift);
85         } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
86                 prof_on = KVM_PROFILING;
87                 if (str[strlen(kvmstr)] == ',')
88                         str += strlen(kvmstr) + 1;
89                 if (get_option(&str, &par))
90                         prof_shift = par;
91                 printk(KERN_INFO
92                         "kernel KVM profiling enabled (shift: %ld)\n",
93                         prof_shift);
94         } else if (get_option(&str, &par)) {
95                 prof_shift = par;
96                 prof_on = CPU_PROFILING;
97                 printk(KERN_INFO "kernel profiling enabled (shift: %ld)\n",
98                         prof_shift);
99         }
100         return 1;
101 }
102 __setup("profile=", profile_setup);
103
104
105 void __init profile_init(void)
106 {
107         if (!prof_on)
108                 return;
109
110         /* only text is profiled */
111         prof_len = (_etext - _stext) >> prof_shift;
112         prof_buffer = alloc_bootmem(prof_len*sizeof(atomic_t));
113 }
114
115 /* Profile event notifications */
116
117 #ifdef CONFIG_PROFILING
118
119 static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
120 static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
121 static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
122
123 void profile_task_exit(struct task_struct *task)
124 {
125         blocking_notifier_call_chain(&task_exit_notifier, 0, task);
126 }
127
128 int profile_handoff_task(struct task_struct *task)
129 {
130         int ret;
131         ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
132         return (ret == NOTIFY_OK) ? 1 : 0;
133 }
134
135 void profile_munmap(unsigned long addr)
136 {
137         blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
138 }
139
140 int task_handoff_register(struct notifier_block *n)
141 {
142         return atomic_notifier_chain_register(&task_free_notifier, n);
143 }
144 EXPORT_SYMBOL_GPL(task_handoff_register);
145
146 int task_handoff_unregister(struct notifier_block *n)
147 {
148         return atomic_notifier_chain_unregister(&task_free_notifier, n);
149 }
150 EXPORT_SYMBOL_GPL(task_handoff_unregister);
151
152 int profile_event_register(enum profile_type type, struct notifier_block *n)
153 {
154         int err = -EINVAL;
155
156         switch (type) {
157         case PROFILE_TASK_EXIT:
158                 err = blocking_notifier_chain_register(
159                                 &task_exit_notifier, n);
160                 break;
161         case PROFILE_MUNMAP:
162                 err = blocking_notifier_chain_register(
163                                 &munmap_notifier, n);
164                 break;
165         }
166
167         return err;
168 }
169 EXPORT_SYMBOL_GPL(profile_event_register);
170
171 int profile_event_unregister(enum profile_type type, struct notifier_block *n)
172 {
173         int err = -EINVAL;
174
175         switch (type) {
176         case PROFILE_TASK_EXIT:
177                 err = blocking_notifier_chain_unregister(
178                                 &task_exit_notifier, n);
179                 break;
180         case PROFILE_MUNMAP:
181                 err = blocking_notifier_chain_unregister(
182                                 &munmap_notifier, n);
183                 break;
184         }
185
186         return err;
187 }
188 EXPORT_SYMBOL_GPL(profile_event_unregister);
189
190 int register_timer_hook(int (*hook)(struct pt_regs *))
191 {
192         if (timer_hook)
193                 return -EBUSY;
194         timer_hook = hook;
195         return 0;
196 }
197 EXPORT_SYMBOL_GPL(register_timer_hook);
198
199 void unregister_timer_hook(int (*hook)(struct pt_regs *))
200 {
201         WARN_ON(hook != timer_hook);
202         timer_hook = NULL;
203         /* make sure all CPUs see the NULL hook */
204         synchronize_sched();  /* Allow ongoing interrupts to complete. */
205 }
206 EXPORT_SYMBOL_GPL(unregister_timer_hook);
207
208 #endif /* CONFIG_PROFILING */
209
210
211 #ifdef CONFIG_SMP
212 /*
213  * Each cpu has a pair of open-addressed hashtables for pending
214  * profile hits. read_profile() IPI's all cpus to request them
215  * to flip buffers and flushes their contents to prof_buffer itself.
216  * Flip requests are serialized by the profile_flip_mutex. The sole
217  * use of having a second hashtable is for avoiding cacheline
218  * contention that would otherwise happen during flushes of pending
219  * profile hits required for the accuracy of reported profile hits
220  * and so resurrect the interrupt livelock issue.
221  *
222  * The open-addressed hashtables are indexed by profile buffer slot
223  * and hold the number of pending hits to that profile buffer slot on
224  * a cpu in an entry. When the hashtable overflows, all pending hits
225  * are accounted to their corresponding profile buffer slots with
226  * atomic_add() and the hashtable emptied. As numerous pending hits
227  * may be accounted to a profile buffer slot in a hashtable entry,
228  * this amortizes a number of atomic profile buffer increments likely
229  * to be far larger than the number of entries in the hashtable,
230  * particularly given that the number of distinct profile buffer
231  * positions to which hits are accounted during short intervals (e.g.
232  * several seconds) is usually very small. Exclusion from buffer
233  * flipping is provided by interrupt disablement (note that for
234  * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
235  * process context).
236  * The hash function is meant to be lightweight as opposed to strong,
237  * and was vaguely inspired by ppc64 firmware-supported inverted
238  * pagetable hash functions, but uses a full hashtable full of finite
239  * collision chains, not just pairs of them.
240  *
241  * -- wli
242  */
243 static void __profile_flip_buffers(void *unused)
244 {
245         int cpu = smp_processor_id();
246
247         per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
248 }
249
250 static void profile_flip_buffers(void)
251 {
252         int i, j, cpu;
253
254         mutex_lock(&profile_flip_mutex);
255         j = per_cpu(cpu_profile_flip, get_cpu());
256         put_cpu();
257         on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
258         for_each_online_cpu(cpu) {
259                 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
260                 for (i = 0; i < NR_PROFILE_HIT; ++i) {
261                         if (!hits[i].hits) {
262                                 if (hits[i].pc)
263                                         hits[i].pc = 0;
264                                 continue;
265                         }
266                         atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
267                         hits[i].hits = hits[i].pc = 0;
268                 }
269         }
270         mutex_unlock(&profile_flip_mutex);
271 }
272
273 static void profile_discard_flip_buffers(void)
274 {
275         int i, cpu;
276
277         mutex_lock(&profile_flip_mutex);
278         i = per_cpu(cpu_profile_flip, get_cpu());
279         put_cpu();
280         on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
281         for_each_online_cpu(cpu) {
282                 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
283                 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
284         }
285         mutex_unlock(&profile_flip_mutex);
286 }
287
288 void profile_hits(int type, void *__pc, unsigned int nr_hits)
289 {
290         unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
291         int i, j, cpu;
292         struct profile_hit *hits;
293
294         if (prof_on != type || !prof_buffer)
295                 return;
296         pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
297         i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
298         secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
299         cpu = get_cpu();
300         hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
301         if (!hits) {
302                 put_cpu();
303                 return;
304         }
305         /*
306          * We buffer the global profiler buffer into a per-CPU
307          * queue and thus reduce the number of global (and possibly
308          * NUMA-alien) accesses. The write-queue is self-coalescing:
309          */
310         local_irq_save(flags);
311         do {
312                 for (j = 0; j < PROFILE_GRPSZ; ++j) {
313                         if (hits[i + j].pc == pc) {
314                                 hits[i + j].hits += nr_hits;
315                                 goto out;
316                         } else if (!hits[i + j].hits) {
317                                 hits[i + j].pc = pc;
318                                 hits[i + j].hits = nr_hits;
319                                 goto out;
320                         }
321                 }
322                 i = (i + secondary) & (NR_PROFILE_HIT - 1);
323         } while (i != primary);
324
325         /*
326          * Add the current hit(s) and flush the write-queue out
327          * to the global buffer:
328          */
329         atomic_add(nr_hits, &prof_buffer[pc]);
330         for (i = 0; i < NR_PROFILE_HIT; ++i) {
331                 atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
332                 hits[i].pc = hits[i].hits = 0;
333         }
334 out:
335         local_irq_restore(flags);
336         put_cpu();
337 }
338
339 static int __devinit profile_cpu_callback(struct notifier_block *info,
340                                         unsigned long action, void *__cpu)
341 {
342         int node, cpu = (unsigned long)__cpu;
343         struct page *page;
344
345         switch (action) {
346         case CPU_UP_PREPARE:
347         case CPU_UP_PREPARE_FROZEN:
348                 node = cpu_to_node(cpu);
349                 per_cpu(cpu_profile_flip, cpu) = 0;
350                 if (!per_cpu(cpu_profile_hits, cpu)[1]) {
351                         page = alloc_pages_node(node,
352                                         GFP_KERNEL | __GFP_ZERO,
353                                         0);
354                         if (!page)
355                                 return NOTIFY_BAD;
356                         per_cpu(cpu_profile_hits, cpu)[1] = page_address(page);
357                 }
358                 if (!per_cpu(cpu_profile_hits, cpu)[0]) {
359                         page = alloc_pages_node(node,
360                                         GFP_KERNEL | __GFP_ZERO,
361                                         0);
362                         if (!page)
363                                 goto out_free;
364                         per_cpu(cpu_profile_hits, cpu)[0] = page_address(page);
365                 }
366                 break;
367 out_free:
368                 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
369                 per_cpu(cpu_profile_hits, cpu)[1] = NULL;
370                 __free_page(page);
371                 return NOTIFY_BAD;
372         case CPU_ONLINE:
373         case CPU_ONLINE_FROZEN:
374                 cpu_set(cpu, prof_cpu_mask);
375                 break;
376         case CPU_UP_CANCELED:
377         case CPU_UP_CANCELED_FROZEN:
378         case CPU_DEAD:
379         case CPU_DEAD_FROZEN:
380                 cpu_clear(cpu, prof_cpu_mask);
381                 if (per_cpu(cpu_profile_hits, cpu)[0]) {
382                         page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
383                         per_cpu(cpu_profile_hits, cpu)[0] = NULL;
384                         __free_page(page);
385                 }
386                 if (per_cpu(cpu_profile_hits, cpu)[1]) {
387                         page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
388                         per_cpu(cpu_profile_hits, cpu)[1] = NULL;
389                         __free_page(page);
390                 }
391                 break;
392         }
393         return NOTIFY_OK;
394 }
395 #else /* !CONFIG_SMP */
396 #define profile_flip_buffers()          do { } while (0)
397 #define profile_discard_flip_buffers()  do { } while (0)
398 #define profile_cpu_callback            NULL
399
400 void profile_hits(int type, void *__pc, unsigned int nr_hits)
401 {
402         unsigned long pc;
403
404         if (prof_on != type || !prof_buffer)
405                 return;
406         pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
407         atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
408 }
409 #endif /* !CONFIG_SMP */
410 EXPORT_SYMBOL_GPL(profile_hits);
411
412 void profile_tick(int type)
413 {
414         struct pt_regs *regs = get_irq_regs();
415
416         if (type == CPU_PROFILING && timer_hook)
417                 timer_hook(regs);
418         if (!user_mode(regs) && cpu_isset(smp_processor_id(), prof_cpu_mask))
419                 profile_hit(type, (void *)profile_pc(regs));
420 }
421
422 #ifdef CONFIG_PROC_FS
423 #include <linux/proc_fs.h>
424 #include <asm/uaccess.h>
425 #include <asm/ptrace.h>
426
427 static int prof_cpu_mask_read_proc(char *page, char **start, off_t off,
428                         int count, int *eof, void *data)
429 {
430         int len = cpumask_scnprintf(page, count, *(cpumask_t *)data);
431         if (count - len < 2)
432                 return -EINVAL;
433         len += sprintf(page + len, "\n");
434         return len;
435 }
436
437 static int prof_cpu_mask_write_proc(struct file *file,
438         const char __user *buffer,  unsigned long count, void *data)
439 {
440         cpumask_t *mask = (cpumask_t *)data;
441         unsigned long full_count = count, err;
442         cpumask_t new_value;
443
444         err = cpumask_parse_user(buffer, count, new_value);
445         if (err)
446                 return err;
447
448         *mask = new_value;
449         return full_count;
450 }
451
452 void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir)
453 {
454         struct proc_dir_entry *entry;
455
456         /* create /proc/irq/prof_cpu_mask */
457         entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir);
458         if (!entry)
459                 return;
460         entry->data = (void *)&prof_cpu_mask;
461         entry->read_proc = prof_cpu_mask_read_proc;
462         entry->write_proc = prof_cpu_mask_write_proc;
463 }
464
465 /*
466  * This function accesses profiling information. The returned data is
467  * binary: the sampling step and the actual contents of the profile
468  * buffer. Use of the program readprofile is recommended in order to
469  * get meaningful info out of these data.
470  */
471 static ssize_t
472 read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
473 {
474         unsigned long p = *ppos;
475         ssize_t read;
476         char *pnt;
477         unsigned int sample_step = 1 << prof_shift;
478
479         profile_flip_buffers();
480         if (p >= (prof_len+1)*sizeof(unsigned int))
481                 return 0;
482         if (count > (prof_len+1)*sizeof(unsigned int) - p)
483                 count = (prof_len+1)*sizeof(unsigned int) - p;
484         read = 0;
485
486         while (p < sizeof(unsigned int) && count > 0) {
487                 if (put_user(*((char *)(&sample_step)+p), buf))
488                         return -EFAULT;
489                 buf++; p++; count--; read++;
490         }
491         pnt = (char *)prof_buffer + p - sizeof(atomic_t);
492         if (copy_to_user(buf, (void *)pnt, count))
493                 return -EFAULT;
494         read += count;
495         *ppos += read;
496         return read;
497 }
498
499 /*
500  * Writing to /proc/profile resets the counters
501  *
502  * Writing a 'profiling multiplier' value into it also re-sets the profiling
503  * interrupt frequency, on architectures that support this.
504  */
505 static ssize_t write_profile(struct file *file, const char __user *buf,
506                              size_t count, loff_t *ppos)
507 {
508 #ifdef CONFIG_SMP
509         extern int setup_profiling_timer(unsigned int multiplier);
510
511         if (count == sizeof(int)) {
512                 unsigned int multiplier;
513
514                 if (copy_from_user(&multiplier, buf, sizeof(int)))
515                         return -EFAULT;
516
517                 if (setup_profiling_timer(multiplier))
518                         return -EINVAL;
519         }
520 #endif
521         profile_discard_flip_buffers();
522         memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
523         return count;
524 }
525
526 static const struct file_operations proc_profile_operations = {
527         .read           = read_profile,
528         .write          = write_profile,
529 };
530
531 #ifdef CONFIG_SMP
532 static void __init profile_nop(void *unused)
533 {
534 }
535
536 static int __init create_hash_tables(void)
537 {
538         int cpu;
539
540         for_each_online_cpu(cpu) {
541                 int node = cpu_to_node(cpu);
542                 struct page *page;
543
544                 page = alloc_pages_node(node,
545                                 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
546                                 0);
547                 if (!page)
548                         goto out_cleanup;
549                 per_cpu(cpu_profile_hits, cpu)[1]
550                                 = (struct profile_hit *)page_address(page);
551                 page = alloc_pages_node(node,
552                                 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
553                                 0);
554                 if (!page)
555                         goto out_cleanup;
556                 per_cpu(cpu_profile_hits, cpu)[0]
557                                 = (struct profile_hit *)page_address(page);
558         }
559         return 0;
560 out_cleanup:
561         prof_on = 0;
562         smp_mb();
563         on_each_cpu(profile_nop, NULL, 0, 1);
564         for_each_online_cpu(cpu) {
565                 struct page *page;
566
567                 if (per_cpu(cpu_profile_hits, cpu)[0]) {
568                         page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
569                         per_cpu(cpu_profile_hits, cpu)[0] = NULL;
570                         __free_page(page);
571                 }
572                 if (per_cpu(cpu_profile_hits, cpu)[1]) {
573                         page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
574                         per_cpu(cpu_profile_hits, cpu)[1] = NULL;
575                         __free_page(page);
576                 }
577         }
578         return -1;
579 }
580 #else
581 #define create_hash_tables()                    ({ 0; })
582 #endif
583
584 static int __init create_proc_profile(void)
585 {
586         struct proc_dir_entry *entry;
587
588         if (!prof_on)
589                 return 0;
590         if (create_hash_tables())
591                 return -1;
592         entry = create_proc_entry("profile", S_IWUSR | S_IRUGO, NULL);
593         if (!entry)
594                 return 0;
595         entry->proc_fops = &proc_profile_operations;
596         entry->size = (1+prof_len) * sizeof(atomic_t);
597         hotcpu_notifier(profile_cpu_callback, 0);
598         return 0;
599 }
600 module_init(create_proc_profile);
601 #endif /* CONFIG_PROC_FS */