Merge branch 'upstream-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mfashe...
[linux-2.6] / arch / x86 / mm / mmio-mod.c
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 2 of the License, or
5  * (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software
14  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15  *
16  * Copyright (C) IBM Corporation, 2005
17  *               Jeff Muizelaar, 2006, 2007
18  *               Pekka Paalanen, 2008 <pq@iki.fi>
19  *
20  * Derived from the read-mod example from relay-examples by Tom Zanussi.
21  */
22 #define DEBUG 1
23
24 #include <linux/module.h>
25 #include <linux/debugfs.h>
26 #include <linux/uaccess.h>
27 #include <linux/io.h>
28 #include <linux/version.h>
29 #include <linux/kallsyms.h>
30 #include <asm/pgtable.h>
31 #include <linux/mmiotrace.h>
32 #include <asm/e820.h> /* for ISA_START_ADDRESS */
33 #include <asm/atomic.h>
34 #include <linux/percpu.h>
35 #include <linux/cpu.h>
36
37 #include "pf_in.h"
38
39 #define NAME "mmiotrace: "
40
41 struct trap_reason {
42         unsigned long addr;
43         unsigned long ip;
44         enum reason_type type;
45         int active_traces;
46 };
47
48 struct remap_trace {
49         struct list_head list;
50         struct kmmio_probe probe;
51         resource_size_t phys;
52         unsigned long id;
53 };
54
55 /* Accessed per-cpu. */
56 static DEFINE_PER_CPU(struct trap_reason, pf_reason);
57 static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace);
58
59 #if 0 /* XXX: no way gather this info anymore */
60 /* Access to this is not per-cpu. */
61 static DEFINE_PER_CPU(atomic_t, dropped);
62 #endif
63
64 static struct dentry *marker_file;
65
66 static DEFINE_MUTEX(mmiotrace_mutex);
67 static DEFINE_SPINLOCK(trace_lock);
68 static atomic_t mmiotrace_enabled;
69 static LIST_HEAD(trace_list);           /* struct remap_trace */
70
71 /*
72  * Locking in this file:
73  * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
74  * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
75  *   and trace_lock.
76  * - Routines depending on is_enabled() must take trace_lock.
77  * - trace_list users must hold trace_lock.
78  * - is_enabled() guarantees that mmio_trace_record is allowed.
79  * - pre/post callbacks assume the effect of is_enabled() being true.
80  */
81
82 /* module parameters */
83 static unsigned long    filter_offset;
84 static int              nommiotrace;
85 static int              trace_pc;
86
87 module_param(filter_offset, ulong, 0);
88 module_param(nommiotrace, bool, 0);
89 module_param(trace_pc, bool, 0);
90
91 MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
92 MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
93 MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
94
95 static bool is_enabled(void)
96 {
97         return atomic_read(&mmiotrace_enabled);
98 }
99
100 #if 0 /* XXX: needs rewrite */
101 /*
102  * Write callback for the debugfs entry:
103  * Read a marker and write it to the mmio trace log
104  */
105 static ssize_t write_marker(struct file *file, const char __user *buffer,
106                                                 size_t count, loff_t *ppos)
107 {
108         char *event = NULL;
109         struct mm_io_header *headp;
110         ssize_t len = (count > 65535) ? 65535 : count;
111
112         event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
113         if (!event)
114                 return -ENOMEM;
115
116         headp = (struct mm_io_header *)event;
117         headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
118         headp->data_len = len;
119
120         if (copy_from_user(event + sizeof(*headp), buffer, len)) {
121                 kfree(event);
122                 return -EFAULT;
123         }
124
125         spin_lock_irq(&trace_lock);
126 #if 0 /* XXX: convert this to use tracing */
127         if (is_enabled())
128                 relay_write(chan, event, sizeof(*headp) + len);
129         else
130 #endif
131                 len = -EINVAL;
132         spin_unlock_irq(&trace_lock);
133         kfree(event);
134         return len;
135 }
136 #endif
137
138 static void print_pte(unsigned long address)
139 {
140         unsigned int level;
141         pte_t *pte = lookup_address(address, &level);
142
143         if (!pte) {
144                 pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
145                                                         __func__, address);
146                 return;
147         }
148
149         if (level == PG_LEVEL_2M) {
150                 pr_emerg(NAME "4MB pages are not currently supported: "
151                                                         "0x%08lx\n", address);
152                 BUG();
153         }
154         pr_info(NAME "pte for 0x%lx: 0x%llx 0x%llx\n", address,
155                 (unsigned long long)pte_val(*pte),
156                 (unsigned long long)pte_val(*pte) & _PAGE_PRESENT);
157 }
158
159 /*
160  * For some reason the pre/post pairs have been called in an
161  * unmatched order. Report and die.
162  */
163 static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
164 {
165         const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
166         pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
167                                         "last fault for address: 0x%08lx\n",
168                                         addr, my_reason->addr);
169         print_pte(addr);
170         print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
171         print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
172 #ifdef __i386__
173         pr_emerg("eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
174                         regs->ax, regs->bx, regs->cx, regs->dx);
175         pr_emerg("esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
176                         regs->si, regs->di, regs->bp, regs->sp);
177 #else
178         pr_emerg("rax: %016lx   rcx: %016lx   rdx: %016lx\n",
179                                         regs->ax, regs->cx, regs->dx);
180         pr_emerg("rsi: %016lx   rdi: %016lx   rbp: %016lx   rsp: %016lx\n",
181                                 regs->si, regs->di, regs->bp, regs->sp);
182 #endif
183         put_cpu_var(pf_reason);
184         BUG();
185 }
186
187 static void pre(struct kmmio_probe *p, struct pt_regs *regs,
188                                                 unsigned long addr)
189 {
190         struct trap_reason *my_reason = &get_cpu_var(pf_reason);
191         struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
192         const unsigned long instptr = instruction_pointer(regs);
193         const enum reason_type type = get_ins_type(instptr);
194         struct remap_trace *trace = p->private;
195
196         /* it doesn't make sense to have more than one active trace per cpu */
197         if (my_reason->active_traces)
198                 die_kmmio_nesting_error(regs, addr);
199         else
200                 my_reason->active_traces++;
201
202         my_reason->type = type;
203         my_reason->addr = addr;
204         my_reason->ip = instptr;
205
206         my_trace->phys = addr - trace->probe.addr + trace->phys;
207         my_trace->map_id = trace->id;
208
209         /*
210          * Only record the program counter when requested.
211          * It may taint clean-room reverse engineering.
212          */
213         if (trace_pc)
214                 my_trace->pc = instptr;
215         else
216                 my_trace->pc = 0;
217
218         /*
219          * XXX: the timestamp recorded will be *after* the tracing has been
220          * done, not at the time we hit the instruction. SMP implications
221          * on event ordering?
222          */
223
224         switch (type) {
225         case REG_READ:
226                 my_trace->opcode = MMIO_READ;
227                 my_trace->width = get_ins_mem_width(instptr);
228                 break;
229         case REG_WRITE:
230                 my_trace->opcode = MMIO_WRITE;
231                 my_trace->width = get_ins_mem_width(instptr);
232                 my_trace->value = get_ins_reg_val(instptr, regs);
233                 break;
234         case IMM_WRITE:
235                 my_trace->opcode = MMIO_WRITE;
236                 my_trace->width = get_ins_mem_width(instptr);
237                 my_trace->value = get_ins_imm_val(instptr);
238                 break;
239         default:
240                 {
241                         unsigned char *ip = (unsigned char *)instptr;
242                         my_trace->opcode = MMIO_UNKNOWN_OP;
243                         my_trace->width = 0;
244                         my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
245                                                                 *(ip + 2);
246                 }
247         }
248         put_cpu_var(cpu_trace);
249         put_cpu_var(pf_reason);
250 }
251
252 static void post(struct kmmio_probe *p, unsigned long condition,
253                                                         struct pt_regs *regs)
254 {
255         struct trap_reason *my_reason = &get_cpu_var(pf_reason);
256         struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
257
258         /* this should always return the active_trace count to 0 */
259         my_reason->active_traces--;
260         if (my_reason->active_traces) {
261                 pr_emerg(NAME "unexpected post handler");
262                 BUG();
263         }
264
265         switch (my_reason->type) {
266         case REG_READ:
267                 my_trace->value = get_ins_reg_val(my_reason->ip, regs);
268                 break;
269         default:
270                 break;
271         }
272
273         mmio_trace_rw(my_trace);
274         put_cpu_var(cpu_trace);
275         put_cpu_var(pf_reason);
276 }
277
278 static void ioremap_trace_core(resource_size_t offset, unsigned long size,
279                                                         void __iomem *addr)
280 {
281         static atomic_t next_id;
282         struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
283         /* These are page-unaligned. */
284         struct mmiotrace_map map = {
285                 .phys = offset,
286                 .virt = (unsigned long)addr,
287                 .len = size,
288                 .opcode = MMIO_PROBE
289         };
290
291         if (!trace) {
292                 pr_err(NAME "kmalloc failed in ioremap\n");
293                 return;
294         }
295
296         *trace = (struct remap_trace) {
297                 .probe = {
298                         .addr = (unsigned long)addr,
299                         .len = size,
300                         .pre_handler = pre,
301                         .post_handler = post,
302                         .private = trace
303                 },
304                 .phys = offset,
305                 .id = atomic_inc_return(&next_id)
306         };
307         map.map_id = trace->id;
308
309         spin_lock_irq(&trace_lock);
310         if (!is_enabled())
311                 goto not_enabled;
312
313         mmio_trace_mapping(&map);
314         list_add_tail(&trace->list, &trace_list);
315         if (!nommiotrace)
316                 register_kmmio_probe(&trace->probe);
317
318 not_enabled:
319         spin_unlock_irq(&trace_lock);
320 }
321
322 void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
323                                                 void __iomem *addr)
324 {
325         if (!is_enabled()) /* recheck and proper locking in *_core() */
326                 return;
327
328         pr_debug(NAME "ioremap_*(0x%llx, 0x%lx) = %p\n",
329                                 (unsigned long long)offset, size, addr);
330         if ((filter_offset) && (offset != filter_offset))
331                 return;
332         ioremap_trace_core(offset, size, addr);
333 }
334
335 static void iounmap_trace_core(volatile void __iomem *addr)
336 {
337         struct mmiotrace_map map = {
338                 .phys = 0,
339                 .virt = (unsigned long)addr,
340                 .len = 0,
341                 .opcode = MMIO_UNPROBE
342         };
343         struct remap_trace *trace;
344         struct remap_trace *tmp;
345         struct remap_trace *found_trace = NULL;
346
347         pr_debug(NAME "Unmapping %p.\n", addr);
348
349         spin_lock_irq(&trace_lock);
350         if (!is_enabled())
351                 goto not_enabled;
352
353         list_for_each_entry_safe(trace, tmp, &trace_list, list) {
354                 if ((unsigned long)addr == trace->probe.addr) {
355                         if (!nommiotrace)
356                                 unregister_kmmio_probe(&trace->probe);
357                         list_del(&trace->list);
358                         found_trace = trace;
359                         break;
360                 }
361         }
362         map.map_id = (found_trace) ? found_trace->id : -1;
363         mmio_trace_mapping(&map);
364
365 not_enabled:
366         spin_unlock_irq(&trace_lock);
367         if (found_trace) {
368                 synchronize_rcu(); /* unregister_kmmio_probe() requirement */
369                 kfree(found_trace);
370         }
371 }
372
373 void mmiotrace_iounmap(volatile void __iomem *addr)
374 {
375         might_sleep();
376         if (is_enabled()) /* recheck and proper locking in *_core() */
377                 iounmap_trace_core(addr);
378 }
379
380 static void clear_trace_list(void)
381 {
382         struct remap_trace *trace;
383         struct remap_trace *tmp;
384
385         /*
386          * No locking required, because the caller ensures we are in a
387          * critical section via mutex, and is_enabled() is false,
388          * i.e. nothing can traverse or modify this list.
389          * Caller also ensures is_enabled() cannot change.
390          */
391         list_for_each_entry(trace, &trace_list, list) {
392                 pr_notice(NAME "purging non-iounmapped "
393                                         "trace @0x%08lx, size 0x%lx.\n",
394                                         trace->probe.addr, trace->probe.len);
395                 if (!nommiotrace)
396                         unregister_kmmio_probe(&trace->probe);
397         }
398         synchronize_rcu(); /* unregister_kmmio_probe() requirement */
399
400         list_for_each_entry_safe(trace, tmp, &trace_list, list) {
401                 list_del(&trace->list);
402                 kfree(trace);
403         }
404 }
405
406 #ifdef CONFIG_HOTPLUG_CPU
407 static cpumask_t downed_cpus;
408
409 static void enter_uniprocessor(void)
410 {
411         int cpu;
412         int err;
413
414         get_online_cpus();
415         downed_cpus = cpu_online_map;
416         cpu_clear(first_cpu(cpu_online_map), downed_cpus);
417         if (num_online_cpus() > 1)
418                 pr_notice(NAME "Disabling non-boot CPUs...\n");
419         put_online_cpus();
420
421         for_each_cpu_mask(cpu, downed_cpus) {
422                 err = cpu_down(cpu);
423                 if (!err)
424                         pr_info(NAME "CPU%d is down.\n", cpu);
425                 else
426                         pr_err(NAME "Error taking CPU%d down: %d\n", cpu, err);
427         }
428         if (num_online_cpus() > 1)
429                 pr_warning(NAME "multiple CPUs still online, "
430                                                 "may miss events.\n");
431 }
432
433 static void leave_uniprocessor(void)
434 {
435         int cpu;
436         int err;
437
438         if (cpus_weight(downed_cpus) == 0)
439                 return;
440         pr_notice(NAME "Re-enabling CPUs...\n");
441         for_each_cpu_mask(cpu, downed_cpus) {
442                 err = cpu_up(cpu);
443                 if (!err)
444                         pr_info(NAME "enabled CPU%d.\n", cpu);
445                 else
446                         pr_err(NAME "cannot re-enable CPU%d: %d\n", cpu, err);
447         }
448 }
449
450 #else /* !CONFIG_HOTPLUG_CPU */
451 static void enter_uniprocessor(void)
452 {
453         if (num_online_cpus() > 1)
454                 pr_warning(NAME "multiple CPUs are online, may miss events. "
455                         "Suggest booting with maxcpus=1 kernel argument.\n");
456 }
457
458 static void leave_uniprocessor(void)
459 {
460 }
461 #endif
462
463 #if 0 /* XXX: out of order */
464 static struct file_operations fops_marker = {
465         .owner =        THIS_MODULE,
466         .write =        write_marker
467 };
468 #endif
469
470 void enable_mmiotrace(void)
471 {
472         mutex_lock(&mmiotrace_mutex);
473         if (is_enabled())
474                 goto out;
475
476 #if 0 /* XXX: tracing does not support text entries */
477         marker_file = debugfs_create_file("marker", 0660, dir, NULL,
478                                                                 &fops_marker);
479         if (!marker_file)
480                 pr_err(NAME "marker file creation failed.\n");
481 #endif
482
483         if (nommiotrace)
484                 pr_info(NAME "MMIO tracing disabled.\n");
485         enter_uniprocessor();
486         spin_lock_irq(&trace_lock);
487         atomic_inc(&mmiotrace_enabled);
488         spin_unlock_irq(&trace_lock);
489         pr_info(NAME "enabled.\n");
490 out:
491         mutex_unlock(&mmiotrace_mutex);
492 }
493
494 void disable_mmiotrace(void)
495 {
496         mutex_lock(&mmiotrace_mutex);
497         if (!is_enabled())
498                 goto out;
499
500         spin_lock_irq(&trace_lock);
501         atomic_dec(&mmiotrace_enabled);
502         BUG_ON(is_enabled());
503         spin_unlock_irq(&trace_lock);
504
505         clear_trace_list(); /* guarantees: no more kmmio callbacks */
506         leave_uniprocessor();
507         if (marker_file) {
508                 debugfs_remove(marker_file);
509                 marker_file = NULL;
510         }
511
512         pr_info(NAME "disabled.\n");
513 out:
514         mutex_unlock(&mmiotrace_mutex);
515 }