Merge branch 'irq/sparseirq' into cpus4096

[linux-2.6] / arch / x86 / kernel / ftrace.c

/*
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

#include <linux/spinlock.h>
#include <linux/hardirq.h>
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/list.h>

#include <asm/ftrace.h>
#include <linux/ftrace.h>
#include <asm/nops.h>
#include <asm/nmi.h>


#ifdef CONFIG_DYNAMIC_FTRACE

union ftrace_code_union {
        char code[MCOUNT_INSN_SIZE];
        struct {
                char e8;
                int offset;
        } __attribute__((packed));
};

static int ftrace_calc_offset(long ip, long addr)
{
        return (int)(addr - ip);
}

static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
{
        static union ftrace_code_union calc;

        calc.e8         = 0xe8;
        calc.offset     = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);

        /*
         * No locking needed, this must be called via kstop_machine
         * which in essence is like running on a uniprocessor machine.
         */
        return calc.code;
}

/*
 * Modifying code must take extra care. On an SMP machine, if
 * the code being modified is also being executed on another CPU
 * that CPU will have undefined results and possibly take a GPF.
 * We use kstop_machine to stop other CPUS from exectuing code.
 * But this does not stop NMIs from happening. We still need
 * to protect against that. We separate out the modification of
 * the code to take care of this.
 *
 * Two buffers are added: An IP buffer and a "code" buffer.
 *
 * 1) Put the instruction pointer into the IP buffer
 *    and the new code into the "code" buffer.
 * 2) Set a flag that says we are modifying code
 * 3) Wait for any running NMIs to finish.
 * 4) Write the code
 * 5) clear the flag.
 * 6) Wait for any running NMIs to finish.
 *
 * If an NMI is executed, the first thing it does is to call
 * "ftrace_nmi_enter". This will check if the flag is set to write
 * and if it is, it will write what is in the IP and "code" buffers.
 *
 * The trick is, it does not matter if everyone is writing the same
 * content to the code location. Also, if a CPU is executing code
 * it is OK to write to that code location if the contents being written
 * are the same as what exists.
 */

static atomic_t in_nmi = ATOMIC_INIT(0);
static int mod_code_status;             /* holds return value of text write */
static int mod_code_write;              /* set when NMI should do the write */
static void *mod_code_ip;               /* holds the IP to write to */
static void *mod_code_newcode;          /* holds the text to write to the IP */

static unsigned nmi_wait_count;
static atomic_t nmi_update_count = ATOMIC_INIT(0);

int ftrace_arch_read_dyn_info(char *buf, int size)
{
        int r;

        r = snprintf(buf, size, "%u %u",
                     nmi_wait_count,
                     atomic_read(&nmi_update_count));
        return r;
}

static void ftrace_mod_code(void)
{
        /*
         * Yes, more than one CPU process can be writing to mod_code_status.
         *    (and the code itself)
         * But if one were to fail, then they all should, and if one were
         * to succeed, then they all should.
         */
        mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
                                             MCOUNT_INSN_SIZE);
}

void ftrace_nmi_enter(void)
{
        atomic_inc(&in_nmi);
        /* Must have in_nmi seen before reading write flag */
        smp_mb();
        if (mod_code_write) {
                ftrace_mod_code();
                atomic_inc(&nmi_update_count);
        }
}

void ftrace_nmi_exit(void)
{
        /* Finish all executions before clearing in_nmi */
        smp_wmb();
        atomic_dec(&in_nmi);
}

static void wait_for_nmi(void)
{
        int waited = 0;

        while (atomic_read(&in_nmi)) {
                waited = 1;
                cpu_relax();
        }

        if (waited)
                nmi_wait_count++;
}

static int
do_ftrace_mod_code(unsigned long ip, void *new_code)
{
        mod_code_ip = (void *)ip;
        mod_code_newcode = new_code;

        /* The buffers need to be visible before we let NMIs write them */
        smp_wmb();

        mod_code_write = 1;

        /* Make sure write bit is visible before we wait on NMIs */
        smp_mb();

        wait_for_nmi();

        /* Make sure all running NMIs have finished before we write the code */
        smp_mb();

        ftrace_mod_code();

        /* Make sure the write happens before clearing the bit */
        smp_wmb();

        mod_code_write = 0;

        /* make sure NMIs see the cleared bit */
        smp_mb();

        wait_for_nmi();

        return mod_code_status;
}




static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];

static unsigned char *ftrace_nop_replace(void)
{
        return ftrace_nop;
}

static int
ftrace_modify_code(unsigned long ip, unsigned char *old_code,
                   unsigned char *new_code)
{
        unsigned char replaced[MCOUNT_INSN_SIZE];

        /*
         * Note: Due to modules and __init, code can
         *  disappear and change, we need to protect against faulting
         *  as well as code changing. We do this by using the
         *  probe_kernel_* functions.
         *
         * No real locking needed, this code is run through
         * kstop_machine, or before SMP starts.
         */

        /* read the text we want to modify */
        if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
                return -EFAULT;

        /* Make sure it is what we expect it to be */
        if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
                return -EINVAL;

        /* replace the text with the new text */
        if (do_ftrace_mod_code(ip, new_code))
                return -EPERM;

        sync_core();

        return 0;
}

int ftrace_make_nop(struct module *mod,
                    struct dyn_ftrace *rec, unsigned long addr)
{
        unsigned char *new, *old;
        unsigned long ip = rec->ip;

        old = ftrace_call_replace(ip, addr);
        new = ftrace_nop_replace();

        return ftrace_modify_code(rec->ip, old, new);
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
        unsigned char *new, *old;
        unsigned long ip = rec->ip;

        old = ftrace_nop_replace();
        new = ftrace_call_replace(ip, addr);

        return ftrace_modify_code(rec->ip, old, new);
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
        unsigned long ip = (unsigned long)(&ftrace_call);
        unsigned char old[MCOUNT_INSN_SIZE], *new;
        int ret;

        memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
        new = ftrace_call_replace(ip, (unsigned long)func);
        ret = ftrace_modify_code(ip, old, new);

        return ret;
}

int __init ftrace_dyn_arch_init(void *data)
{
        extern const unsigned char ftrace_test_p6nop[];
        extern const unsigned char ftrace_test_nop5[];
        extern const unsigned char ftrace_test_jmp[];
        int faulted = 0;

        /*
         * There is no good nop for all x86 archs.
         * We will default to using the P6_NOP5, but first we
         * will test to make sure that the nop will actually
         * work on this CPU. If it faults, we will then
         * go to a lesser efficient 5 byte nop. If that fails
         * we then just use a jmp as our nop. This isn't the most
         * efficient nop, but we can not use a multi part nop
         * since we would then risk being preempted in the middle
         * of that nop, and if we enabled tracing then, it might
         * cause a system crash.
         *
         * TODO: check the cpuid to determine the best nop.
         */
        asm volatile (
                "ftrace_test_jmp:"
                "jmp ftrace_test_p6nop\n"
                "nop\n"
                "nop\n"
                "nop\n"  /* 2 byte jmp + 3 bytes */
                "ftrace_test_p6nop:"
                P6_NOP5
                "jmp 1f\n"
                "ftrace_test_nop5:"
                ".byte 0x66,0x66,0x66,0x66,0x90\n"
                "1:"
                ".section .fixup, \"ax\"\n"
                "2:     movl $1, %0\n"
                "       jmp ftrace_test_nop5\n"
                "3:     movl $2, %0\n"
                "       jmp 1b\n"
                ".previous\n"
                _ASM_EXTABLE(ftrace_test_p6nop, 2b)
                _ASM_EXTABLE(ftrace_test_nop5, 3b)
                : "=r"(faulted) : "0" (faulted));

        switch (faulted) {
        case 0:
                pr_info("ftrace: converting mcount calls to 0f 1f 44 00 00\n");
                memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
                break;
        case 1:
                pr_info("ftrace: converting mcount calls to 66 66 66 66 90\n");
                memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
                break;
        case 2:
                pr_info("ftrace: converting mcount calls to jmp . + 5\n");
                memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
                break;
        }

        /* The return code is retured via data */
        *(unsigned long *)data = 0;

        return 0;
}
#endif

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);

static int ftrace_mod_jmp(unsigned long ip,
                          int old_offset, int new_offset)
{
        unsigned char code[MCOUNT_INSN_SIZE];

        if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
                return -EFAULT;

        if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
                return -EINVAL;

        *(int *)(&code[1]) = new_offset;

        if (do_ftrace_mod_code(ip, &code))
                return -EPERM;

        return 0;
}

int ftrace_enable_ftrace_graph_caller(void)
{
        unsigned long ip = (unsigned long)(&ftrace_graph_call);
        int old_offset, new_offset;

        old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
        new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);

        return ftrace_mod_jmp(ip, old_offset, new_offset);
}

int ftrace_disable_ftrace_graph_caller(void)
{
        unsigned long ip = (unsigned long)(&ftrace_graph_call);
        int old_offset, new_offset;

        old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
        new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);

        return ftrace_mod_jmp(ip, old_offset, new_offset);
}

#else /* CONFIG_DYNAMIC_FTRACE */

/*
 * These functions are picked from those used on
 * this page for dynamic ftrace. They have been
 * simplified to ignore all traces in NMI context.
 */
static atomic_t in_nmi;

void ftrace_nmi_enter(void)
{
        atomic_inc(&in_nmi);
}

void ftrace_nmi_exit(void)
{
        atomic_dec(&in_nmi);
}

#endif /* !CONFIG_DYNAMIC_FTRACE */

/* Add a function return address to the trace stack on thread info.*/
static int push_return_trace(unsigned long ret, unsigned long long time,
                                unsigned long func, int *depth)
{
        int index;

        if (!current->ret_stack)
                return -EBUSY;

        /* The return trace stack is full */
        if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
                atomic_inc(&current->trace_overrun);
                return -EBUSY;
        }

        index = ++current->curr_ret_stack;
        barrier();
        current->ret_stack[index].ret = ret;
        current->ret_stack[index].func = func;
        current->ret_stack[index].calltime = time;
        *depth = index;

        return 0;
}

/* Retrieve a function return address to the trace stack on thread info.*/
static void pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
{
        int index;

        index = current->curr_ret_stack;

        if (unlikely(index < 0)) {
                ftrace_graph_stop();
                WARN_ON(1);
                /* Might as well panic, otherwise we have no where to go */
                *ret = (unsigned long)panic;
                return;
        }

        *ret = current->ret_stack[index].ret;
        trace->func = current->ret_stack[index].func;
        trace->calltime = current->ret_stack[index].calltime;
        trace->overrun = atomic_read(&current->trace_overrun);
        trace->depth = index;
        barrier();
        current->curr_ret_stack--;

}

/*
 * Send the trace to the ring-buffer.
 * @return the original return address.
 */
unsigned long ftrace_return_to_handler(void)
{
        struct ftrace_graph_ret trace;
        unsigned long ret;

        pop_return_trace(&trace, &ret);
        trace.rettime = cpu_clock(raw_smp_processor_id());
        ftrace_graph_return(&trace);

        if (unlikely(!ret)) {
                ftrace_graph_stop();
                WARN_ON(1);
                /* Might as well panic. What else to do? */
                ret = (unsigned long)panic;
        }

        return ret;
}

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
{
        unsigned long old;
        unsigned long long calltime;
        int faulted;
        struct ftrace_graph_ent trace;
        unsigned long return_hooker = (unsigned long)
                                &return_to_handler;

        /* Nmi's are currently unsupported */
        if (unlikely(atomic_read(&in_nmi)))
                return;

        if (unlikely(atomic_read(&current->tracing_graph_pause)))
                return;

        /*
         * Protect against fault, even if it shouldn't
         * happen. This tool is too much intrusive to
         * ignore such a protection.
         */
        asm volatile(
                "1: " _ASM_MOV " (%[parent_old]), %[old]\n"
                "2: " _ASM_MOV " %[return_hooker], (%[parent_replaced])\n"
                "   movl $0, %[faulted]\n"

                ".section .fixup, \"ax\"\n"
                "3: movl $1, %[faulted]\n"
                ".previous\n"

                _ASM_EXTABLE(1b, 3b)
                _ASM_EXTABLE(2b, 3b)

                : [parent_replaced] "=r" (parent), [old] "=r" (old),
                  [faulted] "=r" (faulted)
                : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker)
                : "memory"
        );

        if (unlikely(faulted)) {
                ftrace_graph_stop();
                WARN_ON(1);
                return;
        }

        if (unlikely(!__kernel_text_address(old))) {
                ftrace_graph_stop();
                *parent = old;
                WARN_ON(1);
                return;
        }

        calltime = cpu_clock(raw_smp_processor_id());

        if (push_return_trace(old, calltime,
                                self_addr, &trace.depth) == -EBUSY) {
                *parent = old;
                return;
        }

        trace.func = self_addr;

        /* Only trace if the calling function expects to */
        if (!ftrace_graph_entry(&trace)) {
                current->curr_ret_stack--;
                *parent = old;
        }
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */