x86: return probe-booster for x86-64

This patch adds kretprobe-booster to kprobes_64.c.

- Changes are based on x86-32.
- Rewrite register saving/restoring code

Signed-off-by: Masami Hiramatsu <mhiramat@redhat.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/kernel/kprobes_64.c b/arch/x86/kernel/kprobes_64.c
index bf0e18473677c9724a082c9a1dbffa797a433bbf..bc93b1dd9a01a5c4d009b1e2a01402f16649a01b 100644 (file)
--- a/arch/x86/kernel/kprobes_64.c
+++ b/arch/x86/kernel/kprobes_64.c
@@ -28,6 +28,8 @@
  *             Fixed to handle %rip-relative addressing mode correctly.
  * 2005-May     Rusty Lynch <rusty.lynch@intel.com>
  *              Added function return probes functionality
+ * 2007-Dec    Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
+ *             and kretprobe-booster for x86-64
  */
 
 #include <linux/kprobes.h>
@@ -507,21 +509,65 @@ no_kprobe:
 }
 
 /*
- * For function-return probes, init_kprobes() establishes a probepoint
- * here. When a retprobed function returns, this probe is hit and
- * trampoline_probe_handler() runs, calling the kretprobe's handler.
+ * When a retprobed function returns, this code saves registers and
+ * calls trampoline_handler() runs, which calls the kretprobe's handler.
  */
- void kretprobe_trampoline_holder(void)
+ void __kprobes kretprobe_trampoline_holder(void)
  {
        asm volatile (  ".global kretprobe_trampoline\n"
-                       "kretprobe_trampoline: \n"
-                       "nop\n");
+                       "kretprobe_trampoline: \n"
+                       /* We don't bother saving the ss register */
+                       "       pushq %rsp\n"
+                       "       pushfq\n"
+                       /*
+                        * Skip cs, ip, orig_ax.
+                        * trampoline_handler() will plug in these values
+                        */
+                       "       subq $24, %rsp\n"
+                       "       pushq %rdi\n"
+                       "       pushq %rsi\n"
+                       "       pushq %rdx\n"
+                       "       pushq %rcx\n"
+                       "       pushq %rax\n"
+                       "       pushq %r8\n"
+                       "       pushq %r9\n"
+                       "       pushq %r10\n"
+                       "       pushq %r11\n"
+                       "       pushq %rbx\n"
+                       "       pushq %rbp\n"
+                       "       pushq %r12\n"
+                       "       pushq %r13\n"
+                       "       pushq %r14\n"
+                       "       pushq %r15\n"
+                       "       movq %rsp, %rdi\n"
+                       "       call trampoline_handler\n"
+                       /* Replace saved sp with true return address. */
+                       "       movq %rax, 152(%rsp)\n"
+                       "       popq %r15\n"
+                       "       popq %r14\n"
+                       "       popq %r13\n"
+                       "       popq %r12\n"
+                       "       popq %rbp\n"
+                       "       popq %rbx\n"
+                       "       popq %r11\n"
+                       "       popq %r10\n"
+                       "       popq %r9\n"
+                       "       popq %r8\n"
+                       "       popq %rax\n"
+                       "       popq %rcx\n"
+                       "       popq %rdx\n"
+                       "       popq %rsi\n"
+                       "       popq %rdi\n"
+                       /* Skip orig_ax, ip, cs */
+                       "       addq $24, %rsp\n"
+                       "       popfq\n"
+                       "       ret\n");
  }
 
 /*
- * Called when we hit the probe point at kretprobe_trampoline
+ * Called from kretprobe_trampoline
  */
-int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
+fastcall void * __kprobes trampoline_handler(struct pt_regs *regs)
 {
        struct kretprobe_instance *ri = NULL;
        struct hlist_head *head, empty_rp;
@@ -532,6 +578,10 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
        INIT_HLIST_HEAD(&empty_rp);
        spin_lock_irqsave(&kretprobe_lock, flags);
        head = kretprobe_inst_table_head(current);
+       /* fixup rt_regs */
+       regs->cs = __KERNEL_CS;
+       regs->ip = trampoline_address;
+       regs->orig_ax = 0xffffffffffffffff;
 
        /*
         * It is possible to have multiple instances associated with a given
@@ -551,8 +601,12 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
                        /* another task is sharing our hash bucket */
                        continue;
 
-               if (ri->rp && ri->rp->handler)
+               if (ri->rp && ri->rp->handler) {
+                       __get_cpu_var(current_kprobe) = &ri->rp->kp;
+                       get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
                        ri->rp->handler(ri, regs);
+                       __get_cpu_var(current_kprobe) = NULL;
+               }
 
                orig_ret_address = (unsigned long)ri->ret_addr;
                recycle_rp_inst(ri, &empty_rp);
@@ -567,22 +621,14 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
        }
 
        kretprobe_assert(ri, orig_ret_address, trampoline_address);
-       regs->ip = orig_ret_address;
 
-       reset_current_kprobe();
        spin_unlock_irqrestore(&kretprobe_lock, flags);
-       preempt_enable_no_resched();
 
        hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
                hlist_del(&ri->hlist);
                kfree(ri);
        }
-       /*
-        * By returning a non-zero value, we are telling
-        * kprobe_handler() that we don't want the post_handler
-        * to run (and have re-enabled preemption)
-        */
-       return 1;
+       return (void *)orig_ret_address;
 }
 
 /*
@@ -881,20 +927,12 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
        return 0;
 }
 
-static struct kprobe trampoline_p = {
-       .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
-       .pre_handler = trampoline_probe_handler
-};
-
 int __init arch_init_kprobes(void)
 {
-       return register_kprobe(&trampoline_p);
+       return 0;
 }
 
 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
 {
-       if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
-               return 1;
-
        return 0;
 }