[MIPS] FPU ownership management & preemption fixes

Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

diff --git a/arch/mips/kernel/r2300_switch.S b/arch/mips/kernel/r2300_switch.S
index 656bde2e11b14d37186b5d3ec850b5397a78167d..28c2e2e6af73e7ff65df2c9d56ada233d601aca0 100644 (file)
--- a/arch/mips/kernel/r2300_switch.S
+++ b/arch/mips/kernel/r2300_switch.S
@@ -49,8 +49,7 @@ LEAF(resume)
 #ifndef CONFIG_CPU_HAS_LLSC
        sw      zero, ll_bit
 #endif
-       mfc0    t1, CP0_STATUS
-       sw      t1, THREAD_STATUS(a0)
+       mfc0    t2, CP0_STATUS
        cpu_save_nonscratch a0
        sw      ra, THREAD_REG31(a0)
 
@@ -60,8 +59,8 @@ LEAF(resume)
        lw      t3, TASK_THREAD_INFO(a0)
        lw      t0, TI_FLAGS(t3)
        li      t1, _TIF_USEDFPU
-       and     t2, t0, t1
-       beqz    t2, 1f
+       and     t1, t0
+       beqz    t1, 1f
        nor     t1, zero, t1
 
        and     t0, t0, t1
@@ -74,10 +73,13 @@ LEAF(resume)
        li      t1, ~ST0_CU1
        and     t0, t0, t1
        sw      t0, ST_OFF(t3)
+       /* clear thread_struct CU1 bit */
+       and     t2, t1
 
        fpu_save_single a0, t0                  # clobbers t0
 
 1:
+       sw      t2, THREAD_STATUS(a0)
        /*
         * The order of restoring the registers takes care of the race
         * updating $28, $29 and kernelsp without disabling ints.

diff --git a/arch/mips/kernel/r4k_switch.S b/arch/mips/kernel/r4k_switch.S
index cc566cf122464f8672685766dd595bc8cc50cceb..c7698fd9955c14b121fed998398a4803970aed3c 100644 (file)
--- a/arch/mips/kernel/r4k_switch.S
+++ b/arch/mips/kernel/r4k_switch.S
@@ -48,8 +48,7 @@
 #ifndef CONFIG_CPU_HAS_LLSC
        sw      zero, ll_bit
 #endif
-       mfc0    t1, CP0_STATUS
-       LONG_S  t1, THREAD_STATUS(a0)
+       mfc0    t2, CP0_STATUS
        cpu_save_nonscratch a0
        LONG_S  ra, THREAD_REG31(a0)
 
@@ -59,8 +58,8 @@
        PTR_L   t3, TASK_THREAD_INFO(a0)
        LONG_L  t0, TI_FLAGS(t3)
        li      t1, _TIF_USEDFPU
-       and     t2, t0, t1
-       beqz    t2, 1f
+       and     t1, t0
+       beqz    t1, 1f
        nor     t1, zero, t1
 
        and     t0, t0, t1
@@ -73,10 +72,13 @@
        li      t1, ~ST0_CU1
        and     t0, t0, t1
        LONG_S  t0, ST_OFF(t3)
+       /* clear thread_struct CU1 bit */
+       and     t2, t1
 
        fpu_save_double a0 t0 t1                # c0_status passed in t0
                                                # clobbers t1
 1:
+       LONG_S  t2, THREAD_STATUS(a0)
 
        /*
         * The order of restoring the registers takes care of the race

diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c
index bf094fc4c7ebf64dc891994b9a8de8a8f4485908..8c3c5a5789b0935d9a99334e358663e0e91a69e7 100644 (file)
--- a/arch/mips/kernel/signal.c
+++ b/arch/mips/kernel/signal.c
@@ -82,6 +82,7 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
 {
        int err = 0;
        int i;
+       unsigned int used_math;
 
        err |= __put_user(regs->cp0_epc, &sc->sc_pc);
 
@@ -104,22 +105,18 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
                err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp);
        }
 
-       err |= __put_user(!!used_math(), &sc->sc_used_math);
+       used_math = !!used_math();
+       err |= __put_user(used_math, &sc->sc_used_math);
 
-       if (used_math()) {
+       if (used_math) {
                /*
                 * Save FPU state to signal context. Signal handler
                 * will "inherit" current FPU state.
                 */
-               preempt_disable();
-
-               if (!is_fpu_owner()) {
-                       own_fpu();
-                       restore_fp(current);
-               }
+               own_fpu(1);
+               enable_fp_in_kernel();
                err |= save_fp_context(sc);
-
-               preempt_enable();
+               disable_fp_in_kernel();
        }
        return err;
 }
@@ -188,20 +185,18 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
        err |= __get_user(used_math, &sc->sc_used_math);
        conditional_used_math(used_math);
 
-       preempt_disable();
-
-       if (used_math()) {
+       if (used_math) {
                /* restore fpu context if we have used it before */
-               own_fpu();
+               own_fpu(0);
+               enable_fp_in_kernel();
                if (!err)
                        err = check_and_restore_fp_context(sc);
+               disable_fp_in_kernel();
        } else {
                /* signal handler may have used FPU.  Give it up. */
-               lose_fpu();
+               lose_fpu(0);
        }
 
-       preempt_enable();
-
        return err;
 }
 

diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c
index 20013b6fe7251da0ae0ce048ed2acd14726e0044..151fd2f0893ab7e1c2e2f94f91964d74cdfb1ba7 100644 (file)
--- a/arch/mips/kernel/signal32.c
+++ b/arch/mips/kernel/signal32.c
@@ -181,6 +181,7 @@ static int setup_sigcontext32(struct pt_regs *regs,
 {
        int err = 0;
        int i;
+       u32 used_math;
 
        err |= __put_user(regs->cp0_epc, &sc->sc_pc);
 
@@ -200,22 +201,18 @@ static int setup_sigcontext32(struct pt_regs *regs,
                err |= __put_user(mflo3(), &sc->sc_lo3);
        }
 
-       err |= __put_user(!!used_math(), &sc->sc_used_math);
+       used_math = !!used_math();
+       err |= __put_user(used_math, &sc->sc_used_math);
 
-       if (used_math()) {
+       if (used_math) {
                /*
                 * Save FPU state to signal context.  Signal handler
                 * will "inherit" current FPU state.
                 */
-               preempt_disable();
-
-               if (!is_fpu_owner()) {
-                       own_fpu();
-                       restore_fp(current);
-               }
+               own_fpu(1);
+               enable_fp_in_kernel();
                err |= save_fp_context32(sc);
-
-               preempt_enable();
+               disable_fp_in_kernel();
        }
        return err;
 }
@@ -262,20 +259,18 @@ static int restore_sigcontext32(struct pt_regs *regs,
        err |= __get_user(used_math, &sc->sc_used_math);
        conditional_used_math(used_math);
 
-       preempt_disable();
-
-       if (used_math()) {
+       if (used_math) {
                /* restore fpu context if we have used it before */
-               own_fpu();
+               own_fpu(0);
+               enable_fp_in_kernel();
                if (!err)
                        err = check_and_restore_fp_context32(sc);
+               disable_fp_in_kernel();
        } else {
                /* signal handler may have used FPU.  Give it up. */
-               lose_fpu();
+               lose_fpu(0);
        }
 
-       preempt_enable();
-
        return err;
 }
 

diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 18f56a9dbcfa487831e30d86dff1e4688d006634..7d76a85422b21ca5c250484fda8fc9125d734f5f 100644 (file)
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -610,16 +610,6 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
        if (fcr31 & FPU_CSR_UNI_X) {
                int sig;
 
-               preempt_disable();
-
-#ifdef CONFIG_PREEMPT
-               if (!is_fpu_owner()) {
-                       /* We might lose fpu before disabling preempt... */
-                       own_fpu();
-                       BUG_ON(!used_math());
-                       restore_fp(current);
-               }
-#endif
                /*
                 * Unimplemented operation exception.  If we've got the full
                 * software emulator on-board, let's use it...
@@ -630,18 +620,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
                 * register operands before invoking the emulator, which seems
                 * a bit extreme for what should be an infrequent event.
                 */
-               save_fp(current);
                /* Ensure 'resume' not overwrite saved fp context again. */
-               lose_fpu();
-
-               preempt_enable();
+               lose_fpu(1);
 
                /* Run the emulator */
                sig = fpu_emulator_cop1Handler (regs, &current->thread.fpu, 1);
 
-               preempt_disable();
-
-               own_fpu();      /* Using the FPU again.  */
                /*
                 * We can't allow the emulated instruction to leave any of
                 * the cause bit set in $fcr31.
@@ -649,9 +633,7 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
                current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
 
                /* Restore the hardware register state */
-               restore_fp(current);
-
-               preempt_enable();
+               own_fpu(1);     /* Using the FPU again.  */
 
                /* If something went wrong, signal */
                if (sig)
@@ -775,12 +757,11 @@ asmlinkage void do_cpu(struct pt_regs *regs)
 {
        unsigned int cpid;
 
-       die_if_kernel("do_cpu invoked from kernel context!", regs);
-
        cpid = (regs->cp0_cause >> CAUSEB_CE) & 3;
 
        switch (cpid) {
        case 0:
+               die_if_kernel("do_cpu invoked from kernel context!", regs);
                if (!cpu_has_llsc)
                        if (!simulate_llsc(regs))
                                return;
@@ -791,21 +772,30 @@ asmlinkage void do_cpu(struct pt_regs *regs)
                break;
 
        case 1:
-               preempt_disable();
-
-               own_fpu();
-               if (used_math()) {      /* Using the FPU again.  */
-                       restore_fp(current);
-               } else {                        /* First time FPU user.  */
+               if (!test_thread_flag(TIF_ALLOW_FP_IN_KERNEL))
+                       die_if_kernel("do_cpu invoked from kernel context!",
+                                     regs);
+               if (used_math())        /* Using the FPU again.  */
+                       own_fpu(1);
+               else {                  /* First time FPU user.  */
                        init_fpu();
                        set_used_math();
                }
 
-               if (cpu_has_fpu) {
-                       preempt_enable();
+               if (raw_cpu_has_fpu) {
+                       if (test_thread_flag(TIF_ALLOW_FP_IN_KERNEL)) {
+                               local_irq_disable();
+                               if (cpu_has_fpu)
+                                       regs->cp0_status |= ST0_CU1;
+                               /*
+                                * We must return without enabling
+                                * interrupts to ensure keep FPU
+                                * ownership until resume.
+                                */
+                               return;
+                       }
                } else {
                        int sig;
-                       preempt_enable();
                        sig = fpu_emulator_cop1Handler(regs,
                                                &current->thread.fpu, 0);
                        if (sig)
@@ -1259,26 +1249,26 @@ static inline void mips_srs_init(void)
 /*
  * This is used by native signal handling
  */
-asmlinkage int (*save_fp_context)(struct sigcontext *sc);
-asmlinkage int (*restore_fp_context)(struct sigcontext *sc);
+asmlinkage int (*save_fp_context)(struct sigcontext __user *sc);
+asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc);
 
-extern asmlinkage int _save_fp_context(struct sigcontext *sc);
-extern asmlinkage int _restore_fp_context(struct sigcontext *sc);
+extern asmlinkage int _save_fp_context(struct sigcontext __user *sc);
+extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc);
 
-extern asmlinkage int fpu_emulator_save_context(struct sigcontext *sc);
-extern asmlinkage int fpu_emulator_restore_context(struct sigcontext *sc);
+extern asmlinkage int fpu_emulator_save_context(struct sigcontext __user *sc);
+extern asmlinkage int fpu_emulator_restore_context(struct sigcontext __user *sc);
 
 #ifdef CONFIG_SMP
-static int smp_save_fp_context(struct sigcontext *sc)
+static int smp_save_fp_context(struct sigcontext __user *sc)
 {
-       return cpu_has_fpu
+       return raw_cpu_has_fpu
               ? _save_fp_context(sc)
               : fpu_emulator_save_context(sc);
 }
 
-static int smp_restore_fp_context(struct sigcontext *sc)
+static int smp_restore_fp_context(struct sigcontext __user *sc)
 {
-       return cpu_has_fpu
+       return raw_cpu_has_fpu
               ? _restore_fp_context(sc)
               : fpu_emulator_restore_context(sc);
 }
@@ -1306,14 +1296,14 @@ static inline void signal_init(void)
 /*
  * This is used by 32-bit signal stuff on the 64-bit kernel
  */
-asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc);
-asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc);
+asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc);
+asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc);
 
-extern asmlinkage int _save_fp_context32(struct sigcontext32 *sc);
-extern asmlinkage int _restore_fp_context32(struct sigcontext32 *sc);
+extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc);
+extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc);
 
-extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 *sc);
-extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 *sc);
+extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 __user *sc);
+extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 __user *sc);
 
 static inline void signal32_init(void)
 {

diff --git a/arch/mips/math-emu/kernel_linkage.c b/arch/mips/math-emu/kernel_linkage.c
index 5b3390f64917fda0b53bc0d90be90c201048ae4a..ed49ef01ac533e1a7dbd31291208caab452a0648 100644 (file)
--- a/arch/mips/math-emu/kernel_linkage.c
+++ b/arch/mips/math-emu/kernel_linkage.c
@@ -51,7 +51,7 @@ void fpu_emulator_init_fpu(void)
  * with appropriate macros from uaccess.h
  */
 
-int fpu_emulator_save_context(struct sigcontext *sc)
+int fpu_emulator_save_context(struct sigcontext __user *sc)
 {
        int i;
        int err = 0;
@@ -65,7 +65,7 @@ int fpu_emulator_save_context(struct sigcontext *sc)
        return err;
 }
 
-int fpu_emulator_restore_context(struct sigcontext *sc)
+int fpu_emulator_restore_context(struct sigcontext __user *sc)
 {
        int i;
        int err = 0;
@@ -84,7 +84,7 @@ int fpu_emulator_restore_context(struct sigcontext *sc)
  * This is the o32 version
  */
 
-int fpu_emulator_save_context32(struct sigcontext32 *sc)
+int fpu_emulator_save_context32(struct sigcontext32 __user *sc)
 {
        int i;
        int err = 0;
@@ -98,7 +98,7 @@ int fpu_emulator_save_context32(struct sigcontext32 *sc)
        return err;
 }
 
-int fpu_emulator_restore_context32(struct sigcontext32 *sc)
+int fpu_emulator_restore_context32(struct sigcontext32 __user *sc)
 {
        int i;
        int err = 0;

diff --git a/include/asm-mips/cpu-features.h b/include/asm-mips/cpu-features.h
index eadca266f159bcaae8d1fd5644e4bc191e026b40..5e4bed123b487640f13541f3153e5d1e43960ab1 100644 (file)
--- a/include/asm-mips/cpu-features.h
+++ b/include/asm-mips/cpu-features.h
@@ -40,6 +40,9 @@
 #endif
 #ifndef cpu_has_fpu
 #define cpu_has_fpu            (current_cpu_data.options & MIPS_CPU_FPU)
+#define raw_cpu_has_fpu                (raw_current_cpu_data.options & MIPS_CPU_FPU)
+#else
+#define raw_cpu_has_fpu                cpu_has_fpu
 #endif
 #ifndef cpu_has_32fpr
 #define cpu_has_32fpr          (cpu_data[0].options & MIPS_CPU_32FPR)

diff --git a/include/asm-mips/cpu-info.h b/include/asm-mips/cpu-info.h
index 610d0cdeaa9e164ecdc372f3199d9793429be0c9..22fe8453fcc764427dc1f7f5807d7ee30e817f28 100644 (file)
--- a/include/asm-mips/cpu-info.h
+++ b/include/asm-mips/cpu-info.h
@@ -87,6 +87,7 @@ struct cpuinfo_mips {
 
 extern struct cpuinfo_mips cpu_data[];
 #define current_cpu_data cpu_data[smp_processor_id()]
+#define raw_current_cpu_data cpu_data[raw_smp_processor_id()]
 
 extern void cpu_probe(void);
 extern void cpu_report(void);

diff --git a/include/asm-mips/fpu.h b/include/asm-mips/fpu.h
index efef843b93f0c1694a2084f1a502523a5ac69318..4e12d1f9534f9d99d43123c0cb517887946b5443 100644 (file)
--- a/include/asm-mips/fpu.h
+++ b/include/asm-mips/fpu.h
@@ -27,11 +27,11 @@
 struct sigcontext;
 struct sigcontext32;
 
-extern asmlinkage int (*save_fp_context)(struct sigcontext *sc);
-extern asmlinkage int (*restore_fp_context)(struct sigcontext *sc);
+extern asmlinkage int (*save_fp_context)(struct sigcontext __user *sc);
+extern asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc);
 
-extern asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc);
-extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc);
+extern asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc);
+extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc);
 
 extern void fpu_emulator_init_fpu(void);
 extern void _init_fpu(void);
@@ -68,6 +68,8 @@ do {                                                                  \
        /* We don't care about the c0 hazard here  */                   \
 } while (0)
 
+#define __fpu_enabled()        (read_c0_status() & ST0_CU1)
+
 #define enable_fpu()                                                   \
 do {                                                                   \
        if (cpu_has_fpu)                                                \
@@ -93,31 +95,47 @@ static inline int is_fpu_owner(void)
        return cpu_has_fpu && __is_fpu_owner();
 }
 
-static inline void own_fpu(void)
+static inline void __own_fpu(void)
 {
-       if (cpu_has_fpu) {
-               __enable_fpu();
-               KSTK_STATUS(current) |= ST0_CU1;
-               set_thread_flag(TIF_USEDFPU);
+       __enable_fpu();
+       KSTK_STATUS(current) |= ST0_CU1;
+       set_thread_flag(TIF_USEDFPU);
+}
+
+static inline void own_fpu(int restore)
+{
+       preempt_disable();
+       if (cpu_has_fpu && !__is_fpu_owner()) {
+               __own_fpu();
+               if (restore)
+                       _restore_fp(current);
        }
+       preempt_enable();
 }
 
-static inline void lose_fpu(void)
+static inline void lose_fpu(int save)
 {
-       if (cpu_has_fpu) {
+       preempt_disable();
+       if (is_fpu_owner()) {
+               if (save)
+                       _save_fp(current);
                KSTK_STATUS(current) &= ~ST0_CU1;
                clear_thread_flag(TIF_USEDFPU);
                __disable_fpu();
        }
+       preempt_enable();
 }
 
 static inline void init_fpu(void)
 {
+       preempt_disable();
        if (cpu_has_fpu) {
+               __own_fpu();
                _init_fpu();
        } else {
                fpu_emulator_init_fpu();
        }
+       preempt_enable();
 }
 
 static inline void save_fp(struct task_struct *tsk)
@@ -144,4 +162,18 @@ static inline fpureg_t *get_fpu_regs(struct task_struct *tsk)
        return tsk->thread.fpu.fpr;
 }
 
+static inline void enable_fp_in_kernel(void)
+{
+       set_thread_flag(TIF_ALLOW_FP_IN_KERNEL);
+       /* make sure CU1 and FPU ownership are consistent */
+       if (!__is_fpu_owner() && __fpu_enabled())
+               __disable_fpu();
+}
+
+static inline void disable_fp_in_kernel(void)
+{
+       BUG_ON(!__is_fpu_owner() && __fpu_enabled());
+       clear_thread_flag(TIF_ALLOW_FP_IN_KERNEL);
+}
+
 #endif /* _ASM_FPU_H */

diff --git a/include/asm-mips/thread_info.h b/include/asm-mips/thread_info.h
index fbcda82044738b34674f8630aab3905dc2a7bf44..6cf05f4a4e7e6add8341c8eaaab3a9e36238170d 100644 (file)
--- a/include/asm-mips/thread_info.h
+++ b/include/asm-mips/thread_info.h
@@ -119,6 +119,7 @@ register struct thread_info *__current_thread_info __asm__("$28");
 #define TIF_POLLING_NRFLAG     17      /* true if poll_idle() is polling TIF_NEED_RESCHED */
 #define TIF_MEMDIE             18
 #define TIF_FREEZE             19
+#define TIF_ALLOW_FP_IN_KERNEL 20
 #define TIF_SYSCALL_TRACE      31      /* syscall trace active */
 
 #define _TIF_SYSCALL_TRACE     (1<<TIF_SYSCALL_TRACE)