From 53dc80287da43b75df2fe2658651d3c5160dad8e Mon Sep 17 00:00:00 2001 From: Atsushi Nemoto Date: Sat, 10 Mar 2007 01:07:45 +0900 Subject: [PATCH] [MIPS] FPU ownership management & preemption fixes Signed-off-by: Atsushi Nemoto Signed-off-by: Ralf Baechle --- arch/mips/kernel/r2300_switch.S | 10 ++-- arch/mips/kernel/r4k_switch.S | 10 ++-- arch/mips/kernel/signal.c | 29 +++++----- arch/mips/kernel/signal32.c | 29 +++++----- arch/mips/kernel/traps.c | 84 +++++++++++++---------------- arch/mips/math-emu/kernel_linkage.c | 8 +-- include/asm-mips/cpu-features.h | 3 ++ include/asm-mips/cpu-info.h | 1 + include/asm-mips/fpu.h | 54 +++++++++++++++---- include/asm-mips/thread_info.h | 1 + 10 files changed, 125 insertions(+), 104 deletions(-) diff --git a/arch/mips/kernel/r2300_switch.S b/arch/mips/kernel/r2300_switch.S index 656bde2e11b1..28c2e2e6af73 100644 --- a/arch/mips/kernel/r2300_switch.S +++ b/arch/mips/kernel/r2300_switch.S @@ -49,8 +49,7 @@ LEAF(resume) #ifndef CONFIG_CPU_HAS_LLSC sw zero, ll_bit #endif - mfc0 t1, CP0_STATUS - sw t1, THREAD_STATUS(a0) + mfc0 t2, CP0_STATUS cpu_save_nonscratch a0 sw ra, THREAD_REG31(a0) @@ -60,8 +59,8 @@ LEAF(resume) lw t3, TASK_THREAD_INFO(a0) lw t0, TI_FLAGS(t3) li t1, _TIF_USEDFPU - and t2, t0, t1 - beqz t2, 1f + and t1, t0 + beqz t1, 1f nor t1, zero, t1 and t0, t0, t1 @@ -74,10 +73,13 @@ LEAF(resume) li t1, ~ST0_CU1 and t0, t0, t1 sw t0, ST_OFF(t3) + /* clear thread_struct CU1 bit */ + and t2, t1 fpu_save_single a0, t0 # clobbers t0 1: + sw t2, THREAD_STATUS(a0) /* * The order of restoring the registers takes care of the race * updating $28, $29 and kernelsp without disabling ints. diff --git a/arch/mips/kernel/r4k_switch.S b/arch/mips/kernel/r4k_switch.S index cc566cf12246..c7698fd9955c 100644 --- a/arch/mips/kernel/r4k_switch.S +++ b/arch/mips/kernel/r4k_switch.S @@ -48,8 +48,7 @@ #ifndef CONFIG_CPU_HAS_LLSC sw zero, ll_bit #endif - mfc0 t1, CP0_STATUS - LONG_S t1, THREAD_STATUS(a0) + mfc0 t2, CP0_STATUS cpu_save_nonscratch a0 LONG_S ra, THREAD_REG31(a0) @@ -59,8 +58,8 @@ PTR_L t3, TASK_THREAD_INFO(a0) LONG_L t0, TI_FLAGS(t3) li t1, _TIF_USEDFPU - and t2, t0, t1 - beqz t2, 1f + and t1, t0 + beqz t1, 1f nor t1, zero, t1 and t0, t0, t1 @@ -73,10 +72,13 @@ li t1, ~ST0_CU1 and t0, t0, t1 LONG_S t0, ST_OFF(t3) + /* clear thread_struct CU1 bit */ + and t2, t1 fpu_save_double a0 t0 t1 # c0_status passed in t0 # clobbers t1 1: + LONG_S t2, THREAD_STATUS(a0) /* * The order of restoring the registers takes care of the race diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index bf094fc4c7eb..8c3c5a5789b0 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -82,6 +82,7 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { int err = 0; int i; + unsigned int used_math; err |= __put_user(regs->cp0_epc, &sc->sc_pc); @@ -104,22 +105,18 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); } - err |= __put_user(!!used_math(), &sc->sc_used_math); + used_math = !!used_math(); + err |= __put_user(used_math, &sc->sc_used_math); - if (used_math()) { + if (used_math) { /* * Save FPU state to signal context. Signal handler * will "inherit" current FPU state. */ - preempt_disable(); - - if (!is_fpu_owner()) { - own_fpu(); - restore_fp(current); - } + own_fpu(1); + enable_fp_in_kernel(); err |= save_fp_context(sc); - - preempt_enable(); + disable_fp_in_kernel(); } return err; } @@ -188,20 +185,18 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) err |= __get_user(used_math, &sc->sc_used_math); conditional_used_math(used_math); - preempt_disable(); - - if (used_math()) { + if (used_math) { /* restore fpu context if we have used it before */ - own_fpu(); + own_fpu(0); + enable_fp_in_kernel(); if (!err) err = check_and_restore_fp_context(sc); + disable_fp_in_kernel(); } else { /* signal handler may have used FPU. Give it up. */ - lose_fpu(); + lose_fpu(0); } - preempt_enable(); - return err; } diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c index 20013b6fe725..151fd2f0893a 100644 --- a/arch/mips/kernel/signal32.c +++ b/arch/mips/kernel/signal32.c @@ -181,6 +181,7 @@ static int setup_sigcontext32(struct pt_regs *regs, { int err = 0; int i; + u32 used_math; err |= __put_user(regs->cp0_epc, &sc->sc_pc); @@ -200,22 +201,18 @@ static int setup_sigcontext32(struct pt_regs *regs, err |= __put_user(mflo3(), &sc->sc_lo3); } - err |= __put_user(!!used_math(), &sc->sc_used_math); + used_math = !!used_math(); + err |= __put_user(used_math, &sc->sc_used_math); - if (used_math()) { + if (used_math) { /* * Save FPU state to signal context. Signal handler * will "inherit" current FPU state. */ - preempt_disable(); - - if (!is_fpu_owner()) { - own_fpu(); - restore_fp(current); - } + own_fpu(1); + enable_fp_in_kernel(); err |= save_fp_context32(sc); - - preempt_enable(); + disable_fp_in_kernel(); } return err; } @@ -262,20 +259,18 @@ static int restore_sigcontext32(struct pt_regs *regs, err |= __get_user(used_math, &sc->sc_used_math); conditional_used_math(used_math); - preempt_disable(); - - if (used_math()) { + if (used_math) { /* restore fpu context if we have used it before */ - own_fpu(); + own_fpu(0); + enable_fp_in_kernel(); if (!err) err = check_and_restore_fp_context32(sc); + disable_fp_in_kernel(); } else { /* signal handler may have used FPU. Give it up. */ - lose_fpu(); + lose_fpu(0); } - preempt_enable(); - return err; } diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 18f56a9dbcfa..7d76a85422b2 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -610,16 +610,6 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) if (fcr31 & FPU_CSR_UNI_X) { int sig; - preempt_disable(); - -#ifdef CONFIG_PREEMPT - if (!is_fpu_owner()) { - /* We might lose fpu before disabling preempt... */ - own_fpu(); - BUG_ON(!used_math()); - restore_fp(current); - } -#endif /* * Unimplemented operation exception. If we've got the full * software emulator on-board, let's use it... @@ -630,18 +620,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) * register operands before invoking the emulator, which seems * a bit extreme for what should be an infrequent event. */ - save_fp(current); /* Ensure 'resume' not overwrite saved fp context again. */ - lose_fpu(); - - preempt_enable(); + lose_fpu(1); /* Run the emulator */ sig = fpu_emulator_cop1Handler (regs, ¤t->thread.fpu, 1); - preempt_disable(); - - own_fpu(); /* Using the FPU again. */ /* * We can't allow the emulated instruction to leave any of * the cause bit set in $fcr31. @@ -649,9 +633,7 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; /* Restore the hardware register state */ - restore_fp(current); - - preempt_enable(); + own_fpu(1); /* Using the FPU again. */ /* If something went wrong, signal */ if (sig) @@ -775,12 +757,11 @@ asmlinkage void do_cpu(struct pt_regs *regs) { unsigned int cpid; - die_if_kernel("do_cpu invoked from kernel context!", regs); - cpid = (regs->cp0_cause >> CAUSEB_CE) & 3; switch (cpid) { case 0: + die_if_kernel("do_cpu invoked from kernel context!", regs); if (!cpu_has_llsc) if (!simulate_llsc(regs)) return; @@ -791,21 +772,30 @@ asmlinkage void do_cpu(struct pt_regs *regs) break; case 1: - preempt_disable(); - - own_fpu(); - if (used_math()) { /* Using the FPU again. */ - restore_fp(current); - } else { /* First time FPU user. */ + if (!test_thread_flag(TIF_ALLOW_FP_IN_KERNEL)) + die_if_kernel("do_cpu invoked from kernel context!", + regs); + if (used_math()) /* Using the FPU again. */ + own_fpu(1); + else { /* First time FPU user. */ init_fpu(); set_used_math(); } - if (cpu_has_fpu) { - preempt_enable(); + if (raw_cpu_has_fpu) { + if (test_thread_flag(TIF_ALLOW_FP_IN_KERNEL)) { + local_irq_disable(); + if (cpu_has_fpu) + regs->cp0_status |= ST0_CU1; + /* + * We must return without enabling + * interrupts to ensure keep FPU + * ownership until resume. + */ + return; + } } else { int sig; - preempt_enable(); sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0); if (sig) @@ -1259,26 +1249,26 @@ static inline void mips_srs_init(void) /* * This is used by native signal handling */ -asmlinkage int (*save_fp_context)(struct sigcontext *sc); -asmlinkage int (*restore_fp_context)(struct sigcontext *sc); +asmlinkage int (*save_fp_context)(struct sigcontext __user *sc); +asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc); -extern asmlinkage int _save_fp_context(struct sigcontext *sc); -extern asmlinkage int _restore_fp_context(struct sigcontext *sc); +extern asmlinkage int _save_fp_context(struct sigcontext __user *sc); +extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc); -extern asmlinkage int fpu_emulator_save_context(struct sigcontext *sc); -extern asmlinkage int fpu_emulator_restore_context(struct sigcontext *sc); +extern asmlinkage int fpu_emulator_save_context(struct sigcontext __user *sc); +extern asmlinkage int fpu_emulator_restore_context(struct sigcontext __user *sc); #ifdef CONFIG_SMP -static int smp_save_fp_context(struct sigcontext *sc) +static int smp_save_fp_context(struct sigcontext __user *sc) { - return cpu_has_fpu + return raw_cpu_has_fpu ? _save_fp_context(sc) : fpu_emulator_save_context(sc); } -static int smp_restore_fp_context(struct sigcontext *sc) +static int smp_restore_fp_context(struct sigcontext __user *sc) { - return cpu_has_fpu + return raw_cpu_has_fpu ? _restore_fp_context(sc) : fpu_emulator_restore_context(sc); } @@ -1306,14 +1296,14 @@ static inline void signal_init(void) /* * This is used by 32-bit signal stuff on the 64-bit kernel */ -asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc); -asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc); +asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc); +asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc); -extern asmlinkage int _save_fp_context32(struct sigcontext32 *sc); -extern asmlinkage int _restore_fp_context32(struct sigcontext32 *sc); +extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc); +extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc); -extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 *sc); -extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 *sc); +extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 __user *sc); +extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 __user *sc); static inline void signal32_init(void) { diff --git a/arch/mips/math-emu/kernel_linkage.c b/arch/mips/math-emu/kernel_linkage.c index 5b3390f64917..ed49ef01ac53 100644 --- a/arch/mips/math-emu/kernel_linkage.c +++ b/arch/mips/math-emu/kernel_linkage.c @@ -51,7 +51,7 @@ void fpu_emulator_init_fpu(void) * with appropriate macros from uaccess.h */ -int fpu_emulator_save_context(struct sigcontext *sc) +int fpu_emulator_save_context(struct sigcontext __user *sc) { int i; int err = 0; @@ -65,7 +65,7 @@ int fpu_emulator_save_context(struct sigcontext *sc) return err; } -int fpu_emulator_restore_context(struct sigcontext *sc) +int fpu_emulator_restore_context(struct sigcontext __user *sc) { int i; int err = 0; @@ -84,7 +84,7 @@ int fpu_emulator_restore_context(struct sigcontext *sc) * This is the o32 version */ -int fpu_emulator_save_context32(struct sigcontext32 *sc) +int fpu_emulator_save_context32(struct sigcontext32 __user *sc) { int i; int err = 0; @@ -98,7 +98,7 @@ int fpu_emulator_save_context32(struct sigcontext32 *sc) return err; } -int fpu_emulator_restore_context32(struct sigcontext32 *sc) +int fpu_emulator_restore_context32(struct sigcontext32 __user *sc) { int i; int err = 0; diff --git a/include/asm-mips/cpu-features.h b/include/asm-mips/cpu-features.h index eadca266f159..5e4bed123b48 100644 --- a/include/asm-mips/cpu-features.h +++ b/include/asm-mips/cpu-features.h @@ -40,6 +40,9 @@ #endif #ifndef cpu_has_fpu #define cpu_has_fpu (current_cpu_data.options & MIPS_CPU_FPU) +#define raw_cpu_has_fpu (raw_current_cpu_data.options & MIPS_CPU_FPU) +#else +#define raw_cpu_has_fpu cpu_has_fpu #endif #ifndef cpu_has_32fpr #define cpu_has_32fpr (cpu_data[0].options & MIPS_CPU_32FPR) diff --git a/include/asm-mips/cpu-info.h b/include/asm-mips/cpu-info.h index 610d0cdeaa9e..22fe8453fcc7 100644 --- a/include/asm-mips/cpu-info.h +++ b/include/asm-mips/cpu-info.h @@ -87,6 +87,7 @@ struct cpuinfo_mips { extern struct cpuinfo_mips cpu_data[]; #define current_cpu_data cpu_data[smp_processor_id()] +#define raw_current_cpu_data cpu_data[raw_smp_processor_id()] extern void cpu_probe(void); extern void cpu_report(void); diff --git a/include/asm-mips/fpu.h b/include/asm-mips/fpu.h index efef843b93f0..4e12d1f9534f 100644 --- a/include/asm-mips/fpu.h +++ b/include/asm-mips/fpu.h @@ -27,11 +27,11 @@ struct sigcontext; struct sigcontext32; -extern asmlinkage int (*save_fp_context)(struct sigcontext *sc); -extern asmlinkage int (*restore_fp_context)(struct sigcontext *sc); +extern asmlinkage int (*save_fp_context)(struct sigcontext __user *sc); +extern asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc); -extern asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc); -extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc); +extern asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc); +extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc); extern void fpu_emulator_init_fpu(void); extern void _init_fpu(void); @@ -68,6 +68,8 @@ do { \ /* We don't care about the c0 hazard here */ \ } while (0) +#define __fpu_enabled() (read_c0_status() & ST0_CU1) + #define enable_fpu() \ do { \ if (cpu_has_fpu) \ @@ -93,31 +95,47 @@ static inline int is_fpu_owner(void) return cpu_has_fpu && __is_fpu_owner(); } -static inline void own_fpu(void) +static inline void __own_fpu(void) { - if (cpu_has_fpu) { - __enable_fpu(); - KSTK_STATUS(current) |= ST0_CU1; - set_thread_flag(TIF_USEDFPU); + __enable_fpu(); + KSTK_STATUS(current) |= ST0_CU1; + set_thread_flag(TIF_USEDFPU); +} + +static inline void own_fpu(int restore) +{ + preempt_disable(); + if (cpu_has_fpu && !__is_fpu_owner()) { + __own_fpu(); + if (restore) + _restore_fp(current); } + preempt_enable(); } -static inline void lose_fpu(void) +static inline void lose_fpu(int save) { - if (cpu_has_fpu) { + preempt_disable(); + if (is_fpu_owner()) { + if (save) + _save_fp(current); KSTK_STATUS(current) &= ~ST0_CU1; clear_thread_flag(TIF_USEDFPU); __disable_fpu(); } + preempt_enable(); } static inline void init_fpu(void) { + preempt_disable(); if (cpu_has_fpu) { + __own_fpu(); _init_fpu(); } else { fpu_emulator_init_fpu(); } + preempt_enable(); } static inline void save_fp(struct task_struct *tsk) @@ -144,4 +162,18 @@ static inline fpureg_t *get_fpu_regs(struct task_struct *tsk) return tsk->thread.fpu.fpr; } +static inline void enable_fp_in_kernel(void) +{ + set_thread_flag(TIF_ALLOW_FP_IN_KERNEL); + /* make sure CU1 and FPU ownership are consistent */ + if (!__is_fpu_owner() && __fpu_enabled()) + __disable_fpu(); +} + +static inline void disable_fp_in_kernel(void) +{ + BUG_ON(!__is_fpu_owner() && __fpu_enabled()); + clear_thread_flag(TIF_ALLOW_FP_IN_KERNEL); +} + #endif /* _ASM_FPU_H */ diff --git a/include/asm-mips/thread_info.h b/include/asm-mips/thread_info.h index fbcda8204473..6cf05f4a4e7e 100644 --- a/include/asm-mips/thread_info.h +++ b/include/asm-mips/thread_info.h @@ -119,6 +119,7 @@ register struct thread_info *__current_thread_info __asm__("$28"); #define TIF_POLLING_NRFLAG 17 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_MEMDIE 18 #define TIF_FREEZE 19 +#define TIF_ALLOW_FP_IN_KERNEL 20 #define TIF_SYSCALL_TRACE 31 /* syscall trace active */ #define _TIF_SYSCALL_TRACE (1<