From a866374aecc90c7d90619727ccd851ac096b2fc7 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 6 Dec 2006 20:32:20 -0800 Subject: [PATCH] [PATCH] mm: pagefault_{disable,enable}() Introduce pagefault_{disable,enable}() and use these where previously we did manual preempt increments/decrements to make the pagefault handler do the atomic thing. Currently they still rely on the increased preempt count, but do not rely on the disabled preemption, this might go away in the future. (NOTE: the extra barrier() in pagefault_disable might fix some holes on machines which have too many registers for their own good) [heiko.carstens@de.ibm.com: s390 fix] Signed-off-by: Peter Zijlstra Acked-by: Nick Piggin Cc: Martin Schwidefsky Signed-off-by: Heiko Carstens Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/frv/kernel/futex.c | 4 ++-- arch/i386/mm/highmem.c | 10 ++++------ arch/mips/mm/highmem.c | 10 ++++------ arch/s390/lib/uaccess_std.c | 6 +++--- arch/sparc/mm/highmem.c | 8 +++----- include/asm-frv/highmem.h | 5 ++--- include/asm-generic/futex.h | 4 ++-- include/asm-i386/futex.h | 4 ++-- include/asm-ia64/futex.h | 4 ++-- include/asm-mips/futex.h | 4 ++-- include/asm-parisc/futex.h | 4 ++-- include/asm-powerpc/futex.h | 4 ++-- include/asm-ppc/highmem.h | 8 +++----- include/asm-sparc64/futex.h | 4 ++-- include/asm-x86_64/futex.h | 4 ++-- include/linux/uaccess.h | 39 +++++++++++++++++++++++++++++++++++-- kernel/futex.c | 28 +++++++++++++------------- 17 files changed, 88 insertions(+), 62 deletions(-) diff --git a/arch/frv/kernel/futex.c b/arch/frv/kernel/futex.c index eae874a970..53dc5ed1eb 100644 --- a/arch/frv/kernel/futex.c +++ b/arch/frv/kernel/futex.c @@ -200,7 +200,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -223,7 +223,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) break; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/arch/i386/mm/highmem.c b/arch/i386/mm/highmem.c index f9f647cdbc..178bbfe6cb 100644 --- a/arch/i386/mm/highmem.c +++ b/arch/i386/mm/highmem.c @@ -32,7 +32,7 @@ void *kmap_atomic(struct page *page, enum km_type type) unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ - inc_preempt_count(); + pagefault_disable(); if (!PageHighMem(page)) return page_address(page); @@ -52,8 +52,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) #ifdef CONFIG_DEBUG_HIGHMEM if (vaddr >= PAGE_OFFSET && vaddr < (unsigned long)high_memory) { - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); return; } @@ -68,8 +67,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) */ kpte_clear_flush(kmap_pte-idx, vaddr); - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); } /* This is the same as kmap_atomic() but can map memory that doesn't @@ -80,7 +78,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type) enum fixed_addresses idx; unsigned long vaddr; - inc_preempt_count(); + pagefault_disable(); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c index 99ebf3ccc2..675502ada5 100644 --- a/arch/mips/mm/highmem.c +++ b/arch/mips/mm/highmem.c @@ -39,7 +39,7 @@ void *__kmap_atomic(struct page *page, enum km_type type) unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ - inc_preempt_count(); + pagefault_disable(); if (!PageHighMem(page)) return page_address(page); @@ -62,8 +62,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type) enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < FIXADDR_START) { // FIXME - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); return; } @@ -78,8 +77,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type) local_flush_tlb_one(vaddr); #endif - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); } #ifndef CONFIG_LIMITED_DMA @@ -92,7 +90,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type) enum fixed_addresses idx; unsigned long vaddr; - inc_preempt_count(); + pagefault_disable(); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c index 2d549ed2e1..bbaca66fa2 100644 --- a/arch/s390/lib/uaccess_std.c +++ b/arch/s390/lib/uaccess_std.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include #ifndef __s390x__ @@ -258,7 +258,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old) { int oldval = 0, newval, ret; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -284,7 +284,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old) default: ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); *old = oldval; return ret; } diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 4d8ed9c651..01fc6c2542 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -35,7 +35,7 @@ void *kmap_atomic(struct page *page, enum km_type type) unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ - inc_preempt_count(); + pagefault_disable(); if (!PageHighMem(page)) return page_address(page); @@ -70,8 +70,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) unsigned long idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < FIXADDR_START) { // FIXME - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); return; } @@ -97,8 +96,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) #endif #endif - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); } /* We may be fed a pagetable here by ptep_to_xxx and others. */ diff --git a/include/asm-frv/highmem.h b/include/asm-frv/highmem.h index 0f390f41f8..ff4d6cdeb1 100644 --- a/include/asm-frv/highmem.h +++ b/include/asm-frv/highmem.h @@ -115,7 +115,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type) { unsigned long paddr; - inc_preempt_count(); + pagefault_disable(); paddr = page_to_phys(page); switch (type) { @@ -170,8 +170,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type) default: BUG(); } - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); } #endif /* !__ASSEMBLY__ */ diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h index df893c1603..f422df0956 100644 --- a/include/asm-generic/futex.h +++ b/include/asm-generic/futex.h @@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-i386/futex.h b/include/asm-i386/futex.h index 946d97cfea..438ef0ec71 100644 --- a/include/asm-i386/futex.h +++ b/include/asm-i386/futex.h @@ -56,7 +56,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); if (op == FUTEX_OP_SET) __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg); @@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) } } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-ia64/futex.h b/include/asm-ia64/futex.h index 07d77f3a8c..8a98a26541 100644 --- a/include/asm-ia64/futex.h +++ b/include/asm-ia64/futex.h @@ -59,7 +59,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -83,7 +83,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-mips/futex.h b/include/asm-mips/futex.h index 927a216bd5..47e5679c23 100644 --- a/include/asm-mips/futex.h +++ b/include/asm-mips/futex.h @@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -115,7 +115,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-parisc/futex.h b/include/asm-parisc/futex.h index d84bbb283f..dbee6e60aa 100644 --- a/include/asm-parisc/futex.h +++ b/include/asm-parisc/futex.h @@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-powerpc/futex.h b/include/asm-powerpc/futex.h index 936422e548..3f3673fd3f 100644 --- a/include/asm-powerpc/futex.h +++ b/include/asm-powerpc/futex.h @@ -43,7 +43,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -65,7 +65,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-ppc/highmem.h b/include/asm-ppc/highmem.h index 1d2c4ef81c..f7b21ee302 100644 --- a/include/asm-ppc/highmem.h +++ b/include/asm-ppc/highmem.h @@ -79,7 +79,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type) unsigned long vaddr; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ - inc_preempt_count(); + pagefault_disable(); if (!PageHighMem(page)) return page_address(page); @@ -101,8 +101,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type) unsigned int idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < KMAP_FIX_BEGIN) { // FIXME - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); return; } @@ -115,8 +114,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type) pte_clear(&init_mm, vaddr, kmap_pte+idx); flush_tlb_page(NULL, vaddr); #endif - dec_preempt_count(); - preempt_check_resched(); + pagefault_enable(); } static inline struct page *kmap_atomic_to_page(void *ptr) diff --git a/include/asm-sparc64/futex.h b/include/asm-sparc64/futex.h index 7392fc4a95..876312fe82 100644 --- a/include/asm-sparc64/futex.h +++ b/include/asm-sparc64/futex.h @@ -45,7 +45,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) oparg = 1 << oparg; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -67,7 +67,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/asm-x86_64/futex.h b/include/asm-x86_64/futex.h index 9804bf07b0..5cdfb08013 100644 --- a/include/asm-x86_64/futex.h +++ b/include/asm-x86_64/futex.h @@ -55,7 +55,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; - inc_preempt_count(); + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -78,7 +78,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr) ret = -ENOSYS; } - dec_preempt_count(); + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index a48d7f11c7..67918c2233 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -1,8 +1,43 @@ #ifndef __LINUX_UACCESS_H__ #define __LINUX_UACCESS_H__ +#include #include +/* + * These routines enable/disable the pagefault handler in that + * it will not take any locks and go straight to the fixup table. + * + * They have great resemblance to the preempt_disable/enable calls + * and in fact they are identical; this is because currently there is + * no other way to make the pagefault handlers do this. So we do + * disable preemption but we don't necessarily care about that. + */ +static inline void pagefault_disable(void) +{ + inc_preempt_count(); + /* + * make sure to have issued the store before a pagefault + * can hit. + */ + barrier(); +} + +static inline void pagefault_enable(void) +{ + /* + * make sure to issue those last loads/stores before enabling + * the pagefault handler again. + */ + barrier(); + dec_preempt_count(); + /* + * make sure we do.. + */ + barrier(); + preempt_check_resched(); +} + #ifndef ARCH_HAS_NOCACHE_UACCESS static inline unsigned long __copy_from_user_inatomic_nocache(void *to, @@ -35,9 +70,9 @@ static inline unsigned long __copy_from_user_nocache(void *to, ({ \ long ret; \ \ - inc_preempt_count(); \ + pagefault_disable(); \ ret = __get_user(retval, addr); \ - dec_preempt_count(); \ + pagefault_enable(); \ ret; \ }) diff --git a/kernel/futex.c b/kernel/futex.c index 93ef30ba20..af7b81cbde 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -282,9 +282,9 @@ static inline int get_futex_value_locked(u32 *dest, u32 __user *from) { int ret; - inc_preempt_count(); + pagefault_disable(); ret = __copy_from_user_inatomic(dest, from, sizeof(u32)); - dec_preempt_count(); + pagefault_enable(); return ret ? -EFAULT : 0; } @@ -585,9 +585,9 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) if (!(uval & FUTEX_OWNER_DIED)) { newval = FUTEX_WAITERS | new_owner->pid; - inc_preempt_count(); + pagefault_disable(); curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); - dec_preempt_count(); + pagefault_enable(); if (curval == -EFAULT) return -EFAULT; if (curval != uval) @@ -618,9 +618,9 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval) * There is no waiter, so we unlock the futex. The owner died * bit has not to be preserved here. We are the owner: */ - inc_preempt_count(); + pagefault_disable(); oldval = futex_atomic_cmpxchg_inatomic(uaddr, uval, 0); - dec_preempt_count(); + pagefault_enable(); if (oldval == -EFAULT) return oldval; @@ -1158,9 +1158,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec, */ newval = current->pid; - inc_preempt_count(); + pagefault_disable(); curval = futex_atomic_cmpxchg_inatomic(uaddr, 0, newval); - dec_preempt_count(); + pagefault_enable(); if (unlikely(curval == -EFAULT)) goto uaddr_faulted; @@ -1183,9 +1183,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec, uval = curval; newval = uval | FUTEX_WAITERS; - inc_preempt_count(); + pagefault_disable(); curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); - dec_preempt_count(); + pagefault_enable(); if (unlikely(curval == -EFAULT)) goto uaddr_faulted; @@ -1215,10 +1215,10 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec, newval = current->pid | FUTEX_OWNER_DIED | FUTEX_WAITERS; - inc_preempt_count(); + pagefault_disable(); curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); - dec_preempt_count(); + pagefault_enable(); if (unlikely(curval == -EFAULT)) goto uaddr_faulted; @@ -1390,9 +1390,9 @@ retry_locked: * anyone else up: */ if (!(uval & FUTEX_OWNER_DIED)) { - inc_preempt_count(); + pagefault_disable(); uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0); - dec_preempt_count(); + pagefault_enable(); } if (unlikely(uval == -EFAULT)) -- 2.39.5