[ARM] 4224/2: allow XIP kernel to boot again

[linux-2.6] / include / asm-mips / atomic.h

diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h
index 13d44e14025ac873ce8921bb7dc25d52ca47997d..8578869a8bcfb4556d3e50a62312b0c5abc3f8ee 100644 (file)
--- a/include/asm-mips/atomic.h
+++ b/include/asm-mips/atomic.h
@@ -9,21 +9,14 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 1996, 97, 99, 2000, 03, 04 by Ralf Baechle
+ * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
  */
-
-/*
- * As workaround for the ATOMIC_DEC_AND_LOCK / atomic_dec_and_lock mess in
- * <linux/spinlock.h> we have to include <linux/spinlock.h> outside the
- * main big wrapper ...
- */
-#include <linux/spinlock.h>
-
 #ifndef _ASM_ATOMIC_H
 #define _ASM_ATOMIC_H
 
+#include <linux/irqflags.h>
+#include <asm/barrier.h>
 #include <asm/cpu-features.h>
-#include <asm/interrupt.h>
 #include <asm/war.h>
 
 typedef struct { volatile int counter; } atomic_t;
@@ -76,7 +69,10 @@ static __inline__ void atomic_add(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_add            \n"
                "       addu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
@@ -118,7 +114,10 @@ static __inline__ void atomic_sub(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_sub            \n"
                "       subu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
@@ -138,6 +137,8 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -148,7 +149,6 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                "       sc      %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -161,9 +161,11 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_add_return     \n"
                "       addu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -178,6 +180,8 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -185,6 +189,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -195,7 +201,6 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                "       sc      %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -208,9 +213,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_sub_return     \n"
                "       subu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -225,6 +232,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -240,6 +249,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -253,7 +264,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                "       beqzl   %0, 1b                                  \n"
                "        subu   %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
@@ -269,11 +279,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                "       bltz    %0, 1f                                  \n"
                "       sc      %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        subu   %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -289,6 +301,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -383,7 +397,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 
 #ifdef CONFIG_64BIT
 
-typedef struct { volatile __s64 counter; } atomic64_t;
+typedef struct { volatile long counter; } atomic64_t;
 
 #define ATOMIC64_INIT(i)    { (i) }
 
@@ -430,7 +444,10 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_add          \n"
                "       addu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
@@ -472,7 +489,10 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_sub          \n"
                "       subu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
@@ -492,6 +512,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -502,7 +524,6 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                "       scd     %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -515,9 +536,11 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_add_return   \n"
                "       addu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -532,6 +555,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -539,6 +564,8 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -549,7 +576,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                "       scd     %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -562,9 +588,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_sub_return   \n"
                "       subu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -579,6 +607,8 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -594,6 +624,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -607,7 +639,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                "       beqzl   %0, 1b                                  \n"
                "        dsubu  %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
@@ -623,11 +654,13 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                "       bltz    %0, 1f                                  \n"
                "       scd     %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        dsubu  %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -643,6 +676,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }