[POWERPC] Cell timebase bug workaround

The Cell CPU timebase has an erratum. When reading the entire 64 bits
of the timebase with one mftb instruction, there is a handful of cycles
window during which one might read a value with the low order 32 bits
already reset to 0x00000000 but the high order bits not yet incremeted
by one. This fixes it by reading the timebase again until the low order
32 bits is no longer 0. That might introduce occasional latencies if
hitting mftb just at the wrong time, but no more than 70ns on a cell
blade, and that was considered acceptable.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: Olof Johansson <olof@lixom.net>
Signed-off-by: Paul Mackerras <paulus@samba.org>

diff --git a/arch/powerpc/kernel/vdso64/gettimeofday.S b/arch/powerpc/kernel/vdso64/gettimeofday.S
index 56e76ff5498f49fd2a5b590a8c599c1e606f41cc..40ffd9b6cef7bf55fd81a6c62067afec4175d7c5 100644 (file)
--- a/arch/powerpc/kernel/vdso64/gettimeofday.S
+++ b/arch/powerpc/kernel/vdso64/gettimeofday.S
@@ -229,8 +229,10 @@ V_FUNCTION_BEGIN(__do_get_xsec)
        xor     r0,r8,r8                /* create dependency */
        add     r3,r3,r0
 
-       /* Get TB & offset it */
-       mftb    r7
+       /* Get TB & offset it. We use the MFTB macro which will generate
+        * workaround code for Cell.
+        */
+       MFTB(r7)
        ld      r9,CFG_TB_ORIG_STAMP(r3)
        subf    r7,r9,r7
 

diff --git a/include/asm-powerpc/cputable.h b/include/asm-powerpc/cputable.h
index 02e52d68cbbecffd7abdeaaa0242bdeb3ae575dc..a9a40149a7c0daf66cca975035b4d5863bbc10bc 100644 (file)
--- a/include/asm-powerpc/cputable.h
+++ b/include/asm-powerpc/cputable.h
@@ -147,6 +147,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start,
 #define CPU_FTR_CI_LARGE_PAGE          LONG_ASM_CONST(0x0000100000000000)
 #define CPU_FTR_PAUSE_ZERO             LONG_ASM_CONST(0x0000200000000000)
 #define CPU_FTR_PURR                   LONG_ASM_CONST(0x0000400000000000)
+#define CPU_FTR_CELL_TB_BUG            LONG_ASM_CONST(0x0000800000000000)
 
 #ifndef __ASSEMBLY__
 
@@ -335,7 +336,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start,
 #define CPU_FTRS_CELL  (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \
            CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
            CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
-           CPU_FTR_PAUSE_ZERO | CPU_FTR_CI_LARGE_PAGE)
+           CPU_FTR_PAUSE_ZERO | CPU_FTR_CI_LARGE_PAGE | CPU_FTR_CELL_TB_BUG)
 #define CPU_FTRS_PA6T (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \
            CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | \
            CPU_FTR_ALTIVEC_COMP | CPU_FTR_CI_LARGE_PAGE | \

diff --git a/include/asm-powerpc/ppc_asm.h b/include/asm-powerpc/ppc_asm.h
index a940cfe040da1a4d38b253b09c948bdbdac401a9..fa083d8e46632807b5a206a5ab7fa528e7676fea 100644 (file)
--- a/include/asm-powerpc/ppc_asm.h
+++ b/include/asm-powerpc/ppc_asm.h
@@ -30,9 +30,9 @@ BEGIN_FTR_SECTION;                                                    \
        mfspr   ra,SPRN_PURR;           /* get processor util. reg */   \
 END_FTR_SECTION_IFSET(CPU_FTR_PURR);                                   \
 BEGIN_FTR_SECTION;                                                     \
-       mftb    ra;                     /* or get TB if no PURR */      \
+       MFTB(ra);                       /* or get TB if no PURR */      \
 END_FTR_SECTION_IFCLR(CPU_FTR_PURR);                                   \
-       ld      rb,PACA_STARTPURR(r13);                         \
+       ld      rb,PACA_STARTPURR(r13);                                 \
        std     ra,PACA_STARTPURR(r13);                                 \
        subf    rb,rb,ra;               /* subtract start value */      \
        ld      ra,PACA_USER_TIME(r13);                                 \
@@ -45,9 +45,9 @@ BEGIN_FTR_SECTION;                                                    \
        mfspr   ra,SPRN_PURR;           /* get processor util. reg */   \
 END_FTR_SECTION_IFSET(CPU_FTR_PURR);                                   \
 BEGIN_FTR_SECTION;                                                     \
-       mftb    ra;                     /* or get TB if no PURR */      \
+       MFTB(ra);                       /* or get TB if no PURR */      \
 END_FTR_SECTION_IFCLR(CPU_FTR_PURR);                                   \
-       ld      rb,PACA_STARTPURR(r13);                         \
+       ld      rb,PACA_STARTPURR(r13);                                 \
        std     ra,PACA_STARTPURR(r13);                                 \
        subf    rb,rb,ra;               /* subtract start value */      \
        ld      ra,PACA_SYSTEM_TIME(r13);                               \
@@ -274,6 +274,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_601)
 #define ISYNC_601
 #endif
 
+#ifdef CONFIG_PPC_CELL
+#define MFTB(dest)                     \
+90:    mftb  dest;                     \
+BEGIN_FTR_SECTION_NESTED(96);          \
+       cmpwi dest,0;                   \
+       beq-  90b;                      \
+END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96)
+#else
+#define MFTB(dest)                     mftb dest
+#endif
 
 #ifndef CONFIG_SMP
 #define TLBSYNC

diff --git a/include/asm-powerpc/reg.h b/include/asm-powerpc/reg.h
index fde5c804eccb30384e88bdba176821d4e0750684..6faae7b14d5540f0e5cdcd13b3a31c195d7d980b 100644 (file)
--- a/include/asm-powerpc/reg.h
+++ b/include/asm-powerpc/reg.h
@@ -619,10 +619,35 @@
                                : "=r" (rval)); rval;})
 #define mtspr(rn, v)   asm volatile("mtspr " __stringify(rn) ",%0" : : "r" (v))
 
+#ifdef __powerpc64__
+#ifdef CONFIG_PPC_CELL
+#define mftb()         ({unsigned long rval;                           \
+                       asm volatile(                                   \
+                               "90:    mftb %0;\n"                     \
+                               "97:    cmpwi %0,0;\n"                  \
+                               "       beq- 90b;\n"                    \
+                               "99:\n"                                 \
+                               ".section __ftr_fixup,\"a\"\n"          \
+                               ".align 3\n"                            \
+                               "98:\n"                                 \
+                               "       .llong %1\n"                    \
+                               "       .llong %1\n"                    \
+                               "       .llong 97b-98b\n"               \
+                               "       .llong 99b-98b\n"               \
+                               ".previous"                             \
+                       : "=r" (rval) : "i" (CPU_FTR_CELL_TB_BUG)); rval;})
+#else
 #define mftb()         ({unsigned long rval;   \
                        asm volatile("mftb %0" : "=r" (rval)); rval;})
+#endif /* !CONFIG_PPC_CELL */
+
+#else /* __powerpc64__ */
+
 #define mftbl()                ({unsigned long rval;   \
                        asm volatile("mftbl %0" : "=r" (rval)); rval;})
+#define mftbu()                ({unsigned long rval;   \
+                       asm volatile("mftbu %0" : "=r" (rval)); rval;})
+#endif /* !__powerpc64__ */
 
 #define mttbl(v)       asm volatile("mttbl %0":: "r"(v))
 #define mttbu(v)       asm volatile("mttbu %0":: "r"(v))

diff --git a/include/asm-powerpc/time.h b/include/asm-powerpc/time.h
index b051d4c88c3b46a19b40ba11d2e951010be502d6..a78285010d62f852674327f09a73fda3b7a69a8b 100644 (file)
--- a/include/asm-powerpc/time.h
+++ b/include/asm-powerpc/time.h
@@ -82,30 +82,35 @@ struct div_result {
 #define __USE_RTC()    0
 #endif
 
-/* On ppc64 this gets us the whole timebase; on ppc32 just the lower half */
+#ifdef CONFIG_PPC64
+
+/* For compatibility, get_tbl() is defined as get_tb() on ppc64 */
+#define get_tbl                get_tb
+
+#else
+
 static inline unsigned long get_tbl(void)
 {
-       unsigned long tbl;
-
 #if defined(CONFIG_403GCX)
+       unsigned long tbl;
        asm volatile("mfspr %0, 0x3dd" : "=r" (tbl));
+       return tbl;
 #else
-       asm volatile("mftb %0" : "=r" (tbl));
+       return mftbl();
 #endif
-       return tbl;
 }
 
 static inline unsigned int get_tbu(void)
 {
+#ifdef CONFIG_403GCX
        unsigned int tbu;
-
-#if defined(CONFIG_403GCX)
        asm volatile("mfspr %0, 0x3dc" : "=r" (tbu));
+       return tbu;
 #else
-       asm volatile("mftbu %0" : "=r" (tbu));
+       return mftbu();
 #endif
-       return tbu;
 }
+#endif /* !CONFIG_PPC64 */
 
 static inline unsigned int get_rtcl(void)
 {
@@ -131,7 +136,7 @@ static inline u64 get_tb(void)
 {
        return mftb();
 }
-#else
+#else /* CONFIG_PPC64 */
 static inline u64 get_tb(void)
 {
        unsigned int tbhi, tblo, tbhi2;
@@ -144,7 +149,7 @@ static inline u64 get_tb(void)
 
        return ((u64)tbhi << 32) | tblo;
 }
-#endif
+#endif /* !CONFIG_PPC64 */
 
 static inline void set_tb(unsigned int upper, unsigned int lower)
 {

diff --git a/include/asm-powerpc/timex.h b/include/asm-powerpc/timex.h
index e3f08cf91486afdc7b5e1b8cd33da1de5b3144a3..92dedde761d183a0109dfd7173c37ec1f3a1d43b 100644 (file)
--- a/include/asm-powerpc/timex.h
+++ b/include/asm-powerpc/timex.h
@@ -8,6 +8,7 @@
  */
 
 #include <asm/cputable.h>
+#include <asm/reg.h>
 
 #define CLOCK_TICK_RATE        1024000 /* Underlying HZ */
 
@@ -15,13 +16,11 @@ typedef unsigned long cycles_t;
 
 static inline cycles_t get_cycles(void)
 {
-       cycles_t ret;
-
 #ifdef __powerpc64__
-
-       __asm__ __volatile__("mftb %0" : "=r" (ret) : );
-
+       return mftb();
 #else
+       cycles_t ret;
+
        /*
         * For the "cycle" counter we use the timebase lower half.
         * Currently only used on SMP.
@@ -41,9 +40,8 @@ static inline cycles_t get_cycles(void)
                "       .long 99b-98b\n"
                ".previous"
                : "=r" (ret) : "i" (CPU_FTR_601));
-#endif
-
        return ret;
+#endif
 }
 
 #endif /* __KERNEL__ */