[SPARC64]: Implement atomic backoff.

When the cpu count is high and contention hits an atomic object, the
processors can synchronize such that some cpus continually get knocked
out and cannot complete the atomic update.

So implement an exponential backoff when SMP.

Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
index 9633750167d06b3e9f4d285e105656d01a8ce32f..70ac4186f62b7afc4af36eeb5472d6db868688d8 100644 (file)
--- a/arch/sparc64/lib/atomic.S
+++ b/arch/sparc64/lib/atomic.S
@@ -1,10 +1,10 @@
-/* $Id: atomic.S,v 1.4 2001/11/18 00:12:56 davem Exp $
- * atomic.S: These things are too big to do inline.
+/* atomic.S: These things are too big to do inline.
  *
- * Copyright (C) 1999 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net)
  */
 
 #include <asm/asi.h>
+#include <asm/backoff.h>
 
        .text
 
@@ -16,27 +16,31 @@
        .globl  atomic_add
        .type   atomic_add,#function
 atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
 1:     lduw    [%o1], %g1
        add     %g1, %o0, %g7
        cas     [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %icc, 1b
+       bne,pn  %icc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic_add, .-atomic_add
 
        .globl  atomic_sub
        .type   atomic_sub,#function
 atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
 1:     lduw    [%o1], %g1
        sub     %g1, %o0, %g7
        cas     [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %icc, 1b
+       bne,pn  %icc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic_sub, .-atomic_sub
 
        /* On SMP we need to use memory barriers to ensure
@@ -60,89 +64,101 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
        .globl  atomic_add_ret
        .type   atomic_add_ret,#function
 atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
        ATOMIC_PRE_BARRIER
 1:     lduw    [%o1], %g1
        add     %g1, %o0, %g7
        cas     [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %icc, 1b
+       bne,pn  %icc, 2f
         add    %g7, %o0, %g7
        sra     %g7, 0, %o0
        ATOMIC_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic_add_ret, .-atomic_add_ret
 
        .globl  atomic_sub_ret
        .type   atomic_sub_ret,#function
 atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
        ATOMIC_PRE_BARRIER
 1:     lduw    [%o1], %g1
        sub     %g1, %o0, %g7
        cas     [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %icc, 1b
+       bne,pn  %icc, 2f
         sub    %g7, %o0, %g7
        sra     %g7, 0, %o0
        ATOMIC_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic_sub_ret, .-atomic_sub_ret
 
        .globl  atomic64_add
        .type   atomic64_add,#function
 atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
 1:     ldx     [%o1], %g1
        add     %g1, %o0, %g7
        casx    [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic64_add, .-atomic64_add
 
        .globl  atomic64_sub
        .type   atomic64_sub,#function
 atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
 1:     ldx     [%o1], %g1
        sub     %g1, %o0, %g7
        casx    [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic64_sub, .-atomic64_sub
 
        .globl  atomic64_add_ret
        .type   atomic64_add_ret,#function
 atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
        ATOMIC_PRE_BARRIER
 1:     ldx     [%o1], %g1
        add     %g1, %o0, %g7
        casx    [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         add    %g7, %o0, %g7
        mov     %g7, %o0
        ATOMIC_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic64_add_ret, .-atomic64_add_ret
 
        .globl  atomic64_sub_ret
        .type   atomic64_sub_ret,#function
 atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+       BACKOFF_SETUP(%o2)
        ATOMIC_PRE_BARRIER
 1:     ldx     [%o1], %g1
        sub     %g1, %o0, %g7
        casx    [%o1], %g1, %g7
        cmp     %g1, %g7
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         sub    %g7, %o0, %g7
        mov     %g7, %o0
        ATOMIC_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o2, %o3, 1b)
        .size   atomic64_sub_ret, .-atomic64_sub_ret

diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 892431a821311f639e9a18aec4a70c702bd4c726..6b015a6eefb50017e7d22b2d1edc0637408a1fef 100644 (file)
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -1,10 +1,10 @@
-/* $Id: bitops.S,v 1.3 2001/11/18 00:12:56 davem Exp $
- * bitops.S: Sparc64 atomic bit operations.
+/* bitops.S: Sparc64 atomic bit operations.
  *
- * Copyright (C) 2000 David S. Miller (davem@redhat.com)
+ * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
  */
 
 #include <asm/asi.h>
+#include <asm/backoff.h>
 
        .text
 
@@ -29,6 +29,7 @@
        .globl  test_and_set_bit
        .type   test_and_set_bit,#function
 test_and_set_bit:      /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -40,18 +41,20 @@ test_and_set_bit:   /* %o0=nr, %o1=addr */
        or      %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
        clr     %o0
        movrne  %g2, 1, %o0
        BITOP_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_set_bit, .-test_and_set_bit
 
        .globl  test_and_clear_bit
        .type   test_and_clear_bit,#function
 test_and_clear_bit:    /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -63,18 +66,20 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */
        andn    %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
        clr     %o0
        movrne  %g2, 1, %o0
        BITOP_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_clear_bit, .-test_and_clear_bit
 
        .globl  test_and_change_bit
        .type   test_and_change_bit,#function
 test_and_change_bit:   /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        BITOP_PRE_BARRIER
        srlx    %o0, 6, %g1
        mov     1, %o2
@@ -86,18 +91,20 @@ test_and_change_bit:        /* %o0=nr, %o1=addr */
        xor     %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         and    %g7, %o2, %g2
        clr     %o0
        movrne  %g2, 1, %o0
        BITOP_POST_BARRIER
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   test_and_change_bit, .-test_and_change_bit
 
        .globl  set_bit
        .type   set_bit,#function
 set_bit:               /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -108,15 +115,17 @@ set_bit:          /* %o0=nr, %o1=addr */
        or      %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   set_bit, .-set_bit
 
        .globl  clear_bit
        .type   clear_bit,#function
 clear_bit:             /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -127,15 +136,17 @@ clear_bit:                /* %o0=nr, %o1=addr */
        andn    %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   clear_bit, .-clear_bit
 
        .globl  change_bit
        .type   change_bit,#function
 change_bit:            /* %o0=nr, %o1=addr */
+       BACKOFF_SETUP(%o3)
        srlx    %o0, 6, %g1
        mov     1, %o2
        sllx    %g1, 3, %g3
@@ -146,8 +157,9 @@ change_bit:         /* %o0=nr, %o1=addr */
        xor     %g7, %o2, %g1
        casx    [%o1], %g7, %g1
        cmp     %g7, %g1
-       bne,pn  %xcc, 1b
+       bne,pn  %xcc, 2f
         nop
        retl
         nop
+2:     BACKOFF_SPIN(%o3, %o4, 1b)
        .size   change_bit, .-change_bit

diff --git a/include/asm-sparc64/backoff.h b/include/asm-sparc64/backoff.h
new file mode 100644 (file)
index 0000000..0e32f8b
--- /dev/null
+++ b/include/asm-sparc64/backoff.h
@@ -0,0 +1,28 @@
+#ifndef _SPARC64_BACKOFF_H
+#define _SPARC64_BACKOFF_H
+
+#define BACKOFF_LIMIT  (4 * 1024)
+
+#ifdef CONFIG_SMP
+
+#define BACKOFF_SETUP(reg)     \
+       mov     1, reg
+
+#define BACKOFF_SPIN(reg, tmp, label)  \
+       mov     reg, tmp; \
+88:    brnz,pt tmp, 88b; \
+        sub    tmp, 1, tmp; \
+       cmp     reg, BACKOFF_LIMIT; \
+       bg,pn   %xcc, label; \
+        nop; \
+       ba,pt   %xcc, label; \
+        sllx   reg, 1, reg;
+
+#else
+
+#define BACKOFF_SETUP(reg)
+#define BACKOFF_SPIN(reg, tmp, label)
+
+#endif
+
+#endif /* _SPARC64_BACKOFF_H */