From: Nick Piggin <npiggin@suse.de>

Without branch hints, the very unlikely chance of the loop repeating due to
cmpxchg failure is unrolled with gcc-4 that I have tested.

Improve this for architectures with a native cas/cmpxchg.  llsc archs
should try to implement this natively.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Andi Kleen <ak@muc.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---

 include/asm-i386/atomic.h    |    8 +++++++-
 include/asm-ia64/atomic.h    |    8 +++++++-
 include/asm-m68k/atomic.h    |    8 +++++++-
 include/asm-s390/atomic.h    |   18 ++++++++++++++----
 include/asm-sparc64/atomic.h |   10 ++++++++--
 include/asm-x86_64/atomic.h  |    8 +++++++-
 6 files changed, 50 insertions(+), 10 deletions(-)

diff -puN include/asm-i386/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-i386/atomic.h
--- devel/include/asm-i386/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-i386/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(
 ({								\
 	int c, old;						\
 	c = atomic_read(v);					\
-	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+	for (;;) {						\
+		if (unlikely(c == (u)))				\
+			break;					\
+		old = atomic_cmpxchg((v), c, c + (a));		\
+		if (likely(old == c))				\
+			break;					\
 		c = old;					\
+	}							\
 	c != (u);						\
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff -puN include/asm-ia64/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-ia64/atomic.h
--- devel/include/asm-ia64/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-ia64/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *
 ({								\
 	int c, old;						\
 	c = atomic_read(v);					\
-	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+	for (;;) {						\
+		if (unlikely(c == (u)))				\
+			break;					\
+		old = atomic_cmpxchg((v), c, c + (a));		\
+		if (likely(old == c))				\
+			break;					\
 		c = old;					\
+	}							\
 	c != (u);						\
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff -puN include/asm-m68k/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-m68k/atomic.h
--- devel/include/asm-m68k/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-m68k/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -146,8 +146,14 @@ static inline void atomic_set_mask(unsig
 ({								\
 	int c, old;						\
 	c = atomic_read(v);					\
-	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+	for (;;) {						\
+		if (unlikely(c == (u)))				\
+			break;					\
+		old = atomic_cmpxchg((v), c, c + (a));		\
+		if (likely(old == c))				\
+			break;					\
 		c = old;					\
+	}							\
 	c != (u);						\
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
diff -puN include/asm-s390/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-s390/atomic.h
--- devel/include/asm-s390/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-s390/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(ato
 static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
 {
 	int c, old;
-
 	c = atomic_read(v);
-	while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c)
+	for (;;) {
+		if (unlikely(c == u))
+			break;
+		old = atomic_cmpxchg(v, c, c + a);
+		if (likely(old == c))
+			break;
 		c = old;
+	}
 	return c != u;
 }
 
@@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unles
 					  long long a, long long u)
 {
 	long long c, old;
-
 	c = atomic64_read(v);
-	while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c)
+	for (;;) {
+		if (unlikely(c == u))
+			break;
+		old = atomic64_cmpxchg(v, c, c + a);
+		if (likely(old == c))
+			break;
 		c = old;
+	}
 	return c != u;
 }
 
diff -puN include/asm-sparc64/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-sparc64/atomic.h
--- devel/include/asm-sparc64/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-sparc64/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic6
 ({								\
 	int c, old;						\
 	c = atomic_read(v);					\
-	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+	for (;;) {						\
+		if (unlikely(c == (u)))				\
+			break;					\
+		old = atomic_cmpxchg((v), c, c + (a));		\
+		if (likely(old == c))				\
+			break;					\
 		c = old;					\
-	c != (u);						\
+	}							\
+	likely(c != (u));					\
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
diff -puN include/asm-x86_64/atomic.h~atomic-add_unless-cmpxchg-optimise include/asm-x86_64/atomic.h
--- devel/include/asm-x86_64/atomic.h~atomic-add_unless-cmpxchg-optimise	2006-02-20 02:01:42.000000000 -0800
+++ devel-akpm/include/asm-x86_64/atomic.h	2006-02-20 02:01:42.000000000 -0800
@@ -405,8 +405,14 @@ static __inline__ long atomic64_sub_retu
 ({								\
 	int c, old;						\
 	c = atomic_read(v);					\
-	while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+	for (;;) {						\
+		if (unlikely(c == (u)))				\
+			break;					\
+		old = atomic_cmpxchg((v), c, c + (a));		\
+		if (likely(old == c))				\
+			break;					\
 		c = old;					\
+	}							\
 	c != (u);						\
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
_