From: Nick Piggin Introduce test_and_set_bit_lock / clear_bit_unlock bitops with lock semantics. Convert all architectures to use the generic implementation. Signed-off-by: Nick Piggin Acked-By: David Howells Cc: Richard Henderson Cc: Ivan Kokshaysky Cc: Russell King Cc: Haavard Skinnemoen Cc: Bryan Wu Cc: Mikael Starvik Cc: David Howells Cc: Yoshinori Sato Cc: "Luck, Tony" Cc: Hirokazu Takata Cc: Geert Uytterhoeven Cc: Roman Zippel Cc: Greg Ungerer Cc: Ralf Baechle Cc: Kyle McMartin Cc: Matthew Wilcox Cc: Paul Mackerras Cc: Benjamin Herrenschmidt Cc: Heiko Carstens Cc: Martin Schwidefsky Cc: Paul Mundt Cc: Kazumoto Kojima Cc: Richard Curnow Cc: William Lee Irwin III Cc: "David S. Miller" Cc: Jeff Dike Cc: Paolo 'Blaisorblade' Giarrusso Cc: Miles Bader Cc: Andi Kleen Cc: Chris Zankel Signed-off-by: Andrew Morton --- Documentation/atomic_ops.txt | 14 ++++++++ Documentation/memory-barriers.txt | 14 +++++++- include/asm-alpha/bitops.h | 1 include/asm-arm/bitops.h | 1 include/asm-avr32/bitops.h | 1 include/asm-blackfin/bitops.h | 1 include/asm-cris/bitops.h | 1 include/asm-frv/bitops.h | 1 include/asm-generic/bitops.h | 1 include/asm-generic/bitops/lock.h | 45 ++++++++++++++++++++++++++++ include/asm-h8300/bitops.h | 1 include/asm-i386/bitops.h | 1 include/asm-ia64/bitops.h | 2 + include/asm-m32r/bitops.h | 1 include/asm-m68k/bitops.h | 1 include/asm-m68knommu/bitops.h | 1 include/asm-mips/bitops.h | 1 include/asm-parisc/bitops.h | 1 include/asm-powerpc/bitops.h | 1 include/asm-s390/bitops.h | 1 include/asm-sh/bitops.h | 1 include/asm-sh64/bitops.h | 1 include/asm-sparc/bitops.h | 1 include/asm-sparc64/bitops.h | 1 include/asm-v850/bitops.h | 1 include/asm-x86_64/bitops.h | 1 include/asm-xtensa/bitops.h | 1 27 files changed, 96 insertions(+), 2 deletions(-) diff -puN Documentation/atomic_ops.txt~bitops-introduce-lock-ops Documentation/atomic_ops.txt --- a/Documentation/atomic_ops.txt~bitops-introduce-lock-ops +++ a/Documentation/atomic_ops.txt @@ -418,6 +418,20 @@ brothers: */ smp_mb__after_clear_bit(); +There are two special bitops with lock barrier semantics (acquire/release, +same as spinlocks). These operate in the same way as their non-_lock/unlock +postfixed variants, except that they are to provide acquire/release semantics, +respectively. This means they can be used for bit_spin_trylock and +bit_spin_unlock type operations without specifying any more barriers. + + int test_and_set_bit_lock(unsigned long nr, unsigned long *addr); + void clear_bit_unlock(unsigned long nr, unsigned long *addr); + void __clear_bit_unlock(unsigned long nr, unsigned long *addr); + +The __clear_bit_unlock version is non-atomic, however it still implements +unlock barrier semantics. This can be useful if the lock itself is protecting +the other bits in the word. + Finally, there are non-atomic versions of the bitmask operations provided. They are used in contexts where some other higher-level SMP locking scheme is being used to protect the bitmask, and thus less diff -puN Documentation/memory-barriers.txt~bitops-introduce-lock-ops Documentation/memory-barriers.txt --- a/Documentation/memory-barriers.txt~bitops-introduce-lock-ops +++ a/Documentation/memory-barriers.txt @@ -1479,7 +1479,8 @@ kernel. Any atomic operation that modifies some state in memory and returns information about the state (old or new) implies an SMP-conditional general memory barrier -(smp_mb()) on each side of the actual operation. These include: +(smp_mb()) on each side of the actual operation (with the exception of +explicit lock operations, described later). These include: xchg(); cmpxchg(); @@ -1536,10 +1537,19 @@ If they're used for constructing a lock do need memory barriers as a lock primitive generally has to do things in a specific order. - Basically, each usage case has to be carefully considered as to whether memory barriers are needed or not. +The following operations are special locking primitives: + + test_and_set_bit_lock(); + clear_bit_unlock(); + __clear_bit_unlock(); + +These implement LOCK-class and UNLOCK-class operations. These should be used in +preference to other operations when implementing locking primitives, because +their implementations can be optimised on many architectures. + [!] Note that special memory barrier primitives are available for these situations because on some CPUs the atomic instructions used imply full memory barriers, and so barrier instructions are superfluous in conjunction with them, diff -puN include/asm-alpha/bitops.h~bitops-introduce-lock-ops include/asm-alpha/bitops.h --- a/include/asm-alpha/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-alpha/bitops.h @@ -371,6 +371,7 @@ static inline unsigned int hweight8(unsi #else #include #endif +#include #endif /* __KERNEL__ */ diff -puN include/asm-arm/bitops.h~bitops-introduce-lock-ops include/asm-arm/bitops.h --- a/include/asm-arm/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-arm/bitops.h @@ -290,6 +290,7 @@ static inline int constant_fls(int x) #include #include +#include /* * Ext2 is defined to use little-endian byte ordering. diff -puN include/asm-avr32/bitops.h~bitops-introduce-lock-ops include/asm-avr32/bitops.h --- a/include/asm-avr32/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-avr32/bitops.h @@ -292,6 +292,7 @@ static inline int ffs(unsigned long word #include #include #include +#include #include #include diff -puN include/asm-blackfin/bitops.h~bitops-introduce-lock-ops include/asm-blackfin/bitops.h --- a/include/asm-blackfin/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-blackfin/bitops.h @@ -203,6 +203,7 @@ static __inline__ int __test_bit(int nr, #include #include +#include #include #include diff -puN include/asm-cris/bitops.h~bitops-introduce-lock-ops include/asm-cris/bitops.h --- a/include/asm-cris/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-cris/bitops.h @@ -158,6 +158,7 @@ static inline int test_and_change_bit(in #include #include #include +#include #include diff -puN include/asm-frv/bitops.h~bitops-introduce-lock-ops include/asm-frv/bitops.h --- a/include/asm-frv/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-frv/bitops.h @@ -306,6 +306,7 @@ int __ilog2_u64(u64 n) #include #include +#include #include diff -puN include/asm-generic/bitops.h~bitops-introduce-lock-ops include/asm-generic/bitops.h --- a/include/asm-generic/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-generic/bitops.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include diff -puN /dev/null include/asm-generic/bitops/lock.h --- /dev/null +++ a/include/asm-generic/bitops/lock.h @@ -0,0 +1,45 @@ +#ifndef _ASM_GENERIC_BITOPS_LOCK_H_ +#define _ASM_GENERIC_BITOPS_LOCK_H_ + +/** + * test_and_set_bit_lock - Set a bit and return its old value, for lock + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is atomic and provides acquire barrier semantics. + * It can be used to implement bit locks. + */ +#define test_and_set_bit_lock(nr, addr) test_and_set_bit(nr, addr) + +/** + * clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + */ +#define clear_bit_unlock(nr, addr) \ +do { \ + smp_mb__before_clear_bit(); \ + clear_bit(nr, addr); \ +} while (0) + +/** + * __clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is like clear_bit_unlock, however it is not atomic. + * It does provide release barrier semantics so it can be used to unlock + * a bit lock, however it would only be used if no other CPU can modify + * any bits in the memory until the lock is released (a good example is + * if the bit lock itself protects access to the other bits in the word). + */ +#define __clear_bit_unlock(nr, addr) \ +do { \ + smp_mb(); \ + __clear_bit(nr, addr); \ +} while (0) + +#endif /* _ASM_GENERIC_BITOPS_LOCK_H_ */ + diff -puN include/asm-h8300/bitops.h~bitops-introduce-lock-ops include/asm-h8300/bitops.h --- a/include/asm-h8300/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-h8300/bitops.h @@ -199,6 +199,7 @@ static __inline__ unsigned long __ffs(un #include #include #include +#include #include #include #include diff -puN include/asm-i386/bitops.h~bitops-introduce-lock-ops include/asm-i386/bitops.h --- a/include/asm-i386/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-i386/bitops.h @@ -406,6 +406,7 @@ static inline int fls(int x) } #include +#include #endif /* __KERNEL__ */ diff -puN include/asm-ia64/bitops.h~bitops-introduce-lock-ops include/asm-ia64/bitops.h --- a/include/asm-ia64/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-ia64/bitops.h @@ -375,6 +375,8 @@ hweight64 (unsigned long x) #define hweight16(x) (unsigned int) hweight64((x) & 0xfffful) #define hweight8(x) (unsigned int) hweight64((x) & 0xfful) +#include + #endif /* __KERNEL__ */ #include diff -puN include/asm-m32r/bitops.h~bitops-introduce-lock-ops include/asm-m32r/bitops.h --- a/include/asm-m32r/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-m32r/bitops.h @@ -259,6 +259,7 @@ static __inline__ int test_and_change_bi #include #include #include +#include #endif /* __KERNEL__ */ diff -puN include/asm-m68k/bitops.h~bitops-introduce-lock-ops include/asm-m68k/bitops.h --- a/include/asm-m68k/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-m68k/bitops.h @@ -318,6 +318,7 @@ static inline int fls(int x) #include #include #include +#include /* Bitmap functions for the minix filesystem */ diff -puN include/asm-m68knommu/bitops.h~bitops-introduce-lock-ops include/asm-m68knommu/bitops.h --- a/include/asm-m68knommu/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-m68knommu/bitops.h @@ -164,6 +164,7 @@ static __inline__ int __test_bit(int nr, #include #include +#include static __inline__ int ext2_set_bit(int nr, volatile void * addr) { diff -puN include/asm-mips/bitops.h~bitops-introduce-lock-ops include/asm-mips/bitops.h --- a/include/asm-mips/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-mips/bitops.h @@ -560,6 +560,7 @@ static inline int ffs(int word) #include #include +#include #include #include #include diff -puN include/asm-parisc/bitops.h~bitops-introduce-lock-ops include/asm-parisc/bitops.h --- a/include/asm-parisc/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-parisc/bitops.h @@ -212,6 +212,7 @@ static __inline__ int fls(int x) #include #include +#include #include #endif /* __KERNEL__ */ diff -puN include/asm-powerpc/bitops.h~bitops-introduce-lock-ops include/asm-powerpc/bitops.h --- a/include/asm-powerpc/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-powerpc/bitops.h @@ -270,6 +270,7 @@ static __inline__ int fls(unsigned int x #include #include +#include #define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) unsigned long find_next_zero_bit(const unsigned long *addr, diff -puN include/asm-s390/bitops.h~bitops-introduce-lock-ops include/asm-s390/bitops.h --- a/include/asm-s390/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-s390/bitops.h @@ -750,6 +750,7 @@ static inline int sched_find_first_bit(u #include #include +#include /* * ATTENTION: intel byte ordering convention for ext2 and minix !! diff -puN include/asm-sh/bitops.h~bitops-introduce-lock-ops include/asm-sh/bitops.h --- a/include/asm-sh/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-sh/bitops.h @@ -142,6 +142,7 @@ static inline unsigned long __ffs(unsign #include #include #include +#include #include #include #include diff -puN include/asm-sh64/bitops.h~bitops-introduce-lock-ops include/asm-sh64/bitops.h --- a/include/asm-sh64/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-sh64/bitops.h @@ -141,6 +141,7 @@ static __inline__ unsigned long ffz(unsi #include #include #include +#include #include #include #include diff -puN include/asm-sparc/bitops.h~bitops-introduce-lock-ops include/asm-sparc/bitops.h --- a/include/asm-sparc/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-sparc/bitops.h @@ -100,6 +100,7 @@ static inline void change_bit(unsigned l #include #include #include +#include #include #include #include diff -puN include/asm-sparc64/bitops.h~bitops-introduce-lock-ops include/asm-sparc64/bitops.h --- a/include/asm-sparc64/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-sparc64/bitops.h @@ -85,6 +85,7 @@ static inline unsigned int hweight8(unsi #include #endif +#include #endif /* __KERNEL__ */ #include diff -puN include/asm-v850/bitops.h~bitops-introduce-lock-ops include/asm-v850/bitops.h --- a/include/asm-v850/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-v850/bitops.h @@ -148,6 +148,7 @@ static inline int __test_bit (int nr, co #include #include #include +#include #include #define ext2_set_bit_atomic(l,n,a) test_and_set_bit(n,a) diff -puN include/asm-x86_64/bitops.h~bitops-introduce-lock-ops include/asm-x86_64/bitops.h --- a/include/asm-x86_64/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-x86_64/bitops.h @@ -412,6 +412,7 @@ static __inline__ int fls(int x) #define ARCH_HAS_FAST_MULTIPLIER 1 #include +#include #endif /* __KERNEL__ */ diff -puN include/asm-xtensa/bitops.h~bitops-introduce-lock-ops include/asm-xtensa/bitops.h --- a/include/asm-xtensa/bitops.h~bitops-introduce-lock-ops +++ a/include/asm-xtensa/bitops.h @@ -112,6 +112,7 @@ static inline int fls (unsigned int x) #endif #include +#include #include #include _