From: Andrew Morton Cc: Andi Kleen Cc: Jan Beulich Signed-off-by: Andrew Morton --- arch/i386/lib/semaphore.S | 63 --------------------------------- include/asm-i386/mutex.h | 16 ++++++-- include/asm-i386/rwsem.h | 62 ++++++++++++++++++++++++++------ include/asm-i386/semaphore.h | 49 ++++++++++++++++--------- 4 files changed, 94 insertions(+), 96 deletions(-) diff -puN arch/i386/lib/semaphore.S~revert-x86_64-mm-i386-remove-lock-section arch/i386/lib/semaphore.S --- a/arch/i386/lib/semaphore.S~revert-x86_64-mm-i386-remove-lock-section +++ a/arch/i386/lib/semaphore.S @@ -152,66 +152,3 @@ ENTRY(__read_lock_failed) END(__read_lock_failed) #endif - -/* Fix up special calling conventions */ -ENTRY(call_rwsem_down_read_failed) - CFI_STARTPROC - push %ecx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET ecx,0 - push %edx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET edx,0 - call rwsem_down_read_failed - pop %edx - CFI_ADJUST_CFA_OFFSET -4 - pop %ecx - CFI_ADJUST_CFA_OFFSET -4 - ret - CFI_ENDPROC - END(call_rwsem_down_read_failed) - -ENTRY(call_rwsem_down_write_failed) - CFI_STARTPROC - push %ecx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET ecx,0 - calll rwsem_down_write_failed - pop %ecx - CFI_ADJUST_CFA_OFFSET -4 - ret - CFI_ENDPROC - END(call_rwsem_down_write_failed) - -ENTRY(call_rwsem_wake) - CFI_STARTPROC - decw %dx /* do nothing if still outstanding active readers */ - jnz 1f - push %ecx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET ecx,0 - call rwsem_wake - pop %ecx - CFI_ADJUST_CFA_OFFSET -4 -1: ret - CFI_ENDPROC - END(call_rwsem_wake) - -/* Fix up special calling conventions */ -ENTRY(call_rwsem_downgrade_wake) - CFI_STARTPROC - push %ecx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET ecx,0 - push %edx - CFI_ADJUST_CFA_OFFSET 4 - CFI_REL_OFFSET edx,0 - call rwsem_downgrade_wake - pop %edx - CFI_ADJUST_CFA_OFFSET -4 - pop %ecx - CFI_ADJUST_CFA_OFFSET -4 - ret - CFI_ENDPROC - END(call_rwsem_downgrade_wake) - diff -puN include/asm-i386/mutex.h~revert-x86_64-mm-i386-remove-lock-section include/asm-i386/mutex.h --- a/include/asm-i386/mutex.h~revert-x86_64-mm-i386-remove-lock-section +++ a/include/asm-i386/mutex.h @@ -30,10 +30,14 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " decl (%%eax) \n" \ - " jns 1f \n" \ - " call "#fail_fn" \n" \ + " js 2f \n" \ "1: \n" \ \ + LOCK_SECTION_START("") \ + "2: call "#fail_fn" \n" \ + " jmp 1b \n" \ + LOCK_SECTION_END \ + \ :"=a" (dummy) \ : "a" (count) \ : "memory", "ecx", "edx"); \ @@ -82,10 +86,14 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " incl (%%eax) \n" \ - " jg 1f \n" \ - " call "#fail_fn" \n" \ + " jle 2f \n" \ "1: \n" \ \ + LOCK_SECTION_START("") \ + "2: call "#fail_fn" \n" \ + " jmp 1b \n" \ + LOCK_SECTION_END \ + \ :"=a" (dummy) \ : "a" (count) \ : "memory", "ecx", "edx"); \ diff -puN include/asm-i386/rwsem.h~revert-x86_64-mm-i386-remove-lock-section include/asm-i386/rwsem.h --- a/include/asm-i386/rwsem.h~revert-x86_64-mm-i386-remove-lock-section +++ a/include/asm-i386/rwsem.h @@ -99,9 +99,17 @@ static inline void __down_read(struct rw __asm__ __volatile__( "# beginning down_read\n\t" LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ - " jns 1f\n" - " call call_rwsem_down_read_failed\n" + " js 2f\n\t" /* jump if we weren't granted the lock */ "1:\n\t" + LOCK_SECTION_START("") + "2:\n\t" + " pushl %%ecx\n\t" + " pushl %%edx\n\t" + " call rwsem_down_read_failed\n\t" + " popl %%edx\n\t" + " popl %%ecx\n\t" + " jmp 1b\n" + LOCK_SECTION_END "# ending down_read\n\t" : "+m" (sem->count) : "a" (sem) @@ -143,9 +151,15 @@ static inline void __down_write_nested(s "# beginning down_write\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ " testl %%edx,%%edx\n\t" /* was the count 0 before? */ - " jz 1f\n" - " call call_rwsem_down_write_failed\n" - "1:\n" + " jnz 2f\n\t" /* jump if we weren't granted the lock */ + "1:\n\t" + LOCK_SECTION_START("") + "2:\n\t" + " pushl %%ecx\n\t" + " call rwsem_down_write_failed\n\t" + " popl %%ecx\n\t" + " jmp 1b\n" + LOCK_SECTION_END "# ending down_write" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -179,9 +193,17 @@ static inline void __up_read(struct rw_s __asm__ __volatile__( "# beginning __up_read\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ - " jns 1f\n\t" - " call call_rwsem_wake\n" - "1:\n" + " js 2f\n\t" /* jump if the lock is being waited upon */ + "1:\n\t" + LOCK_SECTION_START("") + "2:\n\t" + " decw %%dx\n\t" /* do nothing if still outstanding active readers */ + " jnz 1b\n\t" + " pushl %%ecx\n\t" + " call rwsem_wake\n\t" + " popl %%ecx\n\t" + " jmp 1b\n" + LOCK_SECTION_END "# ending __up_read\n" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -197,9 +219,17 @@ static inline void __up_write(struct rw_ "# beginning __up_write\n\t" " movl %2,%%edx\n\t" LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ - " jz 1f\n" - " call call_rwsem_wake\n" + " jnz 2f\n\t" /* jump if the lock is being waited upon */ "1:\n\t" + LOCK_SECTION_START("") + "2:\n\t" + " decw %%dx\n\t" /* did the active count reduce to 0? */ + " jnz 1b\n\t" /* jump back if not */ + " pushl %%ecx\n\t" + " call rwsem_wake\n\t" + " popl %%ecx\n\t" + " jmp 1b\n" + LOCK_SECTION_END "# ending __up_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) @@ -214,9 +244,17 @@ static inline void __downgrade_write(str __asm__ __volatile__( "# beginning __downgrade_write\n\t" LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ - " jns 1f\n\t" - " call call_rwsem_downgrade_wake\n" + " js 2f\n\t" /* jump if the lock is being waited upon */ "1:\n\t" + LOCK_SECTION_START("") + "2:\n\t" + " pushl %%ecx\n\t" + " pushl %%edx\n\t" + " call rwsem_downgrade_wake\n\t" + " popl %%edx\n\t" + " popl %%ecx\n\t" + " jmp 1b\n" + LOCK_SECTION_END "# ending __downgrade_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_WAITING_BIAS) diff -puN include/asm-i386/semaphore.h~revert-x86_64-mm-i386-remove-lock-section include/asm-i386/semaphore.h --- a/include/asm-i386/semaphore.h~revert-x86_64-mm-i386-remove-lock-section +++ a/include/asm-i386/semaphore.h @@ -100,10 +100,13 @@ static inline void down(struct semaphore __asm__ __volatile__( "# atomic down operation\n\t" LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "jns 2f\n" - "\tlea %0,%%eax\n\t" - "call __down_failed\n" - "2:" + "js 2f\n" + "1:\n" + LOCK_SECTION_START("") + "2:\tlea %0,%%eax\n\t" + "call __down_failed\n\t" + "jmp 1b\n" + LOCK_SECTION_END :"+m" (sem->count) : :"memory","ax"); @@ -120,12 +123,15 @@ static inline int down_interruptible(str might_sleep(); __asm__ __volatile__( "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "lea %1,%%eax\n\t" - "call __down_failed_interruptible\n" - "2:" + "js 2f\n\t" + "xorl %0,%0\n" + "1:\n" + LOCK_SECTION_START("") + "2:\tlea %1,%%eax\n\t" + "call __down_failed_interruptible\n\t" + "jmp 1b\n" + LOCK_SECTION_END :"=a" (result), "+m" (sem->count) : :"memory"); @@ -142,12 +148,15 @@ static inline int down_trylock(struct se __asm__ __volatile__( "# atomic interruptible down operation\n\t" - "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "jns 2f\n\t" - "lea %1,%%eax\n\t" + "js 2f\n\t" + "xorl %0,%0\n" + "1:\n" + LOCK_SECTION_START("") + "2:\tlea %1,%%eax\n\t" "call __down_failed_trylock\n\t" - "2:\n" + "jmp 1b\n" + LOCK_SECTION_END :"=a" (result), "+m" (sem->count) : :"memory"); @@ -157,16 +166,22 @@ static inline int down_trylock(struct se /* * Note! This is subtle. We jump to wake people up only if * the semaphore was negative (== somebody was waiting on it). + * The default case (no contention) will result in NO + * jumps for both down() and up(). */ static inline void up(struct semaphore * sem) { __asm__ __volatile__( "# atomic up operation\n\t" LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jg 1f\n\t" - "lea %0,%%eax\n\t" - "call __up_wakeup\n" - "1:" + "jle 2f\n" + "1:\n" + LOCK_SECTION_START("") + "2:\tlea %0,%%eax\n\t" + "call __up_wakeup\n\t" + "jmp 1b\n" + LOCK_SECTION_END + ".subsection 0\n" :"+m" (sem->count) : :"memory","ax"); _