x86/atom: add pause in spin loops

arch/x86/include/arch/atom.h

@@ -35,6 +35,7 @@ inline int atom_write(atom_t *atom, int val)
 "	mov	(%2),	%0	\n" /* eax = atom->_value */
 "1:	lock			\n"
 "	cmpxchgl %1, (%2)	\n" /* if (atom->_value == eax) atom->_value = val */
+"	pause			\n" /* intel says you're supposed to do this in spin loops */
 "	jne	1b		\n" /* else goto 1 (eax updated to new atom->_value) */
 	: "=a"(eax)
 	: "r"(val), "r"(&atom->_value)
@@ -112,7 +113,7 @@ inline bool atom_dec(atom_t *atom)
 "	lock		\n"
 "	decl	(%1)	\n"
 "	setne	%0	\n"
-	: "+r"(nonzero) /* read+write to ensure the initial value isn't optimized out */
+	: "+r"(nonzero) /* read+write to ensure the initializer isn't optimized out */
 	: "r"(&atom->_value)
 	: "cc", "memory"
 	);
@@ -136,6 +137,7 @@ inline int atom_and(atom_t *atom, int val)
 "1:	andl	%0, %1		\n" /* val &= eax */
 "	lock			\n"
 "	cmpxchgl %1, (%2)	\n" /* if (atom->_value == eax) atom->_value = val */
+"	pause			\n" /* intel says you're supposed to do this in spin loops */
 "	jne	1b		\n" /* else goto 1 (eax updated to new atom->_value) */
 	: "=a"(eax), "+r"(val)
 	: "r"(&atom->_value)
@@ -161,6 +163,7 @@ inline int atom_or(atom_t *atom, int val)
 "1:	orl	%0,	%1	\n" /* val |= eax */
 "	lock			\n"
 "	cmpxchgl %1,	(%2)	\n" /* if (atom->_value == eax) atom->_value = eax */
+"	pause			\n" /* intel says you're supposed to do this in spin loops */
 "	jne	1b		\n" /* else goto 1 (eax updated to new atom->_value) */
 	: "=a"(eax), "+r"(val)
 	: "r"(&atom->_value)
@@ -186,6 +189,7 @@ inline int atom_xor(atom_t *atom, int val)
 "1:	xorl	%0,	%1	\n" /* val ^= eax */
 "	lock			\n"
 "	cmpxchgl %1,	(%2)	\n" /* if (atom->_value == eax) atom->_value = eax */
+"	pause			\n" /* intel says you're supposed to do this in spin loops */
 "	jne	1b		\n" /* else goto 1 (eax updated to new atom->_value) */
 	: "=a"(eax), "+r"(val)
 	: "r"(&atom->_value)

arch/x86/sys/i386/atom.S

@@ -2,172 +2,137 @@
 
 #include <asm/common.h>
 
-/*
- * These routines are only really used if debugging is enabled, where everything
- * is compiled with -O0.  Otherwise clang would just take the inline definitions
- * from the header file (what an elegant way of creating bugs that only appear
- * in optimized code!).  Therefore, we sacrifice a bit of performance for the
- * sake of being nicer to gdb by creating frame pointers.
- */
-
 /* int atom_read(const atom_t *atom) */
 ASM_ENTRY(atom_read)
-	push	%ebp
-	mov	%esp, %ebp
-
-	mov	8(%ebp), %ecx
+	mov	4(%esp), %ecx
 	mov	(%ecx), %eax
 
-	pop	%ebp
 	ret
 ASM_END(atom_read)
 
 /* int atom_write(atom_t *atom, int val) */
 ASM_ENTRY(atom_write)
-	push	%ebp
-	mov	%esp, %ebp
-
-	mov	8(%ebp), %edx
-	mov	12(%ebp), %ecx
+	mov	4(%esp), %edx
+	mov	8(%esp), %ecx
 	mov	(%edx), %eax
+
 1:	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
-	pop	%ebp
 	ret
 ASM_END(atom_write)
 
 /* bool atom_inc(atom_t *atom) */
 ASM_ENTRY(atom_inc)
-	push	%ebp
-	mov	%esp, %ebp
+	mov	4(%esp), %edx
 
-	mov	8(%ebp), %edx
 	lock
 	incl	(%edx)
 	xor	%eax, %eax
 	setne	%al
 
-	pop	%ebp
 	ret
 ASM_END(atom_inc)
 
 /* bool atom_dec(atom_t *atom) */
 ASM_ENTRY(atom_dec)
-	push	%ebp
-	mov	%esp, %ebp
+	mov	4(%esp), %edx
 
-	mov	8(%ebp), %edx
 	lock
 	decl	(%edx)
 	xor	%eax, %eax
 	setne	%al
 
-	pop	%ebp
 	ret
 ASM_END(atom_dec)
 
 /* int atom_add(atom_t *atom, int val) */
 ASM_ENTRY(atom_add)
-	push	%ebp
-	mov	%esp, %ebp
+	mov	4(%esp), %edx
+	mov	8(%esp), %eax
 
-	mov	8(%ebp), %edx
-	mov	12(%ebp), %eax
 	lock
-	xaddl	%eax, (%edx)
+	xadd	%eax, (%edx)
 
-	pop	%ebp
 	ret
 ASM_END(atom_add)
 
 /* int atom_sub(atom_t *atom, int val) */
 ASM_ENTRY(atom_sub)
-	push	%ebp
-	mov	%esp, %ebp
+	mov	4(%esp), %edx
+	mov	8(%esp), %eax
 
-	mov	8(%ebp), %edx
-	mov	12(%ebp), %eax
 	/* there is no xsubl, so we add the two's complement */
 	neg	%eax
 	lock
-	xaddl	%eax, (%edx)
+	xadd	%eax, (%edx)
 
-	pop	%ebp
 	ret
 ASM_END(atom_sub)
 
 /* int atom_and(atom_t *atom, int val) */
 ASM_ENTRY(atom_and)
-	push	%ebp
-	mov	%esp, %ebp
-
-	mov	8(%ebp), %edx
+	mov	4(%esp), %edx
 	mov	(%edx), %eax
 
 1:	mov	%eax, %ecx
-	and	12(%ebp), %ecx
+	and	8(%esp), %ecx
 	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
-	pop	%ebp
 	ret
 ASM_END(atom_and)
 
 /* int atom_or(atom_t *atom, int val) */
 ASM_ENTRY(atom_or)
-	push	%ebp
-	mov	%esp, %ebp
-
-	mov	8(%ebp), %edx
+	mov	4(%esp), %edx
 	mov	(%edx), %eax
 
 1:	mov	%eax, %ecx
-	and	12(%ebp), %ecx
+	and	8(%esp), %ecx
 	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
-	pop	%ebp
 	ret
 ASM_END(atom_or)
 
 /* int atom_xor(atom_t *atom, int val) */
 ASM_ENTRY(atom_xor)
-	push	%ebp
-	mov	%esp, %ebp
-
-	mov	8(%ebp), %edx
+	mov	4(%esp), %edx
 	mov	(%edx), %eax
 
 1:	mov	%eax, %ecx
-	xor	12(%ebp), %ecx
+	xor	8(%esp), %ecx
 	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
-	pop	%ebp
 	ret
 ASM_END(atom_xor)
 
 /* bool atom_set_bit(atom_t *atom, int bit) */
 ASM_ENTRY(atom_set_bit)
-	push	%ebp
-	mov	%esp, %ebp
-	push	%ebx
+	mov	4(%esp), %edx
+	mov	8(%esp), %ecx
 
-	mov	8(%ebp), %edx
-	mov	12(%ebp), %ecx
+	push	%ebx
 	mov	$1, %ebx
 	shl	%cl, %ebx
+
 	mov	(%edx), %eax
 
 1:	mov	%eax, %ecx
 	or	%ebx, %ecx
 	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
 	/* return true if bit was clear before */
@@ -176,18 +141,15 @@ ASM_ENTRY(atom_set_bit)
 	shr	%cl, %eax
 
 	pop	%ebx
-	pop	%ebp
 	ret
 ASM_END(atom_set_bit)
 
 /* bool atom_clr_bit(atom_t *atom, int bit) */
 ASM_ENTRY(atom_clr_bit)
-	push	%ebp
-	mov	%esp, %ebp
-	push	%ebx
+	mov	4(%esp), %edx
+	mov	8(%esp), %ecx
 
-	mov	8(%ebp), %edx
-	mov	12(%ebp), %ecx
+	push	%ebx
 	mov	$0xfffffffe, %ebx
 	rol	%cl, %ebx
 	mov	(%edx), %eax
@@ -195,7 +157,8 @@ ASM_ENTRY(atom_clr_bit)
 1:	mov	%eax, %ecx
 	and	%ebx, %ecx
 	lock
-	cmpxchgl %ecx, (%edx)
+	cmpxchg	%ecx, (%edx)
+	pause
 	jne	1b
 
 	/* return true if bit was set before */
@@ -204,7 +167,6 @@ ASM_ENTRY(atom_clr_bit)
 	shr	%cl, %eax
 
 	pop	%ebx
-	pop	%ebp
 	ret
 ASM_END(atom_clr_bit)