x86/atom: add pause in spin loops
This commit is contained in:
parent
d464135ff4
commit
45ab27e738
GPG key ID:
EC22E476DC2D3D84
2 changed files with 42 additions and 76 deletions
|
|
@ -35,6 +35,7 @@ inline int atom_write(atom_t *atom, int val)
|
|||
" mov (%2), %0 \n" /* eax = atom->_value */
|
||||
"1: lock \n"
|
||||
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
||||
" pause \n" /* intel says you're supposed to do this in spin loops */
|
||||
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
||||
: "=a"(eax)
|
||||
: "r"(val), "r"(&atom->_value)
|
||||
|
|
@ -112,7 +113,7 @@ inline bool atom_dec(atom_t *atom)
|
|||
" lock \n"
|
||||
" decl (%1) \n"
|
||||
" setne %0 \n"
|
||||
: "+r"(nonzero) /* read+write to ensure the initial value isn't optimized out */
|
||||
: "+r"(nonzero) /* read+write to ensure the initializer isn't optimized out */
|
||||
: "r"(&atom->_value)
|
||||
: "cc", "memory"
|
||||
);
|
||||
|
|
@ -136,6 +137,7 @@ inline int atom_and(atom_t *atom, int val)
|
|||
"1: andl %0, %1 \n" /* val &= eax */
|
||||
" lock \n"
|
||||
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
||||
" pause \n" /* intel says you're supposed to do this in spin loops */
|
||||
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
||||
: "=a"(eax), "+r"(val)
|
||||
: "r"(&atom->_value)
|
||||
|
|
@ -161,6 +163,7 @@ inline int atom_or(atom_t *atom, int val)
|
|||
"1: orl %0, %1 \n" /* val |= eax */
|
||||
" lock \n"
|
||||
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = eax */
|
||||
" pause \n" /* intel says you're supposed to do this in spin loops */
|
||||
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
||||
: "=a"(eax), "+r"(val)
|
||||
: "r"(&atom->_value)
|
||||
|
|
@ -186,6 +189,7 @@ inline int atom_xor(atom_t *atom, int val)
|
|||
"1: xorl %0, %1 \n" /* val ^= eax */
|
||||
" lock \n"
|
||||
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = eax */
|
||||
" pause \n" /* intel says you're supposed to do this in spin loops */
|
||||
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
||||
: "=a"(eax), "+r"(val)
|
||||
: "r"(&atom->_value)
|
||||
|
|
|
|||
|
|
@ -2,172 +2,137 @@
|
|||
|
||||
#include <asm/common.h>
|
||||
|
||||
/*
|
||||
* These routines are only really used if debugging is enabled, where everything
|
||||
* is compiled with -O0. Otherwise clang would just take the inline definitions
|
||||
* from the header file (what an elegant way of creating bugs that only appear
|
||||
* in optimized code!). Therefore, we sacrifice a bit of performance for the
|
||||
* sake of being nicer to gdb by creating frame pointers.
|
||||
*/
|
||||
|
||||
/* int atom_read(const atom_t *atom) */
|
||||
ASM_ENTRY(atom_read)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
|
||||
mov 8(%ebp), %ecx
|
||||
mov 4(%esp), %ecx
|
||||
mov (%ecx), %eax
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_read)
|
||||
|
||||
/* int atom_write(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_write)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 12(%ebp), %ecx
|
||||
mov 4(%esp), %edx
|
||||
mov 8(%esp), %ecx
|
||||
mov (%edx), %eax
|
||||
|
||||
1: lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_write)
|
||||
|
||||
/* bool atom_inc(atom_t *atom) */
|
||||
ASM_ENTRY(atom_inc)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
mov 4(%esp), %edx
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
lock
|
||||
incl (%edx)
|
||||
xor %eax, %eax
|
||||
setne %al
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_inc)
|
||||
|
||||
/* bool atom_dec(atom_t *atom) */
|
||||
ASM_ENTRY(atom_dec)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
mov 4(%esp), %edx
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
lock
|
||||
decl (%edx)
|
||||
xor %eax, %eax
|
||||
setne %al
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_dec)
|
||||
|
||||
/* int atom_add(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_add)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
mov 4(%esp), %edx
|
||||
mov 8(%esp), %eax
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 12(%ebp), %eax
|
||||
lock
|
||||
xaddl %eax, (%edx)
|
||||
xadd %eax, (%edx)
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_add)
|
||||
|
||||
/* int atom_sub(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_sub)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
mov 4(%esp), %edx
|
||||
mov 8(%esp), %eax
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 12(%ebp), %eax
|
||||
/* there is no xsubl, so we add the two's complement */
|
||||
neg %eax
|
||||
lock
|
||||
xaddl %eax, (%edx)
|
||||
xadd %eax, (%edx)
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_sub)
|
||||
|
||||
/* int atom_and(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_and)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 4(%esp), %edx
|
||||
mov (%edx), %eax
|
||||
|
||||
1: mov %eax, %ecx
|
||||
and 12(%ebp), %ecx
|
||||
and 8(%esp), %ecx
|
||||
lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_and)
|
||||
|
||||
/* int atom_or(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_or)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 4(%esp), %edx
|
||||
mov (%edx), %eax
|
||||
|
||||
1: mov %eax, %ecx
|
||||
and 12(%ebp), %ecx
|
||||
and 8(%esp), %ecx
|
||||
lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_or)
|
||||
|
||||
/* int atom_xor(atom_t *atom, int val) */
|
||||
ASM_ENTRY(atom_xor)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 4(%esp), %edx
|
||||
mov (%edx), %eax
|
||||
|
||||
1: mov %eax, %ecx
|
||||
xor 12(%ebp), %ecx
|
||||
xor 8(%esp), %ecx
|
||||
lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_xor)
|
||||
|
||||
/* bool atom_set_bit(atom_t *atom, int bit) */
|
||||
ASM_ENTRY(atom_set_bit)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
push %ebx
|
||||
mov 4(%esp), %edx
|
||||
mov 8(%esp), %ecx
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 12(%ebp), %ecx
|
||||
push %ebx
|
||||
mov $1, %ebx
|
||||
shl %cl, %ebx
|
||||
|
||||
mov (%edx), %eax
|
||||
|
||||
1: mov %eax, %ecx
|
||||
or %ebx, %ecx
|
||||
lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
/* return true if bit was clear before */
|
||||
|
|
@ -176,18 +141,15 @@ ASM_ENTRY(atom_set_bit)
|
|||
shr %cl, %eax
|
||||
|
||||
pop %ebx
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_set_bit)
|
||||
|
||||
/* bool atom_clr_bit(atom_t *atom, int bit) */
|
||||
ASM_ENTRY(atom_clr_bit)
|
||||
push %ebp
|
||||
mov %esp, %ebp
|
||||
push %ebx
|
||||
mov 4(%esp), %edx
|
||||
mov 8(%esp), %ecx
|
||||
|
||||
mov 8(%ebp), %edx
|
||||
mov 12(%ebp), %ecx
|
||||
push %ebx
|
||||
mov $0xfffffffe, %ebx
|
||||
rol %cl, %ebx
|
||||
mov (%edx), %eax
|
||||
|
|
@ -195,7 +157,8 @@ ASM_ENTRY(atom_clr_bit)
|
|||
1: mov %eax, %ecx
|
||||
and %ebx, %ecx
|
||||
lock
|
||||
cmpxchgl %ecx, (%edx)
|
||||
cmpxchg %ecx, (%edx)
|
||||
pause
|
||||
jne 1b
|
||||
|
||||
/* return true if bit was set before */
|
||||
|
|
@ -204,7 +167,6 @@ ASM_ENTRY(atom_clr_bit)
|
|||
shr %cl, %eax
|
||||
|
||||
pop %ebx
|
||||
pop %ebp
|
||||
ret
|
||||
ASM_END(atom_clr_bit)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue