You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
360 lines
8.0 KiB
C
360 lines
8.0 KiB
C
/* Copyright (C) 2021,2022 fef <owo@fef.moe>. All rights reserved. */
|
|
|
|
#pragma once
|
|
#define _ARCH_ATOM_H_
|
|
|
|
#include <gay/cdefs.h>
|
|
#include <gay/config.h>
|
|
#include <gay/types.h>
|
|
|
|
#ifdef CFG_SMP
|
|
#define X86_LOCK_PREFIX "\tlock\n"
|
|
#else
|
|
#define X86_LOCK_PREFIX ""
|
|
#endif
|
|
|
|
/**
|
|
* @brief Initialize an atom non-atomically.
|
|
* This function is **only** for initializing an atom, you should never use it
|
|
* in code that requires actual atomicity (i.e. everything except initializers).
|
|
*
|
|
* @param atom Atom to initialize
|
|
* @param val Initial value for the atom
|
|
*/
|
|
static inline void atom_init(atom_t *atom, int val)
|
|
{
|
|
atom->_value = val;
|
|
}
|
|
|
|
/**
|
|
* @brief Read an atom's current value.
|
|
* You usually shouldn't need this function because all the other atomic
|
|
* primitives return the value before the operation, and we are only really
|
|
* interested in how values *compare* between operations in most cases.
|
|
*
|
|
* @param atom Atom to read the value of
|
|
* @return The atom's "current" value (at the time of reading it)
|
|
*/
|
|
static inline int atom_read(const atom_t *atom)
|
|
{
|
|
return atom->_value;
|
|
}
|
|
|
|
/**
|
|
* @brief Write a new value to an atom.
|
|
*
|
|
* @param atom Atom to write to
|
|
* @param val New value
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_xchg(atom_t *atom, int val)
|
|
{
|
|
/* the intel manual says XCHG is always atomic, and you don't need a LOCK prefix */
|
|
__asm__ volatile(
|
|
" xchgl %0, (%1) \n"
|
|
: "+r"(val)
|
|
: "r"(&atom->_value)
|
|
: "memory"
|
|
);
|
|
|
|
return val;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic compare/exchange.
|
|
* The specified value will only be written out if the current value in the atom
|
|
* matches `compare`.
|
|
*
|
|
* You can remember the order of the arguments by the name `atom_cmp_xchg`:
|
|
* first the `atom` pointer,
|
|
* then the value to `cmp` it with (i.e. `compare`),
|
|
* then the value to `xchg` it with (i.e. `val`).
|
|
*
|
|
* @param atom Atom to write to
|
|
* @param compare Expected current value
|
|
* @param val New value
|
|
* @return The value that was stored in the atom *before* the write.
|
|
* If this is equal to `compare`, the store was successful, otherwise the
|
|
* atom is unmodified.
|
|
*/
|
|
static inline int atom_cmp_xchg(atom_t *atom, int compare, int val)
|
|
{
|
|
int eax = compare;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
|
: "+a"(eax)
|
|
: "r"(val), "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return eax;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic load/add/store.
|
|
*
|
|
* @param atom Atom to add to
|
|
* @param val Value to add
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_add(atom_t *atom, int val)
|
|
{
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" xaddl %0, (%1) \n"
|
|
: "+r"(val)
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return val;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic load/subtract/store.
|
|
*
|
|
* @param atom Atom to subtract from
|
|
* @param val Value to subtract
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_sub(atom_t *atom, int val)
|
|
{
|
|
return atom_add(atom, -val);
|
|
}
|
|
|
|
/**
|
|
* @brief Increment an atom by one.
|
|
*
|
|
* @param atom Atom to increment
|
|
* @return `true` if the vale *after* the operation is nonzero
|
|
*/
|
|
static inline bool atom_inc(atom_t *atom)
|
|
{
|
|
bool nonzero = false;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" incl (%1) \n"
|
|
" setne %0 \n"
|
|
: "+r"(nonzero) /* read+write to ensure the initial value isn't optimized out */
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return nonzero;
|
|
}
|
|
|
|
/**
|
|
* @brief Decrement an atom by one.
|
|
*
|
|
* @param atom Atom to decrement
|
|
* @return `true` if the value *after* the operation is nonzero
|
|
*/
|
|
static inline bool atom_dec(atom_t *atom)
|
|
{
|
|
bool nonzero = false;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" decl (%1) \n"
|
|
" setne %0 \n"
|
|
: "+r"(nonzero) /* read+write to ensure the initializer isn't optimized out */
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return nonzero;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic bitwise AND and write the result back to the atom.
|
|
*
|
|
* @param atom Atom to perform the AND on
|
|
* @param val Value to AND with
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_and(atom_t *atom, int val)
|
|
{
|
|
int eax = atom->_value;
|
|
|
|
__asm__ volatile(
|
|
"1: andl %0, %1 \n" /* val &= eax */
|
|
X86_LOCK_PREFIX
|
|
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
|
" pause \n" /* intel says you're supposed to do this in spin loops */
|
|
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
|
: "+a"(eax), "+r"(val)
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return eax;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic bitwise OR and write the result back to the atom.
|
|
*
|
|
* @param atom Atom to perform the OR on
|
|
* @param val Value to OR with
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_or(atom_t *atom, int val)
|
|
{
|
|
int eax = atom->_value;
|
|
|
|
__asm__ volatile(
|
|
"1: orl %0, %1 \n" /* val |= eax */
|
|
X86_LOCK_PREFIX
|
|
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
|
" pause \n" /* intel says you're supposed to do this in spin loops */
|
|
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
|
: "+a"(eax), "+r"(val)
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return eax;
|
|
}
|
|
|
|
/**
|
|
* @brief Perform an atomic bitwise XOR and write the result back to the atom.
|
|
*
|
|
* @param atom Atom to perform the XOR on
|
|
* @param val Value to XOR with
|
|
* @return The value of `atom` *before* the operation
|
|
*/
|
|
static inline int atom_xor(atom_t *atom, int val)
|
|
{
|
|
int eax;
|
|
|
|
__asm__ volatile(
|
|
" movl (%2), %0 \n" /* eax = atom->_value */
|
|
"1: xorl %0, %1 \n" /* val ^= eax */
|
|
X86_LOCK_PREFIX
|
|
" cmpxchgl %1, (%2) \n" /* if (atom->_value == eax) atom->_value = val */
|
|
" pause \n" /* intel says you're supposed to do this in spin loops */
|
|
" jne 1b \n" /* else goto 1 (eax updated to new atom->_value) */
|
|
: "=a"(eax), "+r"(val)
|
|
: "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return eax;
|
|
}
|
|
|
|
/**
|
|
* @brief Atomically set a bit.
|
|
*
|
|
* @param atom Atom to set a bit of
|
|
* @param pos Bit position (starting from 0 for the LSB)
|
|
* @return `true` the bit was clear *before* the operation
|
|
*/
|
|
static inline bool atom_set_bit(atom_t *atom, int pos)
|
|
{
|
|
bool ret = false;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" btsl %1, (%2) \n"
|
|
" setc %0 \n"
|
|
: "+r"(ret)
|
|
: "r"(pos), "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* @brief Atomically clear a bit.
|
|
*
|
|
* @param atom Atom to clear a bit of
|
|
* @param pos Bit position (starting from 0 for the LSB)
|
|
* @return `true` if the bit was set *before* the operation
|
|
*/
|
|
static inline bool atom_clr_bit(atom_t *atom, int pos)
|
|
{
|
|
bool ret = false;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" btrl %1, (%2) \n"
|
|
" setc %0 \n"
|
|
: "+r"(ret)
|
|
: "r"(pos), "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* @brief Atomically complement a bit.
|
|
*
|
|
* @param atom Atom to flip a bit of
|
|
* @param pos Bit position (starting from 0 for the LSB)
|
|
* @return `true` if the bit was set *before* the operation
|
|
*/
|
|
static inline bool atom_flip_bit(atom_t *atom, int pos)
|
|
{
|
|
bool ret = false;
|
|
|
|
__asm__ volatile(
|
|
X86_LOCK_PREFIX
|
|
" btcl %1, (%2) \n"
|
|
" setc %0 \n"
|
|
: "+r"(ret)
|
|
: "r"(pos), "r"(&atom->_value)
|
|
: "cc", "memory"
|
|
);
|
|
|
|
return ret;
|
|
}
|
|
|
|
#ifdef __x86_64__
|
|
#include <amd64/latom.h>
|
|
#else
|
|
#include <i386/latom.h>
|
|
#endif
|
|
|
|
/*
|
|
* we use ILP32 on i386 and LP64 on amd64, therefore a long is exactly the size
|
|
* of a pointer on both platforms
|
|
*/
|
|
|
|
#if __SIZEOF_LONG__ != __SIZEOF_POINTER__
|
|
#error "sizeof(long) is expected equal sizeof(void *) on x86"
|
|
#endif
|
|
|
|
static inline void patom_init(patom_t *patom, void *val)
|
|
{
|
|
patom->_ptr = val;
|
|
}
|
|
|
|
static inline void *patom_read(const patom_t *patom)
|
|
{
|
|
return patom->_ptr;
|
|
}
|
|
|
|
static __always_inline void *patom_xchg(patom_t *patom, void *val)
|
|
{
|
|
return (void *)latom_xchg((latom_t *)patom, (long)val);
|
|
}
|
|
|
|
static __always_inline void *patom_cmp_xchg(patom_t *patom, void *compare, void *val)
|
|
{
|
|
return (void *)latom_cmp_xchg((latom_t *)patom, (long)compare, (long)val);
|
|
}
|
|
|
|
static __always_inline void *patom_add(patom_t *patom, void *val)
|
|
{
|
|
return (void *)latom_add((latom_t *)patom, (long)val);
|
|
}
|
|
|
|
static __always_inline void *patom_sub(patom_t *patom, void *val)
|
|
{
|
|
return (void *)latom_sub((latom_t *)patom, (long)val);
|
|
}
|