@ -8,6 +8,7 @@
# include <gay/kprintf.h> # include <gay/kprintf.h>
# include <gay/mm.h> # include <gay/mm.h>
# include <gay/mutex.h> # include <gay/mutex.h>
# include <gay/systm.h>
# include <gay/types.h> # include <gay/types.h>
# include <gay/util.h> # include <gay/util.h>
@ -37,6 +38,7 @@
# endif # endif
# if CFG_DEBUG_PAGE_ALLOCS # if CFG_DEBUG_PAGE_ALLOCS
# define PAGE_ASSERT(x) KASSERT(x)
# define page_debug(msg, ...) kprintf("[page] " msg, ##__VA_ARGS__) # define page_debug(msg, ...) kprintf("[page] " msg, ##__VA_ARGS__)
# if CFG_DEBUG_PAGE_ALLOCS_NOISY # if CFG_DEBUG_PAGE_ALLOCS_NOISY
# define page_debug_noisy(msg, ...) kprintf("[page] " msg, ##__VA_ARGS__) # define page_debug_noisy(msg, ...) kprintf("[page] " msg, ##__VA_ARGS__)
@ -44,6 +46,7 @@
# define page_debug_noisy(msg, ...) ({}) # define page_debug_noisy(msg, ...) ({})
# endif # endif
# else # else
# define PAGE_ASSERT(x) ({})
# define page_debug(msg, ...) ({}) # define page_debug(msg, ...) ({})
# define page_debug_noisy(msg, ...) ({}) # define page_debug_noisy(msg, ...) ({})
# endif # endif
@ -54,14 +57,14 @@
* the one below it , starting at one page per entry . The effective result is
* the one below it , starting at one page per entry . The effective result is
* that a single entry in the cache on level L covers ` ( 1 < < L ) ` pages .
* that a single entry in the cache on level L covers ` ( 1 < < L ) ` pages .
*/
*/
# define CACHE_ LEVELS GET_PAGE_LEVEL S# define CACHE_ ORDERS GET_PAGE_ORDER S
# define LEVEL_SHIFT(level) (PAGE_SHIFT + (level ))# define ORDER_SHIFT(order) (PAGE_SHIFT + (order ))
/** @brief There is one of this for every cache level . *//** @brief There is one of this for every cache order . */
struct cache_pool { struct cache_pool {
/**
/**
* @ brief List of free blocks on this level of granularity .
* @ brief List of free blocks on this order of granularity .
* The individual entries sit right at the beginning of each free block ,
* The individual entries sit right at the beginning of each free block ,
* and are always aligned to ` entry_size ` bytes .
* and are always aligned to ` entry_size ` bytes .
*/
*/
@ -74,11 +77,9 @@ struct cache_pool {
/** @brief Number of items in `freelist`. */
/** @brief Number of items in `freelist`. */
usize free_entries ;
usize free_entries ;
} ; } ;
static struct cache_pool caches [ CACHE_ LEVEL S] ; static struct cache_pool caches [ CACHE_ ORDER S] ;
static MTX ( caches_lock ) ; static MTX ( caches_lock ) ;
# define LONG_BIT_MASK (~(LONG_BIT - 1))
/* these get set in kmalloc_init() */ /* these get set in kmalloc_init() */
uintptr_t phys_start ; uintptr_t phys_start ;
uintptr_t phys_end ; uintptr_t phys_end ;
@ -91,28 +92,48 @@ uintptr_t __early_get_page(void)
static int sanity_check ( void ) static int sanity_check ( void )
{ {
KASSERT ( phys_start < phys_end ) ;
KASSERT ( phys_start = = HUGEPAGE_ALIGN ( phys_start ) ) ;
/* phys_end is only page aligned, see kmalloc_init() */
/* phys_end is only page aligned, see kmalloc_init() */
if ( phys_end ! = PAGE_ALIGN ( phys_end ) | | phys_start ! = HUGEPAGE_ALIGN ( phys_start ) ) {
KASSERT ( phys_end = = PAGE_ALIGN ( phys_end ) ) ;
kprintf ( " Unaligned memory, this should never be possible \n " ) ;
return 1 ;
}
if ( ( phys_end - phys_start ) < ( 32 * 1024 * 1024 ) ) {
if ( ( phys_end - phys_start ) < ( 32 * 1024 * 1024 ) ) {
kprintf ( " Less than 32 MB of usable RAM, this wouldn't go well \n " ) ;
kprintf ( " Less than 32 MB of usable RAM, this wouldn't go well \n " ) ;
return 1 ;
return 1 ;
}
}
if ( phys_start > phys_end ) {
kprintf ( " Hey, this is funny. pages_init() was called with parameters "
" such that phys_start > phys_end (%p > %p), which "
" should absolutely never be possible. I can't really continue "
" like this, so have a nice day. \n " , ( void * ) phys_start , ( void * ) phys_end ) ;
return 1 ;
}
return 0 ;
return 0 ;
} }
/*
* Map the entire physical memory into the direct contiguous area .
* __early_map_page ( ) might call __early_get_page ( ) in order to allocate
* new page table structures , which in turn shrinks the physical memory
* size ( see above ) .
*/
static inline void map_direct_area ( void )
{
# ifdef __HAVE_HUGEPAGES
const usize step = HUGEPAGE_SIZE ;
const enum pflags flags = P_PRESENT | P_RW | P_HUGE ;
# else
const usize step = PAGE_SIZE ;
const enum pflags flags = P_PRESENT | P_RW ;
# endif
/*
* It might be necessary to use a volatile pointer to phys_end for this
* loop in case clang does The Optimization and caches its value for
* whatever reason , even though at least for x86 this is not the case
* ( and i don ' t even thing the C standard allows it when calling
* external functions in between , but still , Never Trust The Compiler ) .
*/
for ( uintptr_t pos = phys_start ; pos < = phys_end - step ; pos + = step )
__early_map_page ( pos , __v ( pos ) , flags ) ;
vm_flush ( ) ;
}
/*
/*
* This function maps the entire physical memory into the direct region
* This function maps the entire physical memory into the direct region
* ( DMAP_START - DMAP_END ) and sets up the caches .
* ( DMAP_START - DMAP_END ) and sets up the caches .
@ -124,18 +145,7 @@ int pages_init(void)
if ( sanity_check ( ) ! = 0 )
if ( sanity_check ( ) ! = 0 )
return 1 ;
return 1 ;
/*
map_direct_area ( ) ;
* Map the entire physical memory into the direct contiguous area .
* __early_map_page ( ) might call __early_get_page ( ) in order to allocate
* new page table structures , which in turn shrinks the physical memory
* size ( see above ) .
* It might be necessary to use a volatile pointer to phys_end for this
* loop in case clang does The Optimization and caches its value for
* whatever reason , even though at least for x86 this is not the case .
*/
for ( uintptr_t physptr = phys_start ; physptr < phys_end ; physptr + = HUGEPAGE_SIZE )
__early_map_page ( physptr , __v ( physptr ) , PFLAG_HUGE | PFLAG_RW | PFLAG_GLOBAL ) ;
vm_flush ( ) ;
/* phys_end gets aligned, as promised by the comment in kmalloc_init() */
/* phys_end gets aligned, as promised by the comment in kmalloc_init() */
phys_end = align_floor ( phys_end , HUGEPAGE_SIZE ) ;
phys_end = align_floor ( phys_end , HUGEPAGE_SIZE ) ;
@ -145,13 +155,9 @@ int pages_init(void)
* calculate the size of each bitmap , as well as their combined size
* calculate the size of each bitmap , as well as their combined size
*/
*/
usize bitmap_bytes = 0 ;
usize bitmap_bytes = 0 ;
for ( int i = 0 ; i < CACHE_LEVELS ; i + + ) {
for ( int i = 0 ; i < CACHE_ORDERS ; i + + ) {
usize bits = phys_size > > LEVEL_SHIFT ( i ) ;
usize bits = phys_size > > ORDER_SHIFT ( i ) ;
/* round up to the next full long */
bits = align_ceil ( bits , LONG_BIT ) ;
if ( bits & ~ LONG_BIT_MASK ) {
bits & = LONG_BIT_MASK ;
bits + = LONG_BIT ;
}
bitmap_bytes + = bits / 8 ;
bitmap_bytes + = bits / 8 ;
}
}
@ -169,11 +175,11 @@ int pages_init(void)
* preallocate entries that can ' t be handed out ( i . e . the cache bitmaps )
* preallocate entries that can ' t be handed out ( i . e . the cache bitmaps )
*/
*/
unsigned long * bitmap_pos = bitmap_start ;
unsigned long * bitmap_pos = bitmap_start ;
for ( int i = 0 ; i < CACHE_ LEVEL S; i + + ) {
for ( int i = 0 ; i < CACHE_ ORDER S; i + + ) {
/* total amount of entries on this level */
/* total amount of entries on this level */
usize total_bits = phys_size > > LEVEL _SHIFT( i ) ;
usize total_bits = phys_size > > ORDER _SHIFT( i ) ;
/* number of entries on this level that the bitmap itself takes up */
/* number of entries on this level that the bitmap itself takes up */
usize wasted_bits = bitmap_bytes > > LEVEL _SHIFT( i ) ;
usize wasted_bits = bitmap_bytes > > ORDER _SHIFT( i ) ;
if ( wasted_bits = = 0 )
if ( wasted_bits = = 0 )
wasted_bits = 1 ;
wasted_bits = 1 ;
bit_set_range ( bitmap_pos , total_bits - wasted_bits , wasted_bits ) ;
bit_set_range ( bitmap_pos , total_bits - wasted_bits , wasted_bits ) ;
@ -190,11 +196,11 @@ int pages_init(void)
kheap_end = align_floor ( bitmap_start , HUGEPAGE_SIZE ) ;
kheap_end = align_floor ( bitmap_start , HUGEPAGE_SIZE ) ;
/*
/*
* populate the freelist on the highest level, all level s beneath it
* populate the freelist on the highest order, all order s beneath it
* stay empty until one of the large blocks gets split up
* stay empty until one of the large blocks gets split up
*/
*/
struct cache_pool * high_pool = & caches [ CACHE_ LEVEL S - 1 ] ;
struct cache_pool * high_pool = & caches [ CACHE_ ORDER S - 1 ] ;
usize step = 1 < < LEVEL_SHIFT( CACHE_LEVEL S - 1 ) ;
usize step = 1 < < ORDER_SHIFT( CACHE_ORDER S - 1 ) ;
for ( void * pos = kheap_start ; pos < kheap_end ; pos + = step ) {
for ( void * pos = kheap_start ; pos < kheap_end ; pos + = step ) {
struct clist * entry = pos ;
struct clist * entry = pos ;
clist_add ( & high_pool - > freelist , entry ) ;
clist_add ( & high_pool - > freelist , entry ) ;
@ -218,62 +224,62 @@ static void *split_buddy(void *ptr, int level);
/**
/**
* @ brief Attempt to coalesce a block with its buddy .
* @ brief Attempt to coalesce a block with its buddy .
* If coalition is possible , the buddy is removed from its freelist at ` level ` .
* If coalition is possible , the buddy is removed from its freelist at ` order ` .
*
*
* @ param ptr Pointer to the block
* @ param ptr Pointer to the block
* @ param level Cache level , must be less than ` CACHE_LEVEL S - 1 ` ( because you
* @ param order Cache order , must be less than ` CACHE_ORDER S - 1 ` ( because you
* can ' t join blocks at the highest cache level )
* can ' t join blocks at the highest cache order )
* @ return The joined block , or ` nil ` if coalition was not possible
* @ return The joined block , or ` nil ` if coalition was not possible
*/
*/
static void * try_join_buddy ( void * ptr , int level ) ; static void * try_join_buddy ( void * ptr , int order ) ;
static inline usize get_bit_number ( void * ptr , int level ) static inline usize get_bit_number ( void * ptr , int order )
{ {
return ( ( uintptr_t ) ptr - ( uintptr_t ) kheap_start ) > > LEVEL_SHIFT( level ) ;
return ( ( uintptr_t ) ptr - ( uintptr_t ) kheap_start ) > > ORDER_SHIFT( order ) ;
} }
void * get_pages ( usize count , enum mm_ flags ) void * get_pages ( int order , enum m flags )
{ {
int level ;
PAGE_ASSERT ( order > = 0 ) ;
for ( level = 0 ; level < CACHE_LEVELS ; level + + ) {
if ( ( 1 < < level ) > = count )
if ( order > = GET_PAGE_ORDERS ) {
break ;
page_debug ( " get_pages(%d, %#08x): Order too high! \n " , order , flags ) ;
}
if ( level = = CACHE_LEVELS ) {
page_debug ( " get_pages(%zu, %08x): count too large! \n " , count , flags ) ;
return nil ;
return nil ;
}
}
if ( flags & M M _NOSLEEP) {
if ( flags & M _NOSLEEP) {
kprintf ( " get_pages(): M M _NOSLEEP requested, this is not implemented yet :(\n " ) ;
kprintf ( " get_pages(): M _NOSLEEP requested, this is not implemented yet :(\n " ) ;
return nil ;
return nil ;
}
}
mtx_lock ( & caches_lock ) ;
mtx_lock ( & caches_lock ) ;
struct clist * entry = nil ;
struct clist * entry = nil ;
int entry_ level ;
int entry_ order ;
for ( entry_ level = level ; entry_level < CACHE_LEVELS ; entry_level + + ) {
for ( entry_ order = order ; entry_order < CACHE_ORDERS ; entry_order + + ) {
if ( caches [ entry_ level ] . free_entries > 0 ) {
if ( caches [ entry_ order ] . free_entries > 0 ) {
entry = caches [ entry_ level ] . freelist . next ;
entry = caches [ entry_ order ] . freelist . next ;
break ;
break ;
}
}
}
}
if ( entry_level = = CACHE_LEVELS )
goto unlock ;
if ( entry_order ! = CACHE_ORDERS ) {
clist_del ( entry ) ;
clist_del ( entry ) ;
caches [ entry_order ] . free_entries - - ;
caches [ entry_level ] . free_entries - - ;
usize bit_number = get_bit_number ( entry , entry_order ) ;
usize bit_number = get_bit_number ( entry , entry_level ) ;
while ( entry_order > order ) {
while ( entry_level > level ) {
entry = split_buddy ( entry , entry_order ) ;
entry = split_buddy ( entry , entry_level ) ;
bit_set ( caches [ entry_order ] . bitmap , bit_number ) ;
bit_set ( caches [ entry_level ] . bitmap , bit_number ) ;
entry_order - - ;
entry_level - - ;
bit_number < < = 1 ;
bit_number < < = 1 ;
}
bit_set ( caches [ order ] . bitmap , bit_number ) ;
# if CFG_POISON_PAGES
memset ( entry , ' a ' , 1 < < ORDER_SHIFT ( order ) ) ;
# endif
}
}
bit_set ( caches [ level ] . bitmap , bit_number ) ;
unlock :
mtx_unlock ( & caches_lock ) ;
mtx_unlock ( & caches_lock ) ;
return ( void * ) entry ;
return ( void * ) entry ;
} }
@ -287,54 +293,66 @@ void free_pages(void *ptr)
}
}
# endif # endif
mtx_lock ( & caches_lock ) ;
if ( sus_nil ( ptr ) ) {
page_debug ( " free_pages(%p): tried to free NULL! \n " , ptr ) ;
return ;
}
int level = 0 ;
int order = 0 ;
usize bit_number = get_bit_number ( ptr , level ) ;
usize bit_number = get_bit_number ( ptr , order ) ;
for ( ; level < CACHE_LEVELS ; level + + ) {
for ( ; order < CACHE_ORDERS ; order + + ) {
if ( bit_tst ( caches [ level ] . bitmap , bit_number ) )
if ( bit_tst ( caches [ order ] . bitmap , bit_number ) )
break ;
break ;
bit_number > > = 1 ;
bit_number > > = 1 ;
}
}
if ( level = = CACHE_LEVEL S) {
if ( order = = CACHE_ORDER S) {
page_debug ( " free_pages(%p): double free! \n " , ptr ) ;
page_debug ( " free_pages(%p): double free! \n " , ptr ) ;
goto unlock ;
return ;
}
}
int original_order = order ;
while ( level < CACHE_LEVELS - 1 ) {
mtx_lock ( & caches_lock ) ;
bit_clr ( caches [ level ] . bitmap , bit_number ) ;
while ( order < CACHE_ORDERS - 1 ) {
bit_clr ( caches [ order ] . bitmap , bit_number ) ;
void * tmp = try_join_buddy ( ptr , level ) ;
void * tmp = try_join_buddy ( ptr , order ) ;
if ( tmp = = nil )
if ( tmp = = nil )
break ;
break ;
ptr = tmp ;
ptr = tmp ;
level + + ;
order + + ;
bit_number > > = 1 ;
bit_number > > = 1 ;
}
}
clist_add ( & caches [ level ] . freelist , ( struct clist * ) ptr ) ;
if ( order = = CACHE_ORDERS - 1 & & original_order ! = CACHE_ORDERS - 1 )
caches [ level ] . free_entries + + ;
set_pflags ( HUGEPAGE_ALIGN ( ptr ) , P_HUGE | P_RW ) ;
# if CFG_POISON_PAGES
memset ( ptr , ' A ' , 1 < < ORDER_SHIFT ( order ) ) ;
# endif
clist_add ( & caches [ order ] . freelist , ( struct clist * ) ptr ) ;
caches [ order ] . free_entries + + ;
unlock :
mtx_unlock ( & caches_lock ) ;
mtx_unlock ( & caches_lock ) ;
} }
static inline void * split_buddy ( void * ptr , int level ) static inline void * split_buddy ( void * ptr , int level )
{ {
# if CFG_DEBUG_PAGE_ALLOCS # if CFG_DEBUG_PAGE_ALLOCS
if ( ( uintptr_t ) ptr % ( 1 < < LEVEL _SHIFT( level ) ) ) {
if ( ( uintptr_t ) ptr % ( 1 < < ORDER _SHIFT( level ) ) ) {
kprintf ( " split_buddy(ptr = %p, level = %d): unaligned ptr! \n " , ptr , level ) ;
kprintf ( " split_buddy(ptr = %p, level = %d): unaligned ptr! \n " , ptr , level ) ;
return nil ;
return nil ;
}
}
if ( level < 1 | | level > = CACHE_ LEVEL S) {
if ( level < 1 | | level > = CACHE_ ORDER S) {
kprintf ( " split_buddy(ptr = %p, level = %d): invalid level! \n " , ptr , level ) ;
kprintf ( " split_buddy(ptr = %p, level = %d): invalid level! \n " , ptr , level ) ;
return nil ;
return nil ;
}
}
# endif # endif
struct clist * high_buddy = ptr + ( 1 < < LEVEL _SHIFT( level - 1 ) ) ;
struct clist * high_buddy = ptr + ( 1 < < ORDER _SHIFT( level - 1 ) ) ;
clist_add ( & caches [ level - 1 ] . freelist , high_buddy ) ;
clist_add ( & caches [ level - 1 ] . freelist , high_buddy ) ;
caches [ level - 1 ] . free_entries + + ;
caches [ level - 1 ] . free_entries + + ;
@ -343,19 +361,19 @@ static inline void *split_buddy(void *ptr, int level)
return ptr ;
return ptr ;
} }
static void * try_join_buddy ( void * ptr , int level ) static void * try_join_buddy ( void * ptr , int order )
{ {
const usize entry_size = 1 < < LEVEL_SHIFT( level ) ;
const usize entry_size = 1 < < ORDER_SHIFT( order ) ;
# if CFG_DEBUG_PAGE_ALLOCS # if CFG_DEBUG_PAGE_ALLOCS
if ( ( uintptr_t ) ptr % entry_size ) {
if ( ( uintptr_t ) ptr % entry_size ) {
kprintf ( " try_join_buddy(%p, %d): unaligned ptr! \n " , ptr , level ) ;
kprintf ( " try_join_buddy(%p, %d): unaligned ptr! \n " , ptr , order ) ;
return nil ;
return nil ;
}
}
/* level must be < CACHE_LEVEL S - 1 because you
/* order must be < CACHE_ORDER S - 1 because you
* can ' t join blocks on the topmost level */
* can ' t join blocks on the topmost order */
if ( level > = CACHE_LEVEL S - 1 ) {
if ( order > = CACHE_ORDER S - 1 ) {
kprintf ( " try_join_buddy(%p, %d): level >= CACHE_LEVELS - 1!\n " , ptr , level ) ;
kprintf ( " try_join_buddy(%p, %d): order >= CACHE_ORDERS - 1!\n " , ptr , order ) ;
return nil ;
return nil ;
}
}
# endif # endif
@ -367,15 +385,15 @@ static void *try_join_buddy(void *ptr, int level)
* for any if branches .
* for any if branches .
*/
*/
uintptr_t buddy = ( uintptr_t ) ptr ^ entry_size ;
uintptr_t buddy = ( uintptr_t ) ptr ^ entry_size ;
usize buddy_bitnum = get_bit_number ( ( void * ) buddy , level ) ;
usize buddy_bitnum = get_bit_number ( ( void * ) buddy , order ) ;
if ( bit_tst ( caches [ level ] . bitmap , buddy_bitnum ) )
if ( bit_tst ( caches [ order ] . bitmap , buddy_bitnum ) )
return nil ;
return nil ;
page_debug_noisy ( " join (%p:%p), lvl =%d\n " , ptr , ( void * ) buddy , level ) ;
page_debug_noisy ( " join (%p:%p), order =%d\n " , ptr , ( void * ) buddy , order ) ;
/* If the buddy is free, we remove it from the freelist ... */
/* If the buddy is free, we remove it from the freelist ... */
clist_del ( ( struct clist * ) buddy ) ;
clist_del ( ( struct clist * ) buddy ) ;
caches [ level ] . free_entries - - ;
caches [ order ] . free_entries - - ;
/*
/*
* . . . and return a pointer to the coalesced block .
* . . . and return a pointer to the coalesced block .
