@ -66,19 +66,18 @@ static inline u_int paddr_find_order(vm_paddr_t addr)
}
/** @brief Claim all free pages in one of the memory areas from the boot allocator. */
static inline void claim_bmem_area ( struct mm_zone * zone , _bmem_area * area )
static inline void claim_bmem_area ( struct mm_zone * zone , const struct _bmem_area * area )
{
vm_paddr_t start = area - > start ;
vm_paddr_t end = area - > end ;
vm_paddr_t pos = start ;
vm_size_t nr_pages = end - start / PAGE_SIZE ;
latom_add ( & zone - > free_count , ( long ) nr_pages ) ;
u_int order = paddr_find_order ( area - > start ) ;
while ( area - > start + ORDER_SIZE ( order ) > area - > end )
order - - ;
struct vm_page * const start = paddr2pg ( area - > start ) ;
struct vm_page * const end = paddr2pg ( area - > end ) ;
struct vm_page * pos = start ;
struct vm_page * page = & vm_page_array [ start > > PAGE_SHIFT ] ;
u_int order = paddr_find_order ( start ) ;
/* make sure the boot memory allocator cannot under any circumstances hand
* out pages from this area anymore , even though that should be unnecessary */
clist_del ( & area - > link ) ;
const vm_size_t nr_pages = end - > pfn - start - > pfn ;
latom_add ( & zone - > free_count , ( long ) nr_pages ) ;
/*
* We want to insert pages at the highest possible order . However , the
@ -90,15 +89,21 @@ static inline void claim_bmem_area(struct mm_zone *zone, struct _bmem_area *area
* subsequently lower the order again .
*/
while ( pos < end ) {
struct mm_pool * pool = & zone - > pools [ order ] ;
clist_add ( & pool - > freelist , & p age - > link ) ;
struct mm_pool * const pool = & zone - > pools [ order ] ;
clist_add ( & pool - > freelist , & p os - > link ) ;
pool - > free_entries + + ;
/* only the first page in the order group is inserted into
* the freelist , but all of them need to be initialized */
for ( u_int i = 0 ; i < ( 1 < < order ) ; i + + ) {
atom_init ( & page [ i ] . count , 0 ) ;
atom_init ( & page [ i ] . attr , 0 ) ;
for ( u_int i = 0 ; i < ( 1u < < order ) ; i + + ) {
if ( pos > = end )
panic ( " page %p out of range " , pos ) ;
if ( atom_read ( & pos - > count ) ! = 420 )
panic ( " page %p double initialized \n " , pos ) ;
atom_init ( & pos - > count , 0 ) ;
atom_init ( & pos - > attr , 0 ) ;
pos + + ;
}
/*
@ -111,22 +116,14 @@ static inline void claim_bmem_area(struct mm_zone *zone, struct _bmem_area *area
* | - - - - - - - - - - - - - - - - - - - - - | - - - - > pos
* start end
*/
pos + = ORDER_SIZE ( order ) ;
page + = ( 1 < < order ) ;
if ( order < MM_MAX_ORDER & & pos + ORDER_SIZE ( order ) < = end ) {
if ( order < MM_MAX_ORDER & & pos + ( 1 < < ( order + 1 ) ) < = end ) {
/* this makes the rising part of the graph */
order + + ;
} else if ( order > 0 & & pos > end ) {
/* we have overshot, lower the order */
pos - = ORDER_SIZE ( order ) ;
page - = ( 1 < < order ) ;
} else if ( order > 0 & & pos + ( 1 < < order ) > end ) {
/* this makes the abrupt downwards jump at the end of the graph */
while ( - - order ) {
if ( pos + ORDER_SIZE ( order ) < = end ) {
pos + = ORDER_SIZE ( order ) ;
page + = ( 1 < < order ) ;
if ( pos + ( 1 < < order ) < = end )
break ;
}
}
}
}
@ -141,7 +138,7 @@ void paging_init(vm_paddr_t phys_end)
usize bitmap_total_size = 0 ;
for ( int order = 0 ; order < MM_NR_ORDERS ; order + + ) {
usize pages = phys_end > > ORDER_SHIFT ( order + 1 ) ;
usize pages = phys_end > > ORDER_SHIFT ( order ) ;
pages = align_ceil ( pages , LATOM_BIT * 2 ) ;
usize bytes = pages / ( CHAR_BIT * 2 ) ;
bitmap_sizes [ order ] = bytes ;
@ -158,7 +155,7 @@ void paging_init(vm_paddr_t phys_end)
bitmap_size_log2 - - ; /* the bit index returned by flsl starts at 1 */
if ( bitmap_total_size ^ ( 1ul < < bitmap_size_log2 ) )
bitmap_size_log2 + + ; /* bitmap_total_size is not a power of 2, round up */
uintpt r_t bitmap_start_phys = __boot_pmalloc ( bitmap_size_log2 , MM_ZONE_NORMAL ) ;
vm_padd r_t bitmap_start_phys = __boot_pmalloc ( bitmap_size_log2 , MM_ZONE_NORMAL ) ;
panic_if ( bitmap_start_phys = = BOOT_PMALLOC_ERR ,
" cannot allocate memory for the page bitmaps " ) ;
memset ( __v ( bitmap_start_phys ) , 0 , bitmap_total_size ) ;
@ -168,12 +165,15 @@ void paging_init(vm_paddr_t phys_end)
*/
for ( int zone_index = 0 ; zone_index < ARRAY_SIZE ( mm_zones ) ; zone_index + + ) {
struct mm_zone * zone = & mm_zones [ zone_index ] ;
latom_init ( & zone - > free_count , 0 ) ;
/* we use the same bitmaps for all zones */
latom_t * bitmap_pos = __v ( bitmap_start_phys ) ;
for ( int order = 0 ; order < MM_NR_ORDERS ; order + + ) {
zone - > pools [ order ] . bitmap = bitmap_pos ;
clist_init ( & zone - > pools [ order ] . freelist ) ;
zone - > pools [ order ] . free_entries = 0 ;
latom_init ( & zone - > free_count , 0 ) ;
struct mm_pool * pool = & zone - > pools [ order ] ;
pool - > bitmap = bitmap_pos ;
pool - > free_entries = 0 ;
clist_init ( & pool - > freelist ) ;
spin_init ( & pool - > lock ) ;
bitmap_pos + = bitmap_sizes [ order ] ;
}
@ -188,12 +188,13 @@ void paging_init(vm_paddr_t phys_end)
*
* if the reserved bit is set , all other fields in the page are invalid .
*/
for ( u size i = 0 ; i < phys_end > > PAGE_SHIFT ; i + + ) {
for ( u _long pfn = 0 ; pfn < phys_end > > PAGE_SHIFT ; pfn + + ) {
/* This is merely an optimization to simplify checking whether
* two buddies can be coalesced into one . In reality , the
* reference count is invalid because the page is reserved . */
atom_init ( & vm_page_array [ i ] . count , 1 ) ;
atom_init ( & vm_page_array [ i ] . attr , _PGA_RSVD_MASK ) ;
atom_init ( & vm_page_array [ pfn ] . count , 420 ) ;
atom_init ( & vm_page_array [ pfn ] . attr , _PGA_RSVD_MASK ) ;
vm_page_array [ pfn ] . pfn = pfn ;
}
/*
@ -203,11 +204,15 @@ void paging_init(vm_paddr_t phys_end)
struct mm_zone * zone = & mm_zones [ i ] ;
struct _bmem_area * area , * tmp ;
clist_foreach_entry_safe ( & zone - > _bmem_areas , area , tmp , link ) {
/* make sure the boot memory allocator cannot under any circumstances hand
* out pages from this area anymore , even though that should be unnecessary */
clist_del ( & area - > link ) ;
claim_bmem_area ( zone , area ) ;
zone - > thrsh . emerg = latom_read ( & zone - > free_count ) / CFG_PAGE_EMERG_DENOM ;
if ( zone - > thrsh . emerg > CFG_PAGE_EMERG_MAX )
zone - > thrsh . emerg = CFG_PAGE_EMERG_MAX ;
}
zone - > thrsh . emerg = latom_read ( & zone - > free_count ) / CFG_PAGE_EMERG_DENOM ;
if ( zone - > thrsh . emerg > CFG_PAGE_EMERG_MAX )
zone - > thrsh . emerg = CFG_PAGE_EMERG_MAX ;
}
}
@ -218,22 +223,27 @@ static inline bool pg_flip_bit(struct mm_zone *zone, u_long pfn, u_int order)
return latom_flip_bit ( bitmap , ( int ) ( bit % LATOM_BIT ) ) ;
}
__malloc_like
static void * __get_pages ( u_int order , enum mflags flags )
vm_page_t page_alloc ( u_int order , enum mflags flags )
{
PAGE_ASSERT ( order > = 0 ) ;
struct mm_zone * zone = & mm_zones [ _M_ZONE_INDEX ( flags ) ] ;
if ( order > MM_MAX_ORDER ) {
page_debug ( " get_pages(%d, %#08x): Order too high! \n " , order , flags ) ;
return nil ;
}
u_long count_after = latom_sub ( & zone - > free_count , ( 1 < < order ) ) - ( 1 < < order ) ;
struct mm_zone * zone = & mm_zones [ _M_ZONE_INDEX ( flags ) ] ;
long count_after ;
try_next_zone :
count_after = latom_sub ( & zone - > free_count , ( 1 < < order ) ) - ( 1 < < order ) ;
if ( count_after < zone - > thrsh . emerg ) {
if ( count_after < 0 | | ! ( flags & _M_EMERG ) ) {
latom_add ( & zone - > free_count , ( 1 < < order ) ) ;
return nil ;
/* if we can't allocate from ZONE_NORMAL, fall back to ZONE_DMA */
if ( zone > & mm_zones [ 0 ] ) {
zone - - ;
goto try_next_zone ;
} else {
return nil ;
}
}
}
@ -266,93 +276,76 @@ static void *__get_pages(u_int order, enum mflags flags)
intr_restore ( cpuflags ) ;
}
if ( page = = nil ) {
if ( zone > & mm_zones [ 0 ] ) {
/*
* If we reach this , the current zone technically had enough free
* pages for the allocation , but those pages were split up into
* smaller chunks rather than a contiguous area . However , we don ' t
* give up quite yet : If possible , we fall back to a lower memory
* zone ( ZONE_NORMAL - > ZONE_DMA ) and start over from the top .
*/
zone - - ;
goto try_next_zone ;
} else {
return nil ;
}
}
/*
* if we found a page , check if we need to split it up
* ( which is the case if we took one from a higher order freelist )
*/
if ( page ! = nil ) {
usize pfn = pg2pfn ( page ) ;
page_debug_noisy ( " alloc order %u, split pfn %#lx from order %u \n " ,
order , pfn , page_order ) ;
pg_flip_bit ( zone , pfn , page_order ) ;
/* split the page and insert the upper halves into the
* respective freelist until we reach the requested order */
while ( page_order - - > order ) {
page_debug_noisy ( " split %p (order = %u) \n " , pfn2vaddr ( pfn ) , page_order ) ;
struct mm_pool * pool = & zone - > pools [ page_order ] ;
vm_page_t buddy = page + ( 1 < < page_order ) ;
pga_set_order ( buddy , page_order ) ;
pg_flip_bit ( zone , pfn + ( 1 < < page_order ) , page_order ) ;
disable_intr ( ) ;
spin_lock ( & pool - > lock ) ;
clist_add_first ( & pool - > freelist , & buddy - > link ) ;
pool - > free_entries + + ;
spin_unlock ( & pool - > lock ) ;
intr_restore ( cpuflags ) ;
}
pga_set_order ( page , order ) ;
void * vaddr = pfn2vaddr ( pfn ) ;
usize pfn = pg2pfn ( page ) ;
page_debug_noisy ( " alloc order %u, split pfn %#lx from order %u \n " ,
order , pfn , page_order ) ;
pg_flip_bit ( zone , pfn , page_order ) ;
/* split the page and insert the upper halves into the
* respective freelist until we reach the requested order */
while ( page_order - - > order ) {
page_debug_noisy ( " split %p (order = %u) \n " , pfn2vaddr ( pfn ) , page_order ) ;
struct mm_pool * pool = & zone - > pools [ page_order ] ;
vm_page_t buddy = page + ( 1 < < page_order ) ;
pga_set_order ( buddy , page_order ) ;
pg_flip_bit ( zone , pfn + ( 1 < < page_order ) , page_order ) ;
return vaddr ;
} else {
return nil ;
disable_intr ( ) ;
spin_lock ( & pool - > lock ) ;
clist_add_first ( & pool - > freelist , & buddy - > link ) ;
pool - > free_entries + + ;
spin_unlock ( & pool - > lock ) ;
intr_restore ( cpuflags ) ;
}
}
/* faster memset for whole pages */
static inline void init_pages ( u_long * start , u_long val , u_int order )
{
u_long * end = start + ( ORDER_SIZE ( order ) / sizeof ( * start ) ) ;
do {
* start + + = val ;
} while ( start ! = end ) ;
for ( u_int i = 0 ; i < ( 1 < < order ) ; i + + )
pga_set_order ( & page [ i ] , order ) ;
page_clear ( page ) ;
return page ;
}
/*
* XXX get_page ( ) and get_pages ( ) shouldn ' t depend on the direct map
*
* XXX Do we need these at all ? I don ' t think so .
*/
void * get_pages ( u_int order , enum mflags flags )
{
void * pages = __get_pages ( order , flags ) ;
# if CFG_POISON_PAGES
if ( pages ! = nil )
init_pages ( pages , PAGE_POISON_ALLOC , order ) ;
# endif
return pages ;
vm_page_t page = page_alloc ( order , flags ) ;
if ( page )
return pfn2vaddr ( pg2pfn ( page ) ) ;
else
return nil ;
}
void * get_page ( enum mflags flags )
{
void * pages = __get_pages ( 0 , flags ) ;
# if CFG_POISON_PAGES
if ( pages ! = nil )
init_pages ( pages , PAGE_POISON_ALLOC , 0 ) ;
# endif
return pages ;
}
void * get_zero_pages ( u_int order , enum mflags flags )
{
void * pages = __get_pages ( order , flags ) ;
if ( pages ! = nil )
init_pages ( pages , 0 , order ) ;
return pages ;
}
void * get_zero_page ( enum mflags flags )
{
void * page = __get_pages ( 0 , flags ) ;
if ( page ! = nil )
init_pages ( page , 0 , 0 ) ;
return page ;
vm_page_t page = page_alloc ( 0 , flags ) ;
if ( page )
return pfn2vaddr ( pg2pfn ( page ) ) ;
else
return nil ;
}
/*
@ -377,26 +370,13 @@ static __always_inline bool can_merge(vm_page_t page, vm_page_t buddy)
return merge ;
}
void free_pages( void * ptr )
void page_free( vm_page_t page )
{
PAGE_DEBUG_BLOCK {
if ( ptr < DMAP_START | | ptr > = DMAP_END ) {
panic ( " free_pages(%p): not in DMAP region \n " , ptr ) ;
}
}
register_t cpuflags = read_flags ( ) ;
vm_page_t page = vaddr2pg ( ptr ) ;
panic_if ( pga_rsvd ( page ) , " tried to free reserved page %p " , ptr ) ;
u_int order = pga_order ( page ) ;
PAGE_ASSERT ( ( uintptr_t ) ptr % ORDER_SIZE ( order ) = = 0 ) ;
u_long pfn = vaddr2pfn ( ptr ) ;
# if CFG_POISON_PAGES
init_pages ( ptr , PAGE_POISON_FREE , order ) ;
# endif
u_long pfn = pg2pfn ( page ) ;
PAGE_DEBUG_BLOCK {
int old_count = atom_sub ( & page - > count , 1 ) ;
@ -407,6 +387,8 @@ void free_pages(void *ptr)
page_debug ( " attempted to free %p with references " , ptr ) ;
return ;
}
} else {
atom_dec ( & page - > count ) ;
}
struct mm_zone * zone = & mm_zones [ pga_zone ( page ) ] ;
