kmalloc: add shiny new buddy page frame allocator
This is still kind of a work in progress because it will be the backend to a slab allocator, which in turn is managed by kmalloc().main
parent
f0706b802b
commit
96378f019c
GPG Key ID:
EC22E476DC2D3D84
@ -0,0 +1,464 @@
|
||||
/* See the end of this file for copyright and license terms. */
|
||||
|
||||
#include <arch/page.h>
|
||||
|
||||
#include <gay/arith.h>
|
||||
#include <gay/cdefs.h>
|
||||
#include <gay/clist.h>
|
||||
#include <gay/config.h>
|
||||
#include <gay/errno.h>
|
||||
#include <gay/kprintf.h>
|
||||
#include <gay/mm.h>
|
||||
#include <gay/types.h>
|
||||
|
||||
#include <string.h>
|
||||
|
||||
/*
|
||||
* This allocator is based on the popular design by Doug Lea:
|
||||
* <http://gee.cs.oswego.edu/dl/html/malloc.html>
|
||||
* For a more in-depth description of how the individual parts work together,
|
||||
* see also my implementation for Ardix which is very similar except that it
|
||||
* doesn't have paging:
|
||||
* <https://git.bsd.gay/fef/ardix/src/commit/c767d551d3301fc30f9fce30eda8f04e2f9a42ab/kernel/mm.c>
|
||||
* As a matter of fact, this allocator is merely an extension of the one from
|
||||
* Ardix with the only difference being that the heap can be extended upwards.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Memory block header.
|
||||
* This sits at the beginning of every memory block (duh).
|
||||
*/
|
||||
struct memblk {
|
||||
/**
|
||||
* @brief The usable low_size, i.e. the total block low_size minus `MEMBLK_OVERHEAD`.
|
||||
*
|
||||
* This low_size will also be written to the very end of the block, just after
|
||||
* the last usable address. Additionally, since blocks are always aligned
|
||||
* to at least 4 bytes anyways, we can use the LSB of this low_size as a flag
|
||||
* for whether the block is currently allocated (1) or not (0). This is
|
||||
* going to make it much easier to detect two free neighboring blocks when
|
||||
* `kfree()`ing one.
|
||||
*/
|
||||
usize low_size[1];
|
||||
|
||||
union {
|
||||
/** @brief If the block is allocated, this will be overwritten */
|
||||
struct clist clink;
|
||||
|
||||
/** @brief Used as the return value for `kmalloc()` */
|
||||
u8 data[0];
|
||||
/**
|
||||
* @brief Used to get the copy of the size at the end of
|
||||
* the block, right after the last byte of `data`
|
||||
*/
|
||||
usize high_size[0];
|
||||
};
|
||||
};
|
||||
|
||||
/* overhead per memory block in bytes (the two block sizes at the beginning and end) */
|
||||
#define OVERHEAD (2 * sizeof(usize))
|
||||
/* every allocation is padded to a multiple of this */
|
||||
#define MIN_SIZE (sizeof(struct clist))
|
||||
|
||||
/* memory blocks, sorted by increasing size */
|
||||
static CLIST(blocks);
|
||||
|
||||
/*
|
||||
* We play it *really* simple: Start at an arbitrary (page aligned, preferably
|
||||
* even page table aligned) address in virtual memory and extend the area as
|
||||
* needed as the heap grows. Efficiency doesn't matter for now; we always make
|
||||
* the heap a contiguous area without holes. There isn't even a mechanism for
|
||||
* releasing physical pages yet, i really just want to get to anything that is
|
||||
* at all usable so i can finally work on the core system architecture.
|
||||
*/
|
||||
static void *heap_start = (void *)0xd0000000;
|
||||
/*
|
||||
* Points to the first address that is not part of the heap anymore, such that
|
||||
* sizeof(heap) == heap_end - heap_start
|
||||
* Thus, the heap initially has a size of zero.
|
||||
*/
|
||||
static void *heap_end = (void *)0xd0000000;
|
||||
|
||||
/**
|
||||
* @brief Increase `heap_end` by up to `num_pages * PAGE_SIZE`.
|
||||
*
|
||||
* @param num_pages Number of pages to increase the heap by
|
||||
* @returns The actual number of pages the heap was increased by; this may be
|
||||
* less than `num_pages` if there were not enough free pages left
|
||||
*/
|
||||
static usize grow_heap(usize num_pages);
|
||||
/**
|
||||
* @brief Add a new block at the end of the heap by downloading more RAM (`grow_heap()`, actually). */
|
||||
static struct memblk *blk_create(usize num_pages);
|
||||
/** @brief Get the usable block size in bytes, without flags or overhead. */
|
||||
static usize blk_get_size(struct memblk *blk);
|
||||
/** @brief Set the usable block size without overhead and without affecting flags. */
|
||||
static void blk_set_size(struct memblk *blk, usize size);
|
||||
/** @brief Flag a block as allocated. */
|
||||
static void blk_set_alloc(struct memblk *blk);
|
||||
/** @brief Remove the allocated flag from a block. */
|
||||
static void blk_clear_alloc(struct memblk *blk);
|
||||
/** @brief Return nonzero if the block is allocated. */
|
||||
static bool blk_is_alloc(struct memblk *blk);
|
||||
/** @brief Set the border flag at the start of a block. */
|
||||
static void blk_set_border_start(struct memblk *blk);
|
||||
/** @brief Remove the border flag from the start of a block. */
|
||||
static void blk_clear_border_start(struct memblk *blk);
|
||||
/** @brief Return nonzero if a block has the border flag set at the start. */
|
||||
static bool blk_is_border_start(struct memblk *blk);
|
||||
/** @brief Set the border flag at the end of a block. */
|
||||
static void blk_set_border_end(struct memblk *blk);
|
||||
/** @brief Remove the border flag from the end of a block. */
|
||||
static void blk_clear_border_end(struct memblk *blk);
|
||||
/** @brief Return nonzero if a block has the border flag set at the end. */
|
||||
static bool blk_is_border_end(struct memblk *blk);
|
||||
/** @brief Get a block's immediate lower neighbor, or NULL if it doesn't have one. */
|
||||
static struct memblk *blk_prev(struct memblk *blk);
|
||||
/** @brief Get a block's immediate higher neighbor, or NULL if it doesn't have one. */
|
||||
static struct memblk *blk_next(struct memblk *blk);
|
||||
/** @brief Merge two contiguous free blocks into one, resort the list, and return the block. */
|
||||
static struct memblk *blk_merge(struct memblk *bottom, struct memblk *top);
|
||||
/** @brief Attempt to merge both the lower and higher neighbors of a free block. */
|
||||
static struct memblk *blk_try_merge(struct memblk *blk);
|
||||
/** @brief Cut a slice from a free block and return the slice. */
|
||||
static struct memblk *blk_slice(struct memblk *blk, usize slice_size);
|
||||
|
||||
int kmalloc_init(uintptr_t phys_start, uintptr_t phys_end)
|
||||
{
|
||||
int err = mem_init(phys_start, phys_end);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (grow_heap(1) != 1)
|
||||
return -ENOMEM;
|
||||
|
||||
struct memblk *blk = heap_start;
|
||||
blk_set_size(blk, PAGE_SIZE - OVERHEAD);
|
||||
blk_clear_alloc(blk);
|
||||
blk_set_border_start(blk);
|
||||
blk_set_border_end(blk);
|
||||
clist_add(&blocks, &blk->clink);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void *kmalloc(usize size, enum mm_flags flags)
|
||||
{
|
||||
if (flags != MM_KERN) {
|
||||
kprintf("Invalid flags passed to kmalloc()\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (size == 0)
|
||||
return NULL;
|
||||
|
||||
if (size % MIN_SIZE != 0)
|
||||
size = (size / MIN_SIZE) * MIN_SIZE + MIN_SIZE;
|
||||
|
||||
struct memblk *cursor;
|
||||
struct memblk *blk = NULL;
|
||||
clist_foreach_entry(&blocks, cursor, clink) {
|
||||
if (blk_get_size(cursor) >= size) {
|
||||
blk = cursor;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (blk == NULL) {
|
||||
usize required_pages = ((size + OVERHEAD) / PAGE_SIZE) + 1;
|
||||
blk = blk_create(required_pages);
|
||||
if (blk == NULL) {
|
||||
kprintf("Kernel OOM qwq\n");
|
||||
return NULL;
|
||||
}
|
||||
clist_add(&blocks, &blk->clink);
|
||||
}
|
||||
|
||||
blk = blk_slice(blk, size);
|
||||
blk_set_alloc(blk);
|
||||
# if CFG_POISON_HEAP
|
||||
memset(blk->data, 'a', blk_get_size(blk));
|
||||
# endif
|
||||
return blk->data;
|
||||
}
|
||||
|
||||
void kfree(void *ptr)
|
||||
{
|
||||
# ifdef DEBUG
|
||||
if (ptr < heap_start || ptr > heap_end) {
|
||||
kprintf("Tried to free %p which is outside the heap!\n", ptr);
|
||||
return;
|
||||
}
|
||||
# endif
|
||||
|
||||
struct memblk *blk = ptr - sizeof(blk->low_size);
|
||||
# ifdef DEBUG
|
||||
if (!blk_is_alloc(blk)) {
|
||||
kprintf("Double free of %p!\n", ptr);
|
||||
return;
|
||||
}
|
||||
# endif
|
||||
|
||||
# if CFG_POISON_HEAP
|
||||
memset(blk->data, 'A', blk_get_size(blk));
|
||||
# endif
|
||||
|
||||
blk_clear_alloc(blk);
|
||||
blk_try_merge(blk);
|
||||
}
|
||||
|
||||
/*
|
||||
* These wrappers are used for linking libc against the kernel itself.
|
||||
* This is a "temporary" hack because i haven't figured out the whole C flags
|
||||
* thingy for properly producing two versions of libc (one static one for the
|
||||
* kernel and a shared one for user space).
|
||||
*/
|
||||
|
||||
__weak void *malloc(usize size)
|
||||
{
|
||||
return kmalloc(size, MM_KERN);
|
||||
}
|
||||
|
||||
__weak void free(void *ptr)
|
||||
{
|
||||
kfree(ptr);
|
||||
}
|
||||
|
||||
static inline struct memblk *blk_create(usize num_pages)
|
||||
{
|
||||
usize blksize;
|
||||
if (mul_overflow(&blksize, num_pages, PAGE_SIZE))
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* heap_end points to the first address that is not part of the heap
|
||||
* anymore, so that's where the new block starts when we add pages
|
||||
*/
|
||||
struct memblk *blk = heap_end;
|
||||
if (grow_heap(num_pages) != num_pages)
|
||||
return NULL; /* OOM :( */
|
||||
|
||||
blk_set_size(blk, blksize - OVERHEAD);
|
||||
blk_clear_alloc(blk);
|
||||
blk_set_border_end(blk);
|
||||
|
||||
struct memblk *old_high = blk_prev(blk);
|
||||
blk_clear_border_end(old_high);
|
||||
if (!blk_is_alloc(old_high)) {
|
||||
clist_del(&old_high->clink);
|
||||
blk = blk_merge(old_high, blk);
|
||||
}
|
||||
|
||||
return blk;
|
||||
}
|
||||
|
||||
static inline usize grow_heap(usize num_pages)
|
||||
{
|
||||
usize i;
|
||||
|
||||
for (i = 0; i < num_pages; i++) {
|
||||
uintptr_t page_phys = get_page();
|
||||
if (!page_phys)
|
||||
break;
|
||||
|
||||
if (map_page(page_phys, heap_end, MM_PAGE_RW) != 0) {
|
||||
put_page(page_phys);
|
||||
break;
|
||||
}
|
||||
|
||||
heap_end += PAGE_SIZE;
|
||||
}
|
||||
|
||||
vm_flush();
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
#define ALLOC_FLAG ((usize)1 << 0)
|
||||
#define BORDER_FLAG ((usize)1 << 1)
|
||||
#define SIZE_MASK ( ~(ALLOC_FLAG | BORDER_FLAG) )
|
||||
|
||||
static struct memblk *blk_try_merge(struct memblk *blk)
|
||||
{
|
||||
struct memblk *neighbor = blk_prev(blk);
|
||||
if (neighbor != NULL && !blk_is_alloc(neighbor)) {
|
||||
clist_del(&neighbor->clink);
|
||||
blk = blk_merge(neighbor, blk);
|
||||
}
|
||||
|
||||
neighbor = blk_next(blk);
|
||||
if (neighbor != NULL && !blk_is_alloc(neighbor)) {
|
||||
clist_del(&neighbor->clink);
|
||||
blk = blk_merge(blk, neighbor);
|
||||
}
|
||||
|
||||
struct memblk *cursor;
|
||||
clist_foreach_entry(&blocks, cursor, clink) {
|
||||
if (blk_get_size(cursor) >= blk_get_size(blk))
|
||||
break;
|
||||
}
|
||||
clist_add(&cursor->clink, &blk->clink);
|
||||
|
||||
return blk;
|
||||
}
|
||||
|
||||
static struct memblk *blk_merge(struct memblk *bottom, struct memblk *top)
|
||||
{
|
||||
usize bottom_size = blk_get_size(bottom);
|
||||
usize top_size = blk_get_size(top);
|
||||
usize total_size = bottom_size + top_size + OVERHEAD;
|
||||
|
||||
blk_set_size(bottom, total_size);
|
||||
|
||||
return bottom;
|
||||
}
|
||||
|
||||
static struct memblk *blk_slice(struct memblk *blk, usize slice_size)
|
||||
{
|
||||
struct memblk *cursor = clist_prev_entry(blk, clink);
|
||||
clist_del(&blk->clink);
|
||||
|
||||
/*
|
||||
* If the remaining size is less than the minimum allocation unit, we
|
||||
* hand out the entire block. Additionally, we must add an underflow
|
||||
* check which happens if the slice size is less than OVERHEAD smaller
|
||||
* than the full block size.
|
||||
*/
|
||||
usize rest_size = blk_get_size(blk) - slice_size - OVERHEAD;
|
||||
bool carry = sub_underflow(&rest_size, blk_get_size(blk), slice_size + OVERHEAD);
|
||||
if (rest_size < MIN_SIZE || carry) {
|
||||
blk_set_alloc(blk);
|
||||
return blk;
|
||||
}
|
||||
|
||||
usize slice_words = slice_size / sizeof(blk->low_size);
|
||||
struct memblk *rest = (void *)&blk->high_size[slice_words + 1];
|
||||
blk_set_size(rest, rest_size);
|
||||
blk_clear_alloc(rest);
|
||||
blk_clear_border_start(rest);
|
||||
|
||||
blk_set_size(blk, slice_size);
|
||||
blk_set_alloc(blk);
|
||||
blk_clear_border_end(blk);
|
||||
|
||||
clist_foreach_entry_rev_continue(&blocks, cursor, clink) {
|
||||
if (blk_get_size(cursor) <= rest_size)
|
||||
break;
|
||||
}
|
||||
clist_add_first(&cursor->clink, &rest->clink);
|
||||
|
||||
return blk;
|
||||
}
|
||||
|
||||
static inline struct memblk *blk_prev(struct memblk *blk)
|
||||
{
|
||||
if (blk_is_border_start(blk))
|
||||
return NULL;
|
||||
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Warray-bounds" /* trust me bro, this is fine */
|
||||
return (void *)blk - (blk->low_size[-1] & SIZE_MASK) - OVERHEAD;
|
||||
#pragma clang diagnostic pop
|
||||
}
|
||||
|
||||
static inline struct memblk *blk_next(struct memblk *blk)
|
||||
{
|
||||
if (blk_is_border_end(blk))
|
||||
return NULL;
|
||||
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
return (void *)&blk->high_size[index + 1];
|
||||
}
|
||||
|
||||
static inline usize blk_get_size(struct memblk *blk)
|
||||
{
|
||||
# ifdef DEBUG
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
if ((blk->low_size[0] & SIZE_MASK) != (blk->high_size[index] & SIZE_MASK))
|
||||
kprintf("Memory corruption in block %p detected!\n", blk);
|
||||
# endif
|
||||
return blk->low_size[0] & SIZE_MASK;
|
||||
}
|
||||
|
||||
static void blk_set_size(struct memblk *blk, usize size)
|
||||
{
|
||||
/* don't affect flags */
|
||||
blk->low_size[0] &= ~SIZE_MASK;
|
||||
# ifdef DEBUG
|
||||
if (size & ~SIZE_MASK)
|
||||
kprintf("Unaligned size in blk_set_size()\n");
|
||||
# endif
|
||||
blk->low_size[0] |= size & SIZE_MASK;
|
||||
|
||||
usize index = size / sizeof(blk->low_size[0]);
|
||||
blk->high_size[index] &= ~SIZE_MASK;
|
||||
blk->high_size[index] |= size & SIZE_MASK;
|
||||
}
|
||||
|
||||
static inline void blk_set_alloc(struct memblk *blk)
|
||||
{
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
|
||||
blk->low_size[0] |= ALLOC_FLAG;
|
||||
blk->high_size[index] |= ALLOC_FLAG;
|
||||
}
|
||||
|
||||
static inline void blk_clear_alloc(struct memblk *blk)
|
||||
{
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
|
||||
blk->low_size[0] &= ~ALLOC_FLAG;
|
||||
blk->high_size[index] &= ~ALLOC_FLAG;
|
||||
}
|
||||
|
||||
static inline bool blk_is_alloc(struct memblk *blk)
|
||||
{
|
||||
return (blk->low_size[0] & ALLOC_FLAG) != 0;
|
||||
}
|
||||
|
||||
static inline void blk_set_border_start(struct memblk *blk)
|
||||
{
|
||||
blk->low_size[0] |= BORDER_FLAG;
|
||||
}
|
||||
|
||||
static inline void blk_clear_border_start(struct memblk *blk)
|
||||
{
|
||||
blk->low_size[0] &= ~BORDER_FLAG;
|
||||
}
|
||||
|
||||
static inline bool blk_is_border_start(struct memblk *blk)
|
||||
{
|
||||
return (blk->low_size[0] & BORDER_FLAG) != 0;
|
||||
}
|
||||
|
||||
static inline void blk_set_border_end(struct memblk *blk)
|
||||
{
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
blk->high_size[index] |= BORDER_FLAG;
|
||||
}
|
||||
|
||||
static inline void blk_clear_border_end(struct memblk *blk)
|
||||
{
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
blk->high_size[index] &= ~BORDER_FLAG;
|
||||
}
|
||||
|
||||
static inline bool blk_is_border_end(struct memblk *blk)
|
||||
{
|
||||
usize index = blk->low_size[0] / sizeof(blk->low_size[0]);
|
||||
return (blk->high_size[index] & BORDER_FLAG) != 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* This file is part of GayBSD.
|
||||
* Copyright (c) 2021 fef <owo@fef.moe>.
|
||||
*
|
||||
* GayBSD is nonviolent software: you may only use, redistribute, and/or
|
||||
* modify it under the terms of the Cooperative Nonviolent Public License
|
||||
* (CNPL) as found in the LICENSE file in the source code root directory
|
||||
* or at <https://git.pixie.town/thufie/npl-builder>; either version 7
|
||||
* of the license, or (at your option) any later version.
|
||||
*
|
||||
* GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent
|
||||
* permitted by applicable law. See the CNPL for details.
|
||||
*/
|
||||
@ -0,0 +1,403 @@
|
||||
/* See the end of this file for copyright and license terms. */
|
||||
|
||||
#include <arch/page.h>
|
||||
|
||||
#include <gay/clist.h>
|
||||
#include <gay/config.h>
|
||||
#include <gay/kprintf.h>
|
||||
#include <gay/mm.h>
|
||||
#include <gay/types.h>
|
||||
#include <gay/util.h>
|
||||
|
||||
#include <limits.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifndef __HAVE_HUGEPAGES
|
||||
#error "Systems without huge pages are currently unsupported because i'm a dumb bitch"
|
||||
#endif
|
||||
|
||||
#if DMAP_OFFSET % HUGEPAGE_SIZE != 0
|
||||
#error "DMAP_OFFSET must be an integral multiple of HUGEPAGE_SIZE"
|
||||
#endif
|
||||
|
||||
/* this should be impossible because arch/page.h must also define PAGE_SHIFT
|
||||
* and HUGEPAGE_SHIFT, meaning the two are definitively powers of 2 */
|
||||
#if HUGEPAGE_SIZE % PAGE_SIZE != 0
|
||||
#error "HUGEPAGE_SIZE must be an integral multiple of PAGE_SIZE"
|
||||
#endif
|
||||
|
||||
#if PAGE_SIZE % LONG_BIT != 0
|
||||
#error "PAGE_SIZE must be an integral multiple of LONG_BIT"
|
||||
#endif
|
||||
|
||||
#if CFG_DEBUG_PAGE_ALLOCS
|
||||
#define page_debug(msg, ...) kprintf("[page] " msg, ##__VA_ARGS__)
|
||||
#else
|
||||
#define page_debug(msg, ...)
|
||||
#endif
|
||||
|
||||
/**
|
||||
* We have cache levels for areas ranging from a single page up to a huge page
|
||||
* on a logarithmic scale. Every level covers double the pages per entry than
|
||||
* the one below it, starting at one page per entry. The effective result is
|
||||
* that a single entry in the cache on level L covers `(1 << L)` pages.
|
||||
*/
|
||||
#define CACHE_LEVELS (HUGEPAGE_SHIFT - PAGE_SHIFT + 1)
|
||||
|
||||
struct cache_pool {
|
||||
struct clist freelist;
|
||||
unsigned long *bitmap;
|
||||
usize free_entries;
|
||||
usize bitmap_len;
|
||||
};
|
||||
static struct cache_pool caches[CACHE_LEVELS];
|
||||
|
||||
#define LONG_MASK ( ~(usize)(sizeof(long) - 1) )
|
||||
|
||||
uintptr_t phys_start;
|
||||
uintptr_t phys_end;
|
||||
|
||||
/**
|
||||
* @brief Split a block and return the lower half.
|
||||
* The block is assumed to already have been removed from its freelist.
|
||||
* The high half (i.e. the block that is *not* returned) is inserted into the
|
||||
* freelist one level below `level`.
|
||||
*/
|
||||
static void *split_buddy(void *ptr, int level);
|
||||
/**
|
||||
* @brief Attempt to coalesce a block with its buddy.
|
||||
* If coalition is possible, the buddy is removed from its freelist at
|
||||
* `level` and the union block is inserted at `level + 1`.
|
||||
*
|
||||
* @param ptr Pointer to the block
|
||||
* @param level Cache level, must be less than `CACHE_LEVELS - 1` (because you
|
||||
* can't join blocks at the highest cache level)
|
||||
* @return The joined block, or `nil` if coalition was not possible
|
||||
*/
|
||||
static void *try_join_buddy(void *ptr, int level);
|
||||
|
||||
static usize get_bit_number(void *ptr, int level);
|
||||
|
||||
static void set_bits(unsigned long *bitfield, usize first, usize count);
|
||||
static void clr_bits(unsigned long *bitfield, usize first, usize count);
|
||||
|
||||
static bool get_bit(const unsigned long *bitfield, usize bit_number);
|
||||
static void set_bit(unsigned long *bitfield, usize bit_number);
|
||||
|
||||
static int sanity_check(void)
|
||||
{
|
||||
if (phys_end != HUGEPAGE_ALIGN(phys_end) || phys_start != HUGEPAGE_ALIGN(phys_start)) {
|
||||
kprintf("Unaligned memory, this should never be possible\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if ((phys_end - phys_start) < (32 * 1024 * 1024)) {
|
||||
kprintf("Less than 32 MB of usable RAM, this wouldn't go well\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (phys_start > phys_end) {
|
||||
kprintf("Hey, this is funny. pages_init() was called with parameters "
|
||||
"such that phys_start > phys_end (%p > %p), which "
|
||||
"should absolutely never be possible. I can't really continue "
|
||||
"like this, so have a nice day.\n", (void *)phys_start, (void *)phys_end);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void init_freelist(void)
|
||||
{
|
||||
for (int i = 0; i < CACHE_LEVELS; i++) {
|
||||
clist_init(&caches[i].freelist);
|
||||
caches[i].free_entries = 0;
|
||||
}
|
||||
|
||||
struct cache_pool *pool = &caches[CACHE_LEVELS - 1];
|
||||
const usize step = 1 << (CACHE_LEVELS + PAGE_SHIFT);
|
||||
for (void *pos = kheap_start; pos < kheap_end; pos += step) {
|
||||
struct clist *entry = pos;
|
||||
clist_add(&pool->freelist, entry);
|
||||
pool->free_entries += 1;
|
||||
}
|
||||
}
|
||||
|
||||
int pages_init(void)
|
||||
{
|
||||
usize phys_size = phys_end - phys_start;
|
||||
|
||||
if (sanity_check() != 0)
|
||||
return 1;
|
||||
|
||||
/*
|
||||
* map entire physical memory into the direct contiguous area
|
||||
*/
|
||||
for (uintptr_t physptr = phys_start; physptr < phys_end; physptr += HUGEPAGE_SIZE) {
|
||||
const enum mm_page_flags pflags = MM_PAGE_HUGE | MM_PAGE_RW | MM_PAGE_GLOBAL;
|
||||
map_page(physptr, (void *)(physptr + DMAP_OFFSET), pflags);
|
||||
}
|
||||
vm_flush();
|
||||
|
||||
/*
|
||||
* calculate the size of each bitmap, as well as their combined size
|
||||
*/
|
||||
usize cache_bytes = 0;
|
||||
for (int i = 0; i < CACHE_LEVELS; i++) {
|
||||
usize bits = phys_size >> (PAGE_SHIFT + i);
|
||||
/* round up to the next full long */
|
||||
if (bits % LONG_BIT) {
|
||||
bits &= LONG_MASK;
|
||||
bits += LONG_BIT;
|
||||
}
|
||||
cache_bytes += bits / 8;
|
||||
caches[i].bitmap_len = bits / LONG_BIT;
|
||||
}
|
||||
/* smol buffer in case we overshoot for whatever reason */
|
||||
cache_bytes += sizeof(long);
|
||||
|
||||
page_debug("Page frame overhead = %zu bytes\n", cache_bytes);
|
||||
|
||||
/*
|
||||
* zero out all bitmaps
|
||||
*/
|
||||
uintptr_t cache_start_phys = phys_end - cache_bytes;
|
||||
unsigned long *cache_start = __v(cache_start_phys);
|
||||
memset(cache_start, 0, cache_bytes);
|
||||
|
||||
/*
|
||||
* populate the caches array and preallocate pages that can't be handed
|
||||
* out (i.e. the cache bitmaps)
|
||||
*/
|
||||
unsigned long *cache_pos = cache_start;
|
||||
for (int i = 0; i < CACHE_LEVELS; i++) {
|
||||
usize total_bits = caches[i].bitmap_len * LONG_BIT;
|
||||
usize wasted_bits = total_bits - (cache_bytes >> (PAGE_SHIFT + i));
|
||||
if (wasted_bits == 0)
|
||||
wasted_bits = 1;
|
||||
set_bits(cache_pos, total_bits - wasted_bits, wasted_bits);
|
||||
|
||||
caches[i].bitmap = cache_pos;
|
||||
cache_pos += caches[i].bitmap_len;
|
||||
}
|
||||
|
||||
/* kheap_start and kheap_end are globals */
|
||||
kheap_start = __v(phys_start);
|
||||
kheap_end = cache_start;
|
||||
|
||||
init_freelist();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int get_level(usize count)
|
||||
{
|
||||
int level;
|
||||
for (level = 0; level < CACHE_LEVELS; level++) {
|
||||
if ((1 << level) >= count)
|
||||
break;
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
void *get_pages(usize count, enum mm_flags flags)
|
||||
{
|
||||
int level = get_level(count);
|
||||
if (level == CACHE_LEVELS)
|
||||
return nil;
|
||||
|
||||
struct clist *entry;
|
||||
int entry_level;
|
||||
for (entry_level = level; entry_level < CACHE_LEVELS; entry_level++) {
|
||||
if (caches[entry_level].free_entries > 0) {
|
||||
entry = caches[entry_level].freelist.next;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (entry_level == CACHE_LEVELS)
|
||||
return nil;
|
||||
|
||||
clist_del(entry);
|
||||
caches[entry_level].free_entries--;
|
||||
|
||||
usize bit_number = get_bit_number(entry, level);
|
||||
while (entry_level > level) {
|
||||
entry = split_buddy(entry, entry_level);
|
||||
set_bit(caches[entry_level].bitmap, bit_number);
|
||||
entry_level--;
|
||||
bit_number <<= 1;
|
||||
}
|
||||
|
||||
do {
|
||||
usize bit_count = 1 << (level - entry_level);
|
||||
set_bits(caches[entry_level].bitmap, bit_number, bit_count);
|
||||
} while (entry_level-- != 0);
|
||||
|
||||
return (void *)entry;
|
||||
}
|
||||
|
||||
void free_pages(void *ptr, usize count)
|
||||
{
|
||||
int level = get_level(count);
|
||||
if (level == CACHE_LEVELS)
|
||||
return;
|
||||
|
||||
usize bit_number = get_bit_number(ptr, level);
|
||||
usize bit_count = 1;
|
||||
for (int i = level; i >= 0; i--) {
|
||||
clr_bits(caches[i].bitmap, bit_number, bit_count);
|
||||
bit_number <<= 1;
|
||||
bit_count <<= 1;
|
||||
}
|
||||
|
||||
while (ptr != nil && level < CACHE_LEVELS - 1) {
|
||||
ptr = try_join_buddy(ptr, level);
|
||||
level++;
|
||||
}
|
||||
}
|
||||
|
||||
static inline usize get_bit_number(void *ptr, int level)
|
||||
{
|
||||
return ((uintptr_t)ptr - (uintptr_t)kheap_start) >> (PAGE_SHIFT + level);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Set a range of bits in a bitfield.
|
||||
*
|
||||
* @param bitfield Pointer to the beginning of the bitfield
|
||||
* @param first Number of the first bit to set, counting from 0
|
||||
* @param count Amount of bits to set
|
||||
*/
|
||||
static void set_bits(unsigned long *bitfield, usize first, usize count)
|
||||
{
|
||||
bitfield += first / LONG_BIT;
|
||||
unsigned int bit = first % LONG_BIT;
|
||||
|
||||
if (bit != 0) {
|
||||
unsigned long mask = (1lu << bit) - 1;
|
||||
*bitfield++ |= ~mask;
|
||||
count -= bit;
|
||||
}
|
||||
|
||||
while (count >= LONG_BIT) {
|
||||
*bitfield++ = ULONG_MAX;
|
||||
count -= LONG_BIT;
|
||||
}
|
||||
|
||||
if (count != 0) {
|
||||
unsigned long mask = (1lu << count) - 1;
|
||||
*bitfield |= mask;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Clear a range of bits in a bitfield.
|
||||
*
|
||||
* The algorithm is similar to `set_bits()`, it just does the inverse.
|
||||
*
|
||||
* @param bitfield Pointer to the beginning of the bitfield
|
||||
* @param first Number of the first bit to clear, counting from 0
|
||||
* @param count Amount of bits to clear
|
||||
*/
|
||||
static void clr_bits(unsigned long *bitfield, usize first, usize count)
|
||||
{
|
||||
bitfield += first / LONG_BIT;
|
||||
unsigned int bit = first % LONG_BIT;
|
||||
|
||||
if (bit != 0) {
|
||||
unsigned long mask = (1lu << bit) - 1;
|
||||
*bitfield++ &= mask;
|
||||
count -= bit;
|
||||
}
|
||||
|
||||
while (count >= LONG_BIT) {
|
||||
*bitfield++ = 0;
|
||||
count -= LONG_BIT;
|
||||
}
|
||||
|
||||
if (count != 0) {
|
||||
unsigned long mask = (1lu << bit) - 1;
|
||||
*bitfield &= ~mask;
|
||||
}
|
||||
}
|
||||
|
||||
static inline bool get_bit(const unsigned long *bitfield, usize bit_number)
|
||||
{
|
||||
unsigned long longword = bitfield[bit_number / LONG_BIT];
|
||||
unsigned long mask = 1lu << (bit_number % LONG_BIT);
|
||||
return (longword & mask) != 0;
|
||||
}
|
||||
|
||||
static inline void set_bit(unsigned long *bitfield, usize bit_number)
|
||||
{
|
||||
unsigned long mask = 1lu << (bit_number % LONG_BIT);
|
||||
bitfield[bit_number / LONG_BIT] |= mask;
|
||||
}
|
||||
|
||||
static inline void *split_buddy(void *ptr, int level)
|
||||
{
|
||||
# if CFG_DEBUG_PAGE_ALLOCS
|
||||
if ((uintptr_t)ptr % (1 << (PAGE_SHIFT + level))) {
|
||||
kprintf("split_buddy(%p, %d): unaligned ptr!\n", ptr, level);
|
||||
return nil;
|
||||
}
|
||||
if (level < 1 || level >= CACHE_LEVELS) {
|
||||
kprintf("split_buddy(%p, %d): invalid level!\n", ptr, level);
|
||||
return nil;
|
||||
}
|
||||
# endif
|
||||
|
||||
struct clist *high_buddy = ptr + (1 << (PAGE_SHIFT + level - 1));
|
||||
clist_add(&caches[level - 1].freelist, high_buddy);
|
||||
caches[level - 1].free_entries++;
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static void *try_join_buddy(void *ptr, int level)
|
||||
{
|
||||
const usize entry_size = 1 << (PAGE_SHIFT + level);
|
||||
|
||||
# if CFG_DEBUG_PAGE_ALLOCS
|
||||
if ((uintptr_t)ptr % entry_size) {
|
||||
kprintf("try_join_buddy(%p, %d): unaligned ptr!\n", ptr, level);
|
||||
return nil;
|
||||
}
|
||||
/* level must be < CACHE_LEVELS - 1 because you
|
||||
* can't join blocks on the topmost level */
|
||||
if (level >= CACHE_LEVELS - 1) {
|
||||
kprintf("try_join_buddy(%p, %d): level >= CACHE_LEVELS - 1!\n", ptr, level);
|
||||
return nil;
|
||||
}
|
||||
# endif
|
||||
|
||||
/* test if the buddy block is allocated and return nil if it is */
|
||||
uintptr_t buddy = (uintptr_t)ptr ^ entry_size;
|
||||
usize buddy_bitnum = get_bit_number((void *)buddy, level);
|
||||
if (get_bit(caches[level].bitmap, buddy_bitnum))
|
||||
return nil;
|
||||
|
||||
/* if it is not, remove it from the freelist */
|
||||
clist_del((struct clist *)buddy);
|
||||
caches[level].free_entries--;
|
||||
|
||||
/* add the coalesced block to the freelist one level above */
|
||||
struct clist *even = (struct clist *)((uintptr_t)ptr & ~entry_size);
|
||||
clist_add(&caches[level + 1].freelist, even);
|
||||
caches[level + 1].free_entries++;
|
||||
return even;
|
||||
}
|
||||
|
||||
/*
|
||||
* This file is part of GayBSD.
|
||||
* Copyright (c) 2021 fef <owo@fef.moe>.
|
||||
*
|
||||
* GayBSD is nonviolent software: you may only use, redistribute, and/or
|
||||
* modify it under the terms of the Cooperative Nonviolent Public License
|
||||
* (CNPL) as found in the LICENSE file in the source code root directory
|
||||
* or at <https://git.pixie.town/thufie/npl-builder>; either version 7
|
||||
* of the license, or (at your option) any later version.
|
||||
*
|
||||
* GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent
|
||||
* permitted by applicable law. See the CNPL for details.
|
||||
*/
|
||||
Loading…
Reference in New Issue