diff --git a/arch/x86/mm/page.c b/arch/x86/mm/page.c index 3829e74..43528ce 100644 --- a/arch/x86/mm/page.c +++ b/arch/x86/mm/page.c @@ -26,10 +26,6 @@ extern void _image_start_phys; extern void _image_end_phys; -/* first and last dynamic page address (watch out, these are physical) */ -static uintptr_t dynpage_start; -static uintptr_t dynpage_end; - /** * @brief First page table for low memory (0 - 4 M). * This is initialized by the early boot routine in assembly so that paging @@ -107,6 +103,31 @@ int map_page(uintptr_t phys, void *virt, enum pflags flags) return 0; } +/* + * The only difference between this and map_page() is that we can't allocate + * new pages using get_pages() but have to use __early_get_page() instead here. + * So, all we need to do is ensure that map_page() doesn't need to allocate new + * pages when we call it, which it only does if pflags does not have PFLAG_HUGE + * set and the page table doesn't exist (present bit in the page directory is + * clear). Therefore, we just need to make sure that, if PFLAG_HUGE is *not* + * set, the page table is already allocated and marked as present in the page + * directory. + */ +void __early_map_page(uintptr_t phys, void *virt, enum pflags pflags) +{ + if (!(pflags & PFLAG_HUGE)) { + usize pd_index = ((uintptr_t)virt >> PAGE_SHIFT) / 1024; + struct x86_page_directory_entry *pde = &X86_CURRENT_PD->entries[pd_index]; + if (!pde->present) { + uintptr_t pt_phys = __early_get_page(); + *(unsigned long *)pde = PFLAG_PRESENT | PFLAG_RW; + pde->shifted_address = pt_phys >> PAGE_SHIFT; + } + } + + map_page(phys, virt, pflags); +} + uintptr_t unmap_page(void *virt) { # ifdef DEBUG diff --git a/include/gay/mm.h b/include/gay/mm.h index 769c002..0b15bcf 100644 --- a/include/gay/mm.h +++ b/include/gay/mm.h @@ -12,10 +12,9 @@ * * GayBSD uses a classic slab algorithm for its own data structures, which is * backed by a buddy page frame allocator. The latter is also used for getting - * bigger areas of memory that is not physically contiguous (for regular user - * allocations). The high memory is statically mapped to the area after the - * kernel image, which starts at `CFG_KERN_ORIGIN + KERN_OFFSET`. Om i386, - * this results in a kernel image mapping to `0xf0100000`. + * bigger areas of memory that are not physically contiguous (for regular user + * allocations). The entire physical memory is mapped statically in the range + * `DMAP_START - DMAP_END`. */ #ifdef _KERNEL @@ -73,6 +72,14 @@ enum pflags { #endif }; +/* + * Terrible hack that allows us to map pages before the page frame allocator is + * set up. Don't ever use these anywhere, because they *will* break everything. + */ +void __early_map_page(uintptr_t phys, void *virt, enum pflags flags); +/* This just shrinks the usable physical area by PAGE_SIZE and returns the page */ +uintptr_t __early_get_page(void); + /** * @brief Map a page in physical memory to a virtual address. * Remember that if `vm` is the memory map currently in use, you will most @@ -170,11 +177,33 @@ void slab_free(void *ptr); */ static __always_inline void *__v(uintptr_t phys) { - if (phys > phys_end) - return nil; +# ifdef DEBUG + if (phys > phys_end) + return nil; +# endif return (void *)phys + DMAP_OFFSET; } +/** + * @brief Return where a virtual address in the direct mapping region is in + * physical memory. This does **not** perform a lookup in the page table + * structures, and should generally only be used from within mm code (hence the + * two underscores). The reliable way of determining where any virtual address + * maps to is `vtophys()`. + * + * @param virt Virtual address, must be within `DMAP_START - DMAP_END` + * @return The physical address, i.e. `virt - DMAP_OFFSET` + * @see vtophys() + */ +static __always_inline uintptr_t __p(void *virt) +{ +# ifdef DEBUG + if (virt < DMAP_START || virt >= DMAP_END) + return 0; +# endif + return (uintptr_t)virt - DMAP_OFFSET; +} + #endif /* _KERNEL */ /* diff --git a/kernel/mm/kmalloc.c b/kernel/mm/kmalloc.c index 324a464..9dc1e0e 100644 --- a/kernel/mm/kmalloc.c +++ b/kernel/mm/kmalloc.c @@ -14,6 +14,9 @@ void *kheap_end; int kmalloc_init(uintptr_t _phys_start, uintptr_t _phys_end) { + phys_start = _phys_start; + phys_end = _phys_end; + /* * The kernel image is very likely gonna be within the physical memory * range, so we're gonna need to do some cropping in order to not hand @@ -22,21 +25,30 @@ int kmalloc_init(uintptr_t _phys_start, uintptr_t _phys_end) */ uintptr_t image_start_phys = (uintptr_t)&_image_start_phys; uintptr_t image_end_phys = (uintptr_t)&_image_end_phys; - if (_phys_start < image_start_phys && _phys_end > image_start_phys) { - if (image_start_phys - _phys_start > _phys_end - image_start_phys) - _phys_end = image_start_phys; + if (phys_start < image_start_phys && phys_end > image_start_phys) { + if (image_start_phys - phys_start > phys_end - image_start_phys) + phys_end = image_start_phys; else - _phys_start = image_end_phys; + phys_start = image_end_phys; } - if (_phys_start < image_end_phys && _phys_end > image_end_phys) { - if (image_end_phys - _phys_start > _phys_end - image_end_phys) - _phys_end = image_start_phys; + if (phys_start < image_end_phys && _phys_end > image_end_phys) { + if (image_end_phys - phys_start > phys_end - image_end_phys) + phys_end = image_start_phys; else - _phys_start = image_end_phys; + phys_start = image_end_phys; } - phys_start = uintptr_align(_phys_start, +HUGEPAGE_SHIFT); - phys_end = uintptr_align(_phys_end, -HUGEPAGE_SHIFT); - kprintf("Aligning physical memory to 0x%08x-0x%08x\n", phys_start, phys_end); + + phys_start = uintptr_align(phys_start, +HUGEPAGE_SHIFT); + /* + * This is intentionally not aligned to hugepages, because __early_get_page() + * shrinks it in single PAGE_SIZE steps whenever it is called anyway. + * I know, this is a terrible hack, but it will be aligned to a hugepage + * from within pages_init(), right after the entire physical memory has + * been mapped to the direct area (which is the only reason we need to + * be able to allocate pages before the page frame allocator is set up + * in the first place). + */ + phys_end = uintptr_align(phys_end, -PAGE_SHIFT); int err = pages_init(); if (err) diff --git a/kernel/mm/page.c b/kernel/mm/page.c index 75b6f20..71dd6f5 100644 --- a/kernel/mm/page.c +++ b/kernel/mm/page.c @@ -79,12 +79,20 @@ MTX(caches_lock); #define LONG_BIT_MASK (~(LONG_BIT - 1)) +/* these get set in kmalloc_init() */ uintptr_t phys_start; uintptr_t phys_end; +uintptr_t __early_get_page(void) +{ + phys_end -= PAGE_SIZE; + return phys_end; +} + static int sanity_check(void) { - if (phys_end != HUGEPAGE_ALIGN(phys_end) || phys_start != HUGEPAGE_ALIGN(phys_start)) { + /* phys_end is only page aligned, see kmalloc_init() */ + if (phys_end != PAGE_ALIGN(phys_end) || phys_start != HUGEPAGE_ALIGN(phys_start)) { kprintf("Unaligned memory, this should never be possible\n"); return 1; } @@ -113,20 +121,26 @@ static int sanity_check(void) */ int pages_init(void) { - usize phys_size = phys_end - phys_start; - if (sanity_check() != 0) return 1; /* - * map entire physical memory into the direct contiguous area + * Map the entire physical memory into the direct contiguous area. + * __early_map_page() might call __early_get_page() in order to allocate + * new page table structures, which in turn shrinks the physical memory + * size (see above). + * It might be necessary to use a volatile pointer to phys_end for this + * loop in case clang does The Optimization and caches its value for + * whatever reason, even though at least for x86 this is not the case. */ - for (uintptr_t physptr = phys_start; physptr < phys_end; physptr += HUGEPAGE_SIZE) { - const enum pflags pflags = PFLAG_HUGE | PFLAG_RW | PFLAG_GLOBAL; - map_page(physptr, __v(physptr), pflags); - } + for (uintptr_t physptr = phys_start; physptr < phys_end; physptr += HUGEPAGE_SIZE) + __early_map_page(physptr, __v(physptr), PFLAG_HUGE | PFLAG_RW | PFLAG_GLOBAL); vm_flush(); + /* phys_end gets aligned, as promised by the comment in kmalloc_init() */ + phys_end = uintptr_align(phys_end, -HUGEPAGE_SHIFT); + usize phys_size = phys_end - phys_start; + /* * calculate the size of each bitmap, as well as their combined size */ @@ -141,7 +155,7 @@ int pages_init(void) bitmap_bytes += bits / 8; } - page_debug("Page frame overhead = %zu bytes\n", bitmap_bytes); + page_debug("Page frame overhead = %zu bytes, %zu bytes total\n", bitmap_bytes, phys_size); /* * zero out all bitmaps