page: ensure early page mapping doesn't fail

Up to now, the page frame allocator's
initialization routine relied on the fact that
map_page() never needs to get new pages on i386 if
the mapped page is a hugepage.  This is not at all
true on other architectures, however.

arch/x86/mm/page.c

@@ -26,10 +26,6 @@
 extern void _image_start_phys;
 extern void _image_end_phys;
 
-/* first and last dynamic page address (watch out, these are physical) */
-static uintptr_t dynpage_start;
-static uintptr_t dynpage_end;
-
 /**
  * @brief First page table for low memory (0 - 4 M).
  * This is initialized by the early boot routine in assembly so that paging
@@ -107,6 +103,31 @@ int map_page(uintptr_t phys, void *virt, enum pflags flags)
 	return 0;
 }
 
+/*
+ * The only difference between this and map_page() is that we can't allocate
+ * new pages using get_pages() but have to use __early_get_page() instead here.
+ * So, all we need to do is ensure that map_page() doesn't need to allocate new
+ * pages when we call it, which it only does if pflags does not have PFLAG_HUGE
+ * set and the page table doesn't exist (present bit in the page directory is
+ * clear).  Therefore, we just need to make sure that, if PFLAG_HUGE is *not*
+ * set, the page table is already allocated and marked as present in the page
+ * directory.
+ */
+void __early_map_page(uintptr_t phys, void *virt, enum pflags pflags)
+{
+	if (!(pflags & PFLAG_HUGE)) {
+		usize pd_index = ((uintptr_t)virt >> PAGE_SHIFT) / 1024;
+		struct x86_page_directory_entry *pde = &X86_CURRENT_PD->entries[pd_index];
+		if (!pde->present) {
+			uintptr_t pt_phys = __early_get_page();
+			*(unsigned long *)pde = PFLAG_PRESENT | PFLAG_RW;
+			pde->shifted_address = pt_phys >> PAGE_SHIFT;
+		}
+	}
+
+	map_page(phys, virt, pflags);
+}
+
 uintptr_t unmap_page(void *virt)
 {
 #	ifdef DEBUG

include/gay/mm.h

@@ -12,10 +12,9 @@
  *
  * GayBSD uses a classic slab algorithm for its own data structures, which is
  * backed by a buddy page frame allocator.  The latter is also used for getting
- * bigger areas of memory that is not physically contiguous (for regular user
+ * bigger areas of memory that are not physically contiguous (for regular user
- * allocations).  The high memory is statically mapped to the area after the
+ * allocations).  The entire physical memory is mapped statically in the range
- * kernel image, which starts at `CFG_KERN_ORIGIN + KERN_OFFSET`.  Om i386,
+ * `DMAP_START - DMAP_END`.
- * this results in a kernel image mapping to `0xf0100000`.
  */
 
 #ifdef _KERNEL
@@ -73,6 +72,14 @@ enum pflags {
 #endif
 };
 
+/*
+ * Terrible hack that allows us to map pages before the page frame allocator is
+ * set up.  Don't ever use these anywhere, because they *will* break everything.
+ */
+void __early_map_page(uintptr_t phys, void *virt, enum pflags flags);
+/* This just shrinks the usable physical area by PAGE_SIZE and returns the page */
+uintptr_t __early_get_page(void);
+
 /**
  * @brief Map a page in physical memory to a virtual address.
  * Remember that if `vm` is the memory map currently in use, you will most
@@ -170,11 +177,33 @@ void slab_free(void *ptr);
  */
 static __always_inline void *__v(uintptr_t phys)
 {
-	if (phys > phys_end)
+#	ifdef DEBUG
-		return nil;
+		if (phys > phys_end)
+			return nil;
+#	endif
 	return (void *)phys + DMAP_OFFSET;
 }
 
+/**
+ * @brief Return where a virtual address in the direct mapping region is in
+ * physical memory.  This does **not** perform a lookup in the page table
+ * structures, and should generally only be used from within mm code (hence the
+ * two underscores).  The reliable way of determining where any virtual address
+ * maps to is `vtophys()`.
+ *
+ * @param virt Virtual address, must be within `DMAP_START - DMAP_END`
+ * @return The physical address, i.e. `virt - DMAP_OFFSET`
+ * @see vtophys()
+ */
+static __always_inline uintptr_t __p(void *virt)
+{
+#	ifdef DEBUG
+		if (virt < DMAP_START || virt >= DMAP_END)
+			return 0;
+#	endif
+	return (uintptr_t)virt - DMAP_OFFSET;
+}
+
 #endif /* _KERNEL */
 
 /*

kernel/mm/kmalloc.c

@@ -14,6 +14,9 @@ void *kheap_end;
 
 int kmalloc_init(uintptr_t _phys_start, uintptr_t _phys_end)
 {
+	phys_start = _phys_start;
+	phys_end = _phys_end;
+
 	/*
 	 * The kernel image is very likely gonna be within the physical memory
 	 * range, so we're gonna need to do some cropping in order to not hand
@@ -22,21 +25,30 @@ int kmalloc_init(uintptr_t _phys_start, uintptr_t _phys_end)
 	 */
 	uintptr_t image_start_phys = (uintptr_t)&_image_start_phys;
 	uintptr_t image_end_phys = (uintptr_t)&_image_end_phys;
-	if (_phys_start < image_start_phys && _phys_end > image_start_phys) {
+	if (phys_start < image_start_phys && phys_end > image_start_phys) {
-		if (image_start_phys - _phys_start > _phys_end - image_start_phys)
+		if (image_start_phys - phys_start > phys_end - image_start_phys)
-			_phys_end = image_start_phys;
+			phys_end = image_start_phys;
 		else
-			_phys_start = image_end_phys;
+			phys_start = image_end_phys;
 	}
-	if (_phys_start < image_end_phys && _phys_end > image_end_phys) {
+	if (phys_start < image_end_phys && _phys_end > image_end_phys) {
-		if (image_end_phys - _phys_start > _phys_end - image_end_phys)
+		if (image_end_phys - phys_start > phys_end - image_end_phys)
-			_phys_end = image_start_phys;
+			phys_end = image_start_phys;
 		else
-			_phys_start = image_end_phys;
+			phys_start = image_end_phys;
 	}
-	phys_start = uintptr_align(_phys_start, +HUGEPAGE_SHIFT);
+
-	phys_end = uintptr_align(_phys_end, -HUGEPAGE_SHIFT);
+	phys_start = uintptr_align(phys_start, +HUGEPAGE_SHIFT);
-	kprintf("Aligning physical memory to 0x%08x-0x%08x\n", phys_start, phys_end);
+	/*
+	 * This is intentionally not aligned to hugepages, because __early_get_page()
+	 * shrinks it in single PAGE_SIZE steps whenever it is called anyway.
+	 * I know, this is a terrible hack, but it will be aligned to a hugepage
+	 * from within pages_init(), right after the entire physical memory has
+	 * been mapped to the direct area (which is the only reason we need to
+	 * be able to allocate pages before the page frame allocator is set up
+	 * in the first place).
+	 */
+	phys_end = uintptr_align(phys_end, -PAGE_SHIFT);
 
 	int err = pages_init();
 	if (err)

kernel/mm/page.c

@@ -79,12 +79,20 @@ MTX(caches_lock);
 
 #define LONG_BIT_MASK (~(LONG_BIT - 1))
 
+/* these get set in kmalloc_init() */
 uintptr_t phys_start;
 uintptr_t phys_end;
 
+uintptr_t __early_get_page(void)
+{
+	phys_end -= PAGE_SIZE;
+	return phys_end;
+}
+
 static int sanity_check(void)
 {
-	if (phys_end != HUGEPAGE_ALIGN(phys_end) || phys_start != HUGEPAGE_ALIGN(phys_start)) {
+	/* phys_end is only page aligned, see kmalloc_init() */
+	if (phys_end != PAGE_ALIGN(phys_end) || phys_start != HUGEPAGE_ALIGN(phys_start)) {
 		kprintf("Unaligned memory, this should never be possible\n");
 		return 1;
 	}
@@ -113,20 +121,26 @@ static int sanity_check(void)
  */
 int pages_init(void)
 {
-	usize phys_size = phys_end - phys_start;
-
 	if (sanity_check() != 0)
 		return 1;
 
 	/*
-	 * map entire physical memory into the direct contiguous area
+	 * Map the entire physical memory into the direct contiguous area.
+	 * __early_map_page() might call __early_get_page() in order to allocate
+	 * new page table structures, which in turn shrinks the physical memory
+	 * size (see above).
+	 * It might be necessary to use a volatile pointer to phys_end for this
+	 * loop in case clang does The Optimization and caches its value for
+	 * whatever reason, even though at least for x86 this is not the case.
 	 */
-	for (uintptr_t physptr = phys_start; physptr < phys_end; physptr += HUGEPAGE_SIZE) {
+	for (uintptr_t physptr = phys_start; physptr < phys_end; physptr += HUGEPAGE_SIZE)
-		const enum pflags pflags = PFLAG_HUGE | PFLAG_RW | PFLAG_GLOBAL;
+		__early_map_page(physptr, __v(physptr), PFLAG_HUGE | PFLAG_RW | PFLAG_GLOBAL);
-		map_page(physptr, __v(physptr), pflags);
-	}
 	vm_flush();
 
+	/* phys_end gets aligned, as promised by the comment in kmalloc_init() */
+	phys_end = uintptr_align(phys_end, -HUGEPAGE_SHIFT);
+	usize phys_size = phys_end - phys_start;
+
 	/*
 	 * calculate the size of each bitmap, as well as their combined size
 	 */
@@ -141,7 +155,7 @@ int pages_init(void)
 		bitmap_bytes += bits / 8;
 	}
 
-	page_debug("Page frame overhead = %zu bytes\n", bitmap_bytes);
+	page_debug("Page frame overhead = %zu bytes, %zu bytes total\n", bitmap_bytes, phys_size);
 
 	/*
 	 * zero out all bitmaps