kern/include/gay/vm/page.h

/* Copyright (C) 2021,2022 fef <owo@fef.moe>.  All rights reserved. */

#pragma once

#include <arch/atom.h>
#include <arch/page.h>

#include <gay/cdefs.h>
#include <gay/clist.h>
#include <gay/config.h>
#include <gay/systm.h>
#include <gay/types.h>

/*
 * I'm trying really hard to keep the size of struct vm_page a power of two
 * on LP64 systems, because that way we can quickly get to the page frame number
 * by shifting the byte offset of the vm_page_t in vm_page_array to the right
 * rather than doing a costly divide instruction (or store the page frame number
 * within the structure itself, which takes up precious space).
 *
 * There is insane pressure on the size of this structure, because a typical
 * system will have millions of instances of it.  Every additional byte makes
 * a significant difference in memory management overhead.
 */

union vm_page_attr {
	int _val;
	struct {
		unsigned order:8;	/**< @brief Index into `mm_zones[zone].pools` */
		bool lock:1;		/**< @brief Page is locked */
		bool rsvd:1;		/**< @brief Page is reserved and must never be touched */
		bool pcpu:1;		/**< @brief Page is in a per-cpu cache */
		bool slab:1;		/**< @brief Page is used by the slab allocator */
		unsigned zone:2;	/**< @brief Index into `mm_zones` */
	};
};
#define _PGA_ORDER_SHIFT	0
#define _PGA_ORDER_MASK		(0xf << _PGA_ORDER_SHIFT)
#define _PGA_LOCK_SHIFT		8
#define _PGA_LOCK_MASK		(1 << _PGA_LOCK_SHIFT)
#define _PGA_RSVD_SHIFT		9
#define _PGA_RSVD_MASK		(1 << _PGA_RSVD_SHIFT)
#define _PGA_PCPU_SHIFT		10
#define _PGA_PCPU_MASK		(1 << _PGA_PCPU_SHIFT)
#define _PGA_SLAB_SHIFT		11
#define _PGA_SLAB_MASK		(1 << _PGA_SLAB_SHIFT)
#define _PGA_ZONE_SHIFT		12
#define _PGA_ZONE_MASK		(3 << _PGA_ZONE_SHIFT)

typedef union vm_page_attr vm_page_attr_t;

/**
 * @brief Stores information about a single page in physical memory.
 * There is exactly one of these for every physical page, no matter what that
 * page is used for or whether it is usable at all.
 */
struct vm_page {
	/** @brief Reference count (0 = unused, < 0 = locked) */
	atom_t count;
	/** @brief Page attributes, use the macros below to access this */
	atom_t attr;
	/** @brief If the page is free, this is its freelist. */
	struct clist link;
	/**
	 * @brief Optional extra data pointer, reserved for private use.
	 * The current owner of the page may use this to track the underlying
	 * object in memory (or pretty much anything else), for example the
	 * `struct slab` if this page is currently used by the slab allocator.
	 */
	void *extra;
};

typedef struct vm_page *vm_page_t;

/** @brief Array of every single page in physical memory, indexed by page frame number. */
extern struct vm_page *const vm_page_array;

#if CFG_DEBUG_PGADDRS
extern vm_page_t _vm_page_array_end;
#define PGADDR_ASSERT(x) KASSERT(x)
#else
#define PGADDR_ASSERT(x) ({})
#endif

static inline u8 pga_order(vm_page_t page)
{
	union vm_page_attr attr = { ._val = atom_read(&page->attr) };
	return attr.order;
}

static inline bool pga_rsvd(vm_page_t page)
{
	union vm_page_attr attr = { ._val = atom_read(&page->attr) };
	return attr.rsvd;
}

static inline bool pga_pcpu(vm_page_t page)
{
	union vm_page_attr attr = { ._val = atom_read(&page->attr) };
	return attr.pcpu;
}

static inline bool pga_slab(vm_page_t page)
{
	union vm_page_attr attr = { ._val = atom_read(&page->attr) };
	return attr.slab;
}

static inline enum mm_zone_type pga_zone(vm_page_t page)
{
	union vm_page_attr attr = { ._val = atom_read(&page->attr) };
	return attr.zone;
}

static inline u8 pga_set_order(vm_page_t page, u8 order)
{
	spin_loop {
		union vm_page_attr old = { ._val = atom_read(&page->attr) };
		union vm_page_attr new = old;
		new.order = order;
		if (atom_cmp_xchg(&page->attr, old._val, new._val) == old._val)
			return old.order;
	}
}

static inline bool pga_set_pcpu(vm_page_t page, bool pcpu)
{
	if (pcpu)
		return atom_set_bit(&page->attr, _PGA_PCPU_SHIFT);
	else
		return atom_clr_bit(&page->attr, _PGA_PCPU_SHIFT);
}

static inline bool pga_set_slab(vm_page_t page, bool slab)
{
	if (slab)
		return atom_set_bit(&page->attr, _PGA_SLAB_SHIFT);
	else
		return atom_clr_bit(&page->attr, _PGA_SLAB_SHIFT);
}

static inline enum mm_zone_type pga_set_zone(vm_page_t page, enum mm_zone_type zone)
{
	spin_loop {
		union vm_page_attr old = { ._val = atom_read(&page->attr) };
		union vm_page_attr new = old;
		new.zone = zone;
		if (atom_cmp_xchg(&page->attr, old._val, new._val) == old._val)
			return old.zone;
	}
}

static __always_inline bool page_get(vm_page_t page)
{
	return atom_inc(&page->count);
}

static __always_inline bool page_put(vm_page_t page)
{
	return atom_dec(&page->count);
}

/* XXX we should probably use a wait queue for these rather than a spinlock like thing */

static inline void page_lock(vm_page_t page)
{
	spin_loop {
		if (atom_set_bit(&page->attr, _PGA_LOCK_SHIFT))
			break;
	}
}

static __always_inline void page_unlock(vm_page_t page)
{
	atom_clr_bit(&page->attr, _PGA_LOCK_SHIFT);
}

static __always_inline bool page_trylock(vm_page_t page)
{
	return atom_set_bit(&page->attr, _PGA_LOCK_SHIFT);
}

static inline void __page_set_flag(vm_page_t page, unsigned flag)
{
	atom_or(&page->attr, (int)flag);
}

static inline void __page_clr_flag(vm_page_t page, unsigned mask)
{
	atom_and(&page->attr, (int)~mask);
}

static __always_inline void page_attr_load(vm_page_attr_t *attr, vm_page_t page)
{
	attr->_val = atom_read(&page->attr);
}

static __always_inline void page_attr_copy(vm_page_attr_t *dest, const vm_page_attr_t *src)
{
	dest->_val = src->_val;
}

static __always_inline bool page_attr_cmp_xchg(vm_page_t page, const vm_page_attr_t *cmp,
					       const vm_page_attr_t *val)
{
	return atom_cmp_xchg(&page->attr, cmp->_val, val->_val);
}

/** @brief Get the page frame number of a page. */
__pure2
static inline u_long pg2pfn(vm_page_t page)
{
	PGADDR_ASSERT(page < _vm_page_array_end);
	return page - vm_page_array;
}

/**
 * @brief Get the page that a virtual address points to.
 * The address must point to the DMAP region (i.e. an address that is returned
 * by either `get_pages()` and friends, or `kmalloc()` and friends).
 */
__pure2
static inline vm_page_t vaddr2pg(void *vaddr)
{
	PGADDR_ASSERT(vaddr >= DMAP_START && vaddr < (void *)_vm_page_array_end);
	uintptr_t offset = (uintptr_t)vaddr - DMAP_OFFSET;
	return &vm_page_array[offset >> PAGE_SHIFT];
}

/**
 * @brief Get the page frame number for a virtual address.
 * The address must point to the DMAP region (i.e. an address that is returned
 * by either `get_pages()` and friends, or `kmalloc()` and friends).
 */
__pure2
static inline u_long vaddr2pfn(void *vaddr)
{
	u_long pfn = ((uintptr_t)vaddr - DMAP_OFFSET) >> PAGE_SHIFT;
	PGADDR_ASSERT(vaddr >= DMAP_START && &vm_page_array[pfn] < _vm_page_array_end);
	return pfn;
}

/** @brief Get the page frame number for a physical address. */
__pure2
static inline u_long paddr2pfn(vm_paddr_t paddr)
{
	PGADDR_ASSERT(&vm_page_array[paddr >> PAGE_SHIFT] < _vm_page_array_end);
	return paddr >> PAGE_SHIFT;
}

/** @brief Get the page that a physical address belongs to. */
__pure2
static inline vm_page_t paddr2pg(vm_paddr_t paddr)
{
	vm_page_t page = vm_page_array + (paddr >> PAGE_SHIFT);
	PGADDR_ASSERT(page < _vm_page_array_end);
	return page;
}

/**
 * @brief Translate a page frame number to its corresponding virtual address
 * in the DMAP region.
 */
__pure2
static inline void *pfn2vaddr(u_long pfn)
{
	PGADDR_ASSERT(&vm_page_array[pfn] < _vm_page_array_end);
	return DMAP_START + (pfn << PAGE_SHIFT);
}

/*
 * We have to be careful in this macro, because only the first page in the
 * order group has the correct order set.  So we can only read it once at
 * the beginning of the loop, since the page pointer is being updated.
 */

/**
 * @brief Iterate over every page in its order group.
 *
 * @param page The first `vm_page_t` in the group.
 */
#define vm_page_foreach_in_order(page)		\
	for (int __i = 1 << pga_order(page);	\
	     __i >= 0;				\
	     __i = ({ ++(page); --__i; }))