add kqueue API

kqueues are going to form the basis for anything
related to I/O and IPC.  They are a lock-free,
atomic FIFO queue and support multiple emitters
and consumers.

include/gay/kqueue.h

@@ -0,0 +1,115 @@
+/* See the end of this file for copyright and license terms. */
+
+#pragma once
+
+#include <arch/atom.h>
+
+#include <gay/mm.h>
+#include <gay/types.h>
+#include <gay/util.h>
+
+/**
+ * @defgroup kqueue Lock Free FIFO (kqueue)
+ *
+ * kqueue is a simple API that allows any structure to become part of a
+ * lock-free FIFO queue.  It only uses atomic primitives and spin-wait loops,
+ * which makes it safe to use even from irq context.  Furthermore, it supports
+ * multiple concurrent emitters *and* consumers, making it suitable for stuff
+ * like serving as the backend for a basic batch processing system distributed
+ * across several CPUs.
+ *
+ * @{
+ */
+
+/** @brief Embed this into structs that you want to put in a kqueue. */
+struct kq_node {
+	patom_t next;
+};
+
+/** @brief Lock free FIFO with support for multiple emitters and consumers. */
+struct kqueue {
+	patom_t head;
+	patom_t tail;
+	struct kq_node _dummy;
+};
+
+/**
+ * @brief Initialize a kqueue.
+ *
+ * @param kq kqueue to initialize
+ */
+static inline void kq_init(struct kqueue *kq)
+{
+	patom_init(&kq->head, &kq->_dummy);
+	patom_init(&kq->tail, &kq->_dummy);
+	patom_init(&kq->_dummy.next, nil);
+}
+
+/**
+ * @brief Allocate a new kqueue and initialize it.
+ *
+ * @param flags Flags for `kmalloc()`
+ * @return The initialized kqueue, or `nil` if OOM
+ */
+struct kqueue *kq_create(enum mflags flags);
+
+/**
+ * @brief Add a new entry to the end of a kqueue.
+ *
+ * @param kq kqueue to append the entry to
+ * @param node New node to append to the queue
+ */
+void kq_add(struct kqueue *kq, struct kq_node *node);
+
+/**
+ * @brief Remove the first item in a kqueue.
+ *
+ * @param kq kqueue to remove an item from
+ * @return The removed item, or `nil` if the queue was empty
+ */
+struct kq_node *kq_del(struct kqueue *kq);
+
+/**
+ * @brief Cast a kqueue entry out to the structure it is embedded in.
+ *
+ * @param head The `kq_node_t *` to cast out of
+ * @param type Type of the containing structure
+ * @param member Name of the `kq_node_t` within the structure
+ * @returns The `struct *` that `head` is embedded in
+ */
+#define kq_entry(head, type, member) container_of(head, type, member)
+
+/**
+ * @brief Remove the first item in a kqueue and return the structure
+ * it is embedded in.
+ *
+ * @param kq kqueue to remove an item from
+ * @param type Type of the containing structure
+ * @param member Name of the `kq_node_t` within the structure
+ * @returns The `struct *` that was removed from the queue,
+ *	or `nil` if the queue was empty
+ */
+#define kq_del_entry(kq, type, member) ({			\
+	struct kq_node *__node = kq_del(kq);			\
+        /* NOLINTNEXTLINE(bugprone-macro-parentheses) */	\
+	type *__entry = nil;					\
+	if (__node != nil)					\
+		__entry = kq_entry(__node, type, member);	\
+	__entry;						\
+})
+
+/** @} */
+
+/*
+ * This file is part of GayBSD.
+ * Copyright (c) 2021 fef <owo@fef.moe>.
+ *
+ * GayBSD is nonviolent software: you may only use, redistribute, and/or
+ * modify it under the terms of the Cooperative Nonviolent Public License
+ * (CNPL) as found in the LICENSE file in the source code root directory
+ * or at <https://git.pixie.town/thufie/npl-builder>; either version 7
+ * of the license, or (at your option) any later version.
+ *
+ * GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent
+ * permitted by applicable law.  See the CNPL for details.
+ */

include/gay/util.h

@@ -30,6 +30,9 @@
 	__a < __b ? __a : __b;	\
 })
 
+/** @brief Get type type of a structure member. */
+#define __typeof_member(type, member) typeof( ((type *)0)->member )
+
 /**
  * @brief Cast a pointer to a struct member out to the containing structure.
  *
@@ -38,8 +41,11 @@
  * @param member Name of the member within the containing structure
  * @returns Pointer to the containing structure
  */
-#define container_of(ptr, type, member) \
+#define container_of(ptr, type, member) ({				\
-	((type *)( (void *)(ptr) - offsetof(type, member) ))
+	/* implicit type check to avoid obvious mistakes */		\
+        __typeof_member(type, member) *__membptr = (ptr);		\
+	(type *)( (void *)__membptr - offsetof(type, member) );		\
+})
 
 /** @brief Determine whether `ptr` is kinda sus. */
 static __always_inline bool sus_nil(const void *ptr)

kernel/CMakeLists.txt

@@ -11,6 +11,7 @@ target_sources(gay_kernel PRIVATE
     clist.c
     irq.c
     kprintf.c
+    kqueue.c
     main.c
     mutex.c
     panic.c

kernel/kqueue.c

@@ -0,0 +1,160 @@
+/* See the end of this file for copyright and license terms. */
+
+/*
+ * This is an implementation of the nonblocking queue algorithm presented in:
+ * M. M. Michael and M. L. Scott, "Nonblocking Algorithms and Preemption-Safe
+ * Locking on Multiprogrammed Shared Memory Multiprocessors,"
+ * Journal of Parallel and Distributed Computing, vol 51, no 1, pp 1–26, 1998.
+ *
+ * <https://doi.org/10.1006/jpdc.1998.1446>
+ * <https://www.cs.rochester.edu/u/scott/papers/1998_JPDC_nonblocking.pdf>
+ */
+
+#include <arch/atom.h>
+#include <arch/cpufunc.h>
+
+#include <gay/kqueue.h>
+#include <gay/mm.h>
+#include <gay/types.h>
+#include <gay/util.h>
+
+#if __SIZEOF_POINTER__ == 4
+#define VERSION_BITS 2
+#elif __SIZEOF_POINTER__ == 8
+#define VERSION_BITS 3
+#else
+#error "Unsupported pointer size"
+#endif
+
+/**
+ * We use the least significant bits of pointers as a "version number" in order
+ * to (kind of) solve the ABA problem while still being able to write out both
+ * version and pointer in one atomic instruction.
+ * It is safe to use these bits because both the stack and memory returned by
+ * `kmalloc()` are always aligned to least a full pointer size, and structure
+ * members always get aligned to their own size as well.  So, on 32-bit we
+ * have 2 bits and on 64-bit we have 3 bits that are known to always be zero.
+ *
+ * This is still going to break if exactly `n * (1 << VERSION_BITS)` items are
+ * added/removed consecutively between our own read/exchange cycle, *and* the
+ * last item has the same address as the original one.  I assume it's *very*
+ * unlikely this is ever gonna happen though, since it would already be unusual
+ * for the same item to be enqueued more than once at all.
+ */
+union verptr {
+	unsigned ver:VERSION_BITS;
+	struct kq_node *ptr;
+};
+/** Get the actual pointer value, without the version number. */
+#define VERPTR(verptr) align_floor((verptr).ptr, 1 << VERSION_BITS)
+
+struct kqueue *kq_create(enum mflags flags)
+{
+	struct kqueue *kq = kmalloc(sizeof(*kq), flags);
+	if (kq != nil)
+		kq_init(kq);
+	return kq;
+}
+
+void kq_add(struct kqueue *kq, struct kq_node *_node)
+{
+	patom_init(&_node->next, nil);
+
+	union verptr node, tail, next;
+	node.ptr = _node;
+
+	spin_loop {
+		tail.ptr = patom_read(&kq->tail);
+		/* there is always at least one entry in the queue (the dummy),
+		 * so we know for sure that the tail can never be nil */
+		next.ptr = patom_read(&VERPTR(tail)->next);
+
+		/* If the head has changed during the time we read the other pointer,
+		 * it means another thread has added an item.  The tail and next
+		 * pointers are inconsistent in that case, so we have to try again. */
+		if (tail.ptr == patom_read(&kq->tail)) {
+			/* Check if tail really points to the last node */
+			if (VERPTR(next) == nil) {
+				/* tail is up-to-date, try appending the new node */
+				node.ver = next.ver + 1u;
+				void *tmp = patom_cmp_xchg(&VERPTR(tail)->next, next.ptr, node.ptr);
+				if (tmp == next.ptr)
+					break;
+			} else {
+				/* tail is falling behind, try to make it catch up
+				 * (this loops until it points to the actual end) */
+				next.ver = tail.ver + 1u;
+				patom_cmp_xchg(&kq->tail, tail.ptr, next.ptr);
+			}
+		}
+	}
+
+	/*
+	 * When we reach this, the new node has been inserted into the actual
+	 * linked list.  What's left is to update the tail pointer, which we
+	 * optimistically try once.  If that fails, we take care of the mess
+	 * next time an item is added or removed from the queue, because it's
+	 * not *critical* that the tail always stays consistent.  It's only
+	 * there so we don't need to iterate over the entire list whenever we
+	 * insert something.
+	 */
+	node.ver = tail.ver + 1u;
+	patom_cmp_xchg(&kq->tail, tail.ptr, node.ptr);
+}
+
+struct kq_node *kq_del(struct kqueue *kq)
+{
+	union verptr head, tail, next, ret;
+
+	spin_loop {
+		head.ptr = patom_read(&kq->head);
+		tail.ptr = patom_read(&kq->tail);
+		/* there is always at least one entry in the queue (the dummy),
+		 * so we know for sure that the head can never be nil */
+		next.ptr = patom_read(&VERPTR(head)->next);
+
+		/* If the head has changed during the time we read the other two pointers,
+		 * it means another thread has removed an item.  The head, tail, and next
+		 * pointers are inconsistent in that case, so we have to try again. */
+		if (head.ptr == patom_read(&kq->head)) {
+			/* Check if the queue is empty or if the tail is falling behind,
+			 * the latter happens if the last kq_add() failed to update it */
+			if (head.ptr == tail.ptr) {
+				/* Check if the queue is empty */
+				if (VERPTR(next) == nil) {
+					/* it is, so we are done here */
+					ret.ptr = nil;
+					break;
+				} else {
+					/* it is not, so try to update the tail */
+					next.ver = tail.ver + 1u;
+					patom_cmp_xchg(&kq->tail, tail.ptr, next.ptr);
+				}
+			} else {
+				/* Queue is not empty and the tail is consistent,
+				 * that means we can try removing the first node */
+				ret.ptr = next.ptr;
+				next.ver = head.ver + 1u;
+				void *tmp = patom_cmp_xchg(&kq->head, head.ptr, next.ptr);
+				if (tmp == head.ptr)
+					break;
+			}
+		}
+	}
+
+	return VERPTR(ret);
+}
+
+/*
+ * This file is part of GayBSD.
+ * Copyright (c) 2021 fef <owo@fef.moe>.
+ *
+ * GayBSD is nonviolent software: you may only use, redistribute, and/or
+ * modify it under the terms of the Cooperative Nonviolent Public License
+ * (CNPL) as found in the LICENSE file in the source code root directory
+ * or at <https://git.pixie.town/thufie/npl-builder>; either version 7
+ * of the license, or (at your option) any later version.
+ *
+ * GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent
+ * permitted by applicable law.  See the CNPL for details.
+ */