diff --git a/include/gay/kqueue.h b/include/gay/kqueue.h new file mode 100644 index 0000000..2cc8abd --- /dev/null +++ b/include/gay/kqueue.h @@ -0,0 +1,115 @@ +/* See the end of this file for copyright and license terms. */ + +#pragma once + +#include + +#include +#include +#include + +/** + * @defgroup kqueue Lock Free FIFO (kqueue) + * + * kqueue is a simple API that allows any structure to become part of a + * lock-free FIFO queue. It only uses atomic primitives and spin-wait loops, + * which makes it safe to use even from irq context. Furthermore, it supports + * multiple concurrent emitters *and* consumers, making it suitable for stuff + * like serving as the backend for a basic batch processing system distributed + * across several CPUs. + * + * @{ + */ + +/** @brief Embed this into structs that you want to put in a kqueue. */ +struct kq_node { + patom_t next; +}; + +/** @brief Lock free FIFO with support for multiple emitters and consumers. */ +struct kqueue { + patom_t head; + patom_t tail; + struct kq_node _dummy; +}; + +/** + * @brief Initialize a kqueue. + * + * @param kq kqueue to initialize + */ +static inline void kq_init(struct kqueue *kq) +{ + patom_init(&kq->head, &kq->_dummy); + patom_init(&kq->tail, &kq->_dummy); + patom_init(&kq->_dummy.next, nil); +} + +/** + * @brief Allocate a new kqueue and initialize it. + * + * @param flags Flags for `kmalloc()` + * @return The initialized kqueue, or `nil` if OOM + */ +struct kqueue *kq_create(enum mflags flags); + +/** + * @brief Add a new entry to the end of a kqueue. + * + * @param kq kqueue to append the entry to + * @param node New node to append to the queue + */ +void kq_add(struct kqueue *kq, struct kq_node *node); + +/** + * @brief Remove the first item in a kqueue. + * + * @param kq kqueue to remove an item from + * @return The removed item, or `nil` if the queue was empty + */ +struct kq_node *kq_del(struct kqueue *kq); + +/** + * @brief Cast a kqueue entry out to the structure it is embedded in. + * + * @param head The `kq_node_t *` to cast out of + * @param type Type of the containing structure + * @param member Name of the `kq_node_t` within the structure + * @returns The `struct *` that `head` is embedded in + */ +#define kq_entry(head, type, member) container_of(head, type, member) + +/** + * @brief Remove the first item in a kqueue and return the structure + * it is embedded in. + * + * @param kq kqueue to remove an item from + * @param type Type of the containing structure + * @param member Name of the `kq_node_t` within the structure + * @returns The `struct *` that was removed from the queue, + * or `nil` if the queue was empty + */ +#define kq_del_entry(kq, type, member) ({ \ + struct kq_node *__node = kq_del(kq); \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + type *__entry = nil; \ + if (__node != nil) \ + __entry = kq_entry(__node, type, member); \ + __entry; \ +}) + +/** @} */ + +/* + * This file is part of GayBSD. + * Copyright (c) 2021 fef . + * + * GayBSD is nonviolent software: you may only use, redistribute, and/or + * modify it under the terms of the Cooperative Nonviolent Public License + * (CNPL) as found in the LICENSE file in the source code root directory + * or at ; either version 7 + * of the license, or (at your option) any later version. + * + * GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent + * permitted by applicable law. See the CNPL for details. + */ diff --git a/include/gay/util.h b/include/gay/util.h index 096c774..386462d 100644 --- a/include/gay/util.h +++ b/include/gay/util.h @@ -30,6 +30,9 @@ __a < __b ? __a : __b; \ }) +/** @brief Get type type of a structure member. */ +#define __typeof_member(type, member) typeof( ((type *)0)->member ) + /** * @brief Cast a pointer to a struct member out to the containing structure. * @@ -38,8 +41,11 @@ * @param member Name of the member within the containing structure * @returns Pointer to the containing structure */ -#define container_of(ptr, type, member) \ - ((type *)( (void *)(ptr) - offsetof(type, member) )) +#define container_of(ptr, type, member) ({ \ + /* implicit type check to avoid obvious mistakes */ \ + __typeof_member(type, member) *__membptr = (ptr); \ + (type *)( (void *)__membptr - offsetof(type, member) ); \ +}) /** @brief Determine whether `ptr` is kinda sus. */ static __always_inline bool sus_nil(const void *ptr) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 8baf616..8f78549 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -11,6 +11,7 @@ target_sources(gay_kernel PRIVATE clist.c irq.c kprintf.c + kqueue.c main.c mutex.c panic.c diff --git a/kernel/kqueue.c b/kernel/kqueue.c new file mode 100644 index 0000000..2730fea --- /dev/null +++ b/kernel/kqueue.c @@ -0,0 +1,160 @@ +/* See the end of this file for copyright and license terms. */ + +/* + * This is an implementation of the nonblocking queue algorithm presented in: + * M. M. Michael and M. L. Scott, "Nonblocking Algorithms and Preemption-Safe + * Locking on Multiprogrammed Shared Memory Multiprocessors," + * Journal of Parallel and Distributed Computing, vol 51, no 1, pp 1–26, 1998. + * + * + * + */ + +#include +#include + +#include +#include +#include +#include + +#if __SIZEOF_POINTER__ == 4 +#define VERSION_BITS 2 +#elif __SIZEOF_POINTER__ == 8 +#define VERSION_BITS 3 +#else +#error "Unsupported pointer size" +#endif + +/** + * We use the least significant bits of pointers as a "version number" in order + * to (kind of) solve the ABA problem while still being able to write out both + * version and pointer in one atomic instruction. + * It is safe to use these bits because both the stack and memory returned by + * `kmalloc()` are always aligned to least a full pointer size, and structure + * members always get aligned to their own size as well. So, on 32-bit we + * have 2 bits and on 64-bit we have 3 bits that are known to always be zero. + * + * This is still going to break if exactly `n * (1 << VERSION_BITS)` items are + * added/removed consecutively between our own read/exchange cycle, *and* the + * last item has the same address as the original one. I assume it's *very* + * unlikely this is ever gonna happen though, since it would already be unusual + * for the same item to be enqueued more than once at all. + */ +union verptr { + unsigned ver:VERSION_BITS; + struct kq_node *ptr; +}; +/** Get the actual pointer value, without the version number. */ +#define VERPTR(verptr) align_floor((verptr).ptr, 1 << VERSION_BITS) + +struct kqueue *kq_create(enum mflags flags) +{ + struct kqueue *kq = kmalloc(sizeof(*kq), flags); + if (kq != nil) + kq_init(kq); + return kq; +} + +void kq_add(struct kqueue *kq, struct kq_node *_node) +{ + patom_init(&_node->next, nil); + + union verptr node, tail, next; + node.ptr = _node; + + spin_loop { + tail.ptr = patom_read(&kq->tail); + /* there is always at least one entry in the queue (the dummy), + * so we know for sure that the tail can never be nil */ + next.ptr = patom_read(&VERPTR(tail)->next); + + /* If the head has changed during the time we read the other pointer, + * it means another thread has added an item. The tail and next + * pointers are inconsistent in that case, so we have to try again. */ + if (tail.ptr == patom_read(&kq->tail)) { + /* Check if tail really points to the last node */ + if (VERPTR(next) == nil) { + /* tail is up-to-date, try appending the new node */ + node.ver = next.ver + 1u; + void *tmp = patom_cmp_xchg(&VERPTR(tail)->next, next.ptr, node.ptr); + if (tmp == next.ptr) + break; + } else { + /* tail is falling behind, try to make it catch up + * (this loops until it points to the actual end) */ + next.ver = tail.ver + 1u; + patom_cmp_xchg(&kq->tail, tail.ptr, next.ptr); + } + } + } + + /* + * When we reach this, the new node has been inserted into the actual + * linked list. What's left is to update the tail pointer, which we + * optimistically try once. If that fails, we take care of the mess + * next time an item is added or removed from the queue, because it's + * not *critical* that the tail always stays consistent. It's only + * there so we don't need to iterate over the entire list whenever we + * insert something. + */ + node.ver = tail.ver + 1u; + patom_cmp_xchg(&kq->tail, tail.ptr, node.ptr); +} + +struct kq_node *kq_del(struct kqueue *kq) +{ + union verptr head, tail, next, ret; + + spin_loop { + head.ptr = patom_read(&kq->head); + tail.ptr = patom_read(&kq->tail); + /* there is always at least one entry in the queue (the dummy), + * so we know for sure that the head can never be nil */ + next.ptr = patom_read(&VERPTR(head)->next); + + /* If the head has changed during the time we read the other two pointers, + * it means another thread has removed an item. The head, tail, and next + * pointers are inconsistent in that case, so we have to try again. */ + if (head.ptr == patom_read(&kq->head)) { + /* Check if the queue is empty or if the tail is falling behind, + * the latter happens if the last kq_add() failed to update it */ + if (head.ptr == tail.ptr) { + /* Check if the queue is empty */ + if (VERPTR(next) == nil) { + /* it is, so we are done here */ + ret.ptr = nil; + break; + } else { + /* it is not, so try to update the tail */ + next.ver = tail.ver + 1u; + patom_cmp_xchg(&kq->tail, tail.ptr, next.ptr); + } + } else { + /* Queue is not empty and the tail is consistent, + * that means we can try removing the first node */ + ret.ptr = next.ptr; + next.ver = head.ver + 1u; + void *tmp = patom_cmp_xchg(&kq->head, head.ptr, next.ptr); + if (tmp == head.ptr) + break; + } + } + } + + return VERPTR(ret); +} + +/* + * This file is part of GayBSD. + * Copyright (c) 2021 fef . + * + * GayBSD is nonviolent software: you may only use, redistribute, and/or + * modify it under the terms of the Cooperative Nonviolent Public License + * (CNPL) as found in the LICENSE file in the source code root directory + * or at ; either version 7 + * of the license, or (at your option) any later version. + * + * GayBSD comes with ABSOLUTELY NO WARRANTY, to the extent + * permitted by applicable law. See the CNPL for details. + */