diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 72523685..00ffe09a 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -24,7 +24,7 @@ jobs: ########################################################################### Run-Checks: runs-on: ubuntu-22.04 - container: ghcr.io/open-s4c/vsyncer-ci:sha-25de355efbbb7bf9d85523aeb50864b118755392 + container: ghcr.io/open-s4c/vsyncer-ci:sha-76569f0d2ee9a688da95b8025116b3a202e67321 strategy: matrix: test-dir: [ {p: "test", c: "spinlock"}, {p: "test", c: "quack"}, {p: "test", c: "queue"}, @@ -32,7 +32,8 @@ jobs: {p: "verify", c: "stack"}, {p: "verify", c: "thread"}, {p: "verify", c: "simpleht"}, {p: "verify", c: "bitmap"}, {p: "verify", c: "treeset"}, {p: "verify", c: "pool"}, {p: "verify", c: "cachedq"}, {p: "verify", c: "chaselev"} , {p: "verify", c: "priority_queue"}, - {p: "verify", c: "skiplist"}, {p: "verify", c: "smr"} + {p: "verify", c: "skiplist"}, {p: "verify", c: "smr"}, {p: "verify", c: "bbq"}, + {p: "verify", c: "mpsc"} ] steps: - name: Print vsyncer version diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e611902..a0397c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ is not guaranteed to result in increment of major version. Please note that the version correlates to the internal libvsync, which is a superset of what exists in open-s4c libvsync. +## [4.3.0] + +### Added + +- mpsc queue +- bbq_mpmc and bbq_spsc + ## [4.2.2] ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 301c4070..52c83a5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16) project( libvsync LANGUAGES C - VERSION 4.2.2 + VERSION 4.3.0 DESCRIPTION "Verified library of atomics, synchronization primitives and concurrent data structures" ) diff --git a/cmake/check.cmake b/cmake/check.cmake index b9f97dcf..08e02cd6 100644 --- a/cmake/check.cmake +++ b/cmake/check.cmake @@ -141,7 +141,7 @@ function(add_vsyncer_check) --timeout ${TIMEOUT}s) add_test(NAME ${TEST_NAME} COMMAND ${VSYNCER_CMD} ${VSYNCER_CHECK_LL}) - set_property(TEST ${TEST_NAME} PROPERTY SKIP_RETURN_CODE 1) + # set_property(TEST ${TEST_NAME} PROPERTY SKIP_RETURN_CODE 1) math(EXPR CTEST_TIMEOUT "${TIMEOUT} + 5") set_tests_properties(${TEST_NAME} PROPERTIES TIMEOUT ${CTEST_TIMEOUT}) endforeach() diff --git a/doc/api/vsync/GROUP_linearizable.md b/doc/api/vsync/GROUP_linearizable.md index 9463e130..838e4f6f 100644 --- a/doc/api/vsync/GROUP_linearizable.md +++ b/doc/api/vsync/GROUP_linearizable.md @@ -19,6 +19,7 @@ _Group of algorithms linearizable algorithms._ | [vsync/map/treeset_bst_fine.h](map/treeset_bst_fine.h.md)|This implementation of treeset uses unbalanced binary search tree (BST) and fine-grained locking. | ✔ | ❌ | ❌ | ❌ | | [vsync/map/treeset_rb_coarse.h](map/treeset_rb_coarse.h.md)|This implementation of treeset uses balanced red-black tree (RB) and coarse-grained locking. | ✔ | ❌ | ❌ | ❌ | | [vsync/map/treeset_rb_fine.h](map/treeset_rb_fine.h.md)|This implementation of treeset uses balanced red-black tree (RB) and fine-grained locking. | ✔ | ❌ | ❌ | ❌ | +| [vsync/queue/bbq_spsc.h](queue/bbq_spsc.h.md)|Block-based Bounded Queue single-producer/single-consumer. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/bounded_locked.h](queue/bounded_locked.h.md)|Multi-producer, multi-consumer bounded queue protected by a spinlock. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_mpmc.h](queue/bounded_mpmc.h.md)|Lockless, multi-producer, multi-consumer bounded queue. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_spsc.h](queue/bounded_spsc.h.md)|Single-producer, single-consumer, wait-free bounded queue. | ✔ | ✔ | ❌ | ❌ | diff --git a/doc/api/vsync/GROUP_lock_free.md b/doc/api/vsync/GROUP_lock_free.md index 02463814..dd0d153d 100644 --- a/doc/api/vsync/GROUP_lock_free.md +++ b/doc/api/vsync/GROUP_lock_free.md @@ -11,6 +11,7 @@ _Group of algorithms with lock-free progress condition._ | [vsync/map/listset_lf.h](map/listset_lf.h.md)|Lock-free implementation of listset. | ✔ | ✔ | ✔ | ❌ | | [vsync/map/simpleht.h](map/simpleht.h.md)|Simple lock-free hashtable. | ✔ | ✔ | ❌ | ❌ | | [vsync/map/skiplist_lf.h](map/skiplist_lf.h.md)|Lock-free concurrent skiplist. | ✔ | ✔ | ✔ | ❌ | +| [vsync/queue/bbq_spsc.h](queue/bbq_spsc.h.md)|Block-based Bounded Queue single-producer/single-consumer. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/bounded_spsc.h](queue/bounded_spsc.h.md)|Single-producer, single-consumer, wait-free bounded queue. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/chaselev.h](queue/chaselev.h.md)|Chase-Lev Work-Stealing bounded deque. | ❌ | ✔ | ❌ | ❌ | | [vsync/queue/unbounded_queue_lf.h](queue/unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | diff --git a/doc/api/vsync/pool/cached_pool.h.md b/doc/api/vsync/pool/cached_pool.h.md index 6c72a4d4..848f3f04 100644 --- a/doc/api/vsync/pool/cached_pool.h.md +++ b/doc/api/vsync/pool/cached_pool.h.md @@ -96,7 +96,7 @@ _Calculate the needed memory space for creating a pool._ **Parameters:** -- `thread_num`: maxinum thread number +- `thread_num`: maximum thread number - `entry_num`: minimal number of entires - `entry_size`: size of each entry @@ -129,7 +129,7 @@ Make sure the buffer has enough size (calculated by cached_pool_memsize) **Parameters:** - `buf`: pointer to the buffer -- `thread_num`: maxinum thread number +- `thread_num`: maximum thread number - `entry_num`: minimal number of entires - `entry_size`: size of each entry diff --git a/doc/api/vsync/queue/GROUP_unbounded_queue.md b/doc/api/vsync/queue/GROUP_unbounded_queue.md index 707b5d70..a8495c81 100644 --- a/doc/api/vsync/queue/GROUP_unbounded_queue.md +++ b/doc/api/vsync/queue/GROUP_unbounded_queue.md @@ -9,6 +9,7 @@ These queues have no capacity limit and thus enqueue operations shall always suc | File|Description|Linearizable|Lock-free|SMR-required|Unbounded-Queue| | --- | --- | --- | --- | --- | --- | +| [vsync/queue/mpsc.h](mpsc.h.md)|Multi-producer single-consumer queue. | ❌ | ❌ | ❌ | ✔ | | [vsync/queue/unbounded_queue_lf.h](unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | | [vsync/queue/unbounded_queue_lf_recycle.h](unbounded_queue_lf_recycle.h.md)|Lock-free recycle unbounded queue. | ✔ | ✔ | ❌ | ✔ | | [vsync/queue/unbounded_queue_total.h](unbounded_queue_total.h.md)|Unbounded blocking total queue. | ✔ | ❌ | ❌ | ✔ | diff --git a/doc/api/vsync/queue/README.md b/doc/api/vsync/queue/README.md index da4fcb2f..d2360215 100644 --- a/doc/api/vsync/queue/README.md +++ b/doc/api/vsync/queue/README.md @@ -7,11 +7,14 @@ _Queues, priority queues and ringbuffers._ | File|Description|Linearizable|Lock-free|SMR-required|Unbounded-Queue| | --- | --- | --- | --- | --- | --- | +| [vsync/queue/bbq_mpmc.h](bbq_mpmc.h.md)|Block-based Bounded Queue multi-producer/multi-consumer. | ❌ | ❌ | ❌ | ❌ | +| [vsync/queue/bbq_spsc.h](bbq_spsc.h.md)|Block-based Bounded Queue single-producer/single-consumer. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/bounded_locked.h](bounded_locked.h.md)|Multi-producer, multi-consumer bounded queue protected by a spinlock. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_mpmc.h](bounded_mpmc.h.md)|Lockless, multi-producer, multi-consumer bounded queue. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_spsc.h](bounded_spsc.h.md)|Single-producer, single-consumer, wait-free bounded queue. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/cachedq.h](cachedq.h.md)|Lockless, multi-producer, multi-consumer queue. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/chaselev.h](chaselev.h.md)|Chase-Lev Work-Stealing bounded deque. | ❌ | ✔ | ❌ | ❌ | +| [vsync/queue/mpsc.h](mpsc.h.md)|Multi-producer single-consumer queue. | ❌ | ❌ | ❌ | ✔ | | [vsync/queue/unbounded_queue_lf.h](unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | | [vsync/queue/unbounded_queue_lf_recycle.h](unbounded_queue_lf_recycle.h.md)|Lock-free recycle unbounded queue. | ✔ | ✔ | ❌ | ✔ | | [vsync/queue/unbounded_queue_total.h](unbounded_queue_total.h.md)|Unbounded blocking total queue. | ✔ | ❌ | ❌ | ✔ | diff --git a/doc/api/vsync/queue/bbq_mpmc.h.md b/doc/api/vsync/queue/bbq_mpmc.h.md new file mode 100644 index 00000000..0421c746 --- /dev/null +++ b/doc/api/vsync/queue/bbq_mpmc.h.md @@ -0,0 +1,239 @@ +# [vsync](../README.md) / [queue](README.md) / bbq_mpmc.h +_Block-based Bounded Queue multi-producer/multi-consumer._ + +A highly performant bounded queue that splits the buffer in multiple blocks. + +### Remarks: + +In this implementation, values have the fixed size (pointer size). This implementation does not support `DROP_OLD` mode as described in the original paper. + + +### References: + [BBQ: A Block-based Bounded Queue for Exchanging Data and Profiling](https://www.usenix.org/conference/atc22/presentation/wang-jiawei) + + +### Example: + + + +```c +#include +#include +#include +#include + +#define BUFFER_ENTRY_NUM 4096U +#define NUM 10U +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL +#define NUM_WRITER 4U +#define NUM_READER 5U + +#define NON_BLOCKING false +#define BLOCKING true + +bbq_mpmc_t *g_bbq_mpmc = NULL; + +void * +writer(void *arg) +{ + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM * NUM_READER; + + while (rest) { + vuint32_t count = rest < ENQUEUE_BATCH ? rest : ENQUEUE_BATCH; + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ++ptr; + } + rest -= bbq_mpmc_enqueue(g_bbq_mpmc, buf, count, BLOCKING); + } + + (void)arg; + return NULL; +} + +void * +reader(void *arg) +{ + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t rest = NUM * NUM_WRITER; + + while (rest) { + vuint32_t batch_size = rest < DEQUEUE_BATCH ? rest : DEQUEUE_BATCH; + vuint32_t count = + bbq_mpmc_dequeue(g_bbq_mpmc, buf, batch_size, NON_BLOCKING); + + for (vuint32_t i = 0; i < count; i++) { + ASSERT(buf[i] > 0); + printf("dequeue item %lu\n", buf[i]); + } + rest -= count; + } + (void)arg; + return NULL; +} + +int +main(void) +{ + pthread_t t_writer[NUM_WRITER]; + pthread_t t_reader[NUM_READER]; + + // allocate + vsize_t sz = bbq_mpmc_memsize(BUFFER_ENTRY_NUM); + g_bbq_mpmc = (bbq_mpmc_t *)malloc(sz); + ASSERT(g_bbq_mpmc); + + // init + vbool_t success = bbq_mpmc_init(g_bbq_mpmc, sz); + ASSERT(success); + + for (vsize_t i = 0; i < NUM_WRITER; i++) { + pthread_create(&t_writer[i], NULL, writer, NULL); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_create(&t_reader[i], NULL, reader, NULL); + } + for (vsize_t i = 0; i < NUM_WRITER; i++) { + pthread_join(t_writer[i], NULL); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_join(t_reader[i], NULL); + } + + // deallocate + free(g_bbq_mpmc); + return 0; +} +``` + + + +--- +# Macros + +| Macro | Description | +|---|---| +| [BBQ_BLOCK_NUM_LOG](bbq_mpmc.h.md#macro-bbq_block_num_log) | Define this macro with `-DBBQ_BLOCK_NUM_LOG=N` to define the total number of blocks equals to `2^N` | +| [BBQ_ENTRY_SIZE_LOG](bbq_mpmc.h.md#macro-bbq_entry_size_log) | Define this macro with `-BBQ_ENTRY_SIZE_LOG=N` to define an entry size equals to `2^N` | + +## Macro `BBQ_BLOCK_NUM_LOG` + + +_Define this macro with_ `-DBBQ_BLOCK_NUM_LOG=N` _to define the total number of blocks equals to_ `2^N` __ + + +> **Note:** default value is `3U` -> (`8` blocks) + + +## Macro `BBQ_ENTRY_SIZE_LOG` + + +_Define this macro with_ `-BBQ_ENTRY_SIZE_LOG=N` _to define an entry size equals to_ `2^N` __ + + +> **Note:** default value is `log2(sizeof(vuintptr_t))` + + +--- +# Functions + +| Function | Description | +|---|---| +| [bbq_mpmc_enqueue](bbq_mpmc.h.md#function-bbq_mpmc_enqueue) | Enqueues one or more entries. | +| [bbq_mpmc_dequeue](bbq_mpmc.h.md#function-bbq_mpmc_dequeue) | Dequeues one or more entries. | +| [bbq_mpmc_memsize](bbq_mpmc.h.md#function-bbq_mpmc_memsize) | Calculates the size of the bbq queue. | +| [bbq_mpmc_init](bbq_mpmc.h.md#function-bbq_mpmc_init) | Initializes a bbq data structure. | + +## Function `bbq_mpmc_enqueue` + +```c +static vuint32_t bbq_mpmc_enqueue(bbq_mpmc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +``` +_Enqueues one or more entries._ + + +Multiple entries can be enqueued if `src` points to an array. Use `count` to indicate how many entries should be enqueued, starting from `src`. + + + +**Parameters:** + +- `q`: address of `bbq_mpmc_t` object. +- `src`: pointer to first entry. +- `count`: number of entries to enqueue. +- `wait`: should wait for space to be available. + + +**Returns:** number of enqueued entries. + + + +## Function `bbq_mpmc_dequeue` + +```c +static vuint32_t bbq_mpmc_dequeue(bbq_mpmc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +``` +_Dequeues one or more entries._ + + +Multiple entries can be dequeued if `src` points to an array. Use `count` to indicate how many entries should be dequeued. + + + +**Parameters:** + +- `q`: address of `bbq_mpmc_t` object. +- `src`: pointer to preallocated memory for the first entry. +- `count`: number of entries to dequeue. +- `wait`: should wait for entries to be available. + + +**Returns:** number of dequeued entries. + + + +## Function `bbq_mpmc_memsize` + +```c +static vsize_t bbq_mpmc_memsize(vsize_t capacity) +``` +_Calculates the size of the bbq queue._ + + + + +**Parameters:** + +- `capacity`: maximum number of entries that can fit in the queue. + + +**Returns:** size to be allocated in bytes. + + + +## Function `bbq_mpmc_init` + +```c +static vbool_t bbq_mpmc_init(bbq_mpmc_t *q, vsize_t size) +``` +_Initializes a bbq data structure._ + + + + +**Parameters:** + +- `q`: pointer to bbq data structure. +- `size`: number of bytes allocated for bbq data structure. + + +**Returns:** true initialization succeeded. + +**Returns:** false initialization failed. + + + + +--- diff --git a/doc/api/vsync/queue/bbq_spsc.h.md b/doc/api/vsync/queue/bbq_spsc.h.md new file mode 100644 index 00000000..ca2bed9e --- /dev/null +++ b/doc/api/vsync/queue/bbq_spsc.h.md @@ -0,0 +1,343 @@ +# [vsync](../README.md) / [queue](README.md) / bbq_spsc.h +_Block-based Bounded Queue single-producer/single-consumer._ + +**Groups:** [Linearizable](../GROUP_linearizable.md), [Lock-free](../GROUP_lock_free.md) + +A highly performant bounded queue that splits the buffer in multiple blocks. + +### Remarks: + +In this implementations, values have a fixed size equal to pointer size. This implementation does not support `DROP_OLD` mode as described in the original paper. + + +### References: + [BBQ: A Block-based Bounded Queue for Exchanging Data and Profiling](https://www.usenix.org/conference/atc22/presentation/wang-jiawei) + + +### Example: + +### Multi-threaded + + + +```c +#include +#include +#include +#include + +#define BUFFER_ENTRY_NUM 4096U +#define NUM 10U +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL + +#define NON_BLOCKING false +#define BLOCKING true + +bbq_spsc_t *g_bbq_spsc = NULL; + +void * +writer(void *arg) +{ + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + + while (rest) { + vuint32_t count = rest < ENQUEUE_BATCH ? rest : ENQUEUE_BATCH; + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ptr++; + } + rest -= bbq_spsc_enqueue(g_bbq_spsc, buf, count, BLOCKING); + } + + (void)arg; + return NULL; +} + +void * +reader(void *arg) +{ + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + + while (rest) { + vuint32_t count = + bbq_spsc_dequeue(g_bbq_spsc, buf, DEQUEUE_BATCH, NON_BLOCKING); + + for (vuint32_t i = 0; i < count; i++) { + ASSERT(buf[i] == ptr++); + printf("dequeue item %lu\n", buf[i]); + } + rest -= count; + } + (void)arg; + return NULL; +} + +int +main(void) +{ + pthread_t t_writer; + pthread_t t_reader; + + // allocate + vsize_t sz = bbq_spsc_memsize(BUFFER_ENTRY_NUM); + g_bbq_spsc = (bbq_spsc_t *)malloc(sz); + ASSERT(g_bbq_spsc); + + // init + vbool_t success = bbq_spsc_init(g_bbq_spsc, sz); + ASSERT(success); + + pthread_create(&t_writer, NULL, writer, NULL); + pthread_create(&t_reader, NULL, reader, NULL); + pthread_join(t_reader, NULL); + pthread_join(t_writer, NULL); + + // deallocate + free(g_bbq_spsc); + return 0; +} +``` + + + +### Multi-process + + + +```c +#include +#include +#include +#include +#include +#include +#include +#include + +#define QUEUE_SIZE 4095U +#define EPOCH_SIZE 100U +#define NUM_ENTRIES 1U +#define SLEEP_MICRO_SEC 10 +#define NON_BLOCKING false +#define BLOCKING true + +#include +#include +#include + +bbq_spsc_t *g_bbq; + +void * +create_shared_memory(vsize_t size) +{ + int protection = PROT_READ | PROT_WRITE; + int visibility = MAP_SHARED | MAP_ANONYMOUS; + return mmap(NULL, size, protection, visibility, -1, 0); +} + +void +check_space(void) +{ + vuint64_t data = 0; + vuint64_t count = 0; + + while (bbq_spsc_dequeue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + + while (bbq_spsc_enqueue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) { + count++; + } + while (bbq_spsc_dequeue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + + printf( + "Current size of the queue is %ld after %d crashes, shouldn't be " + "zero.\n", + count, EPOCH_SIZE); + assert(count != 0); +} + +void * +writer(void *arg) +{ + V_UNUSED(arg); + vuint64_t data = 0; + while (true) { + data = (data + 1) % (QUEUE_SIZE - 1); + while (!bbq_spsc_enqueue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + usleep(SLEEP_MICRO_SEC); + } +} + +void * +reader(void *arg) +{ + V_UNUSED(arg); + vuint64_t data; + while (true) { + while (!bbq_spsc_dequeue(g_bbq, &data, 1, NON_BLOCKING)) {} + usleep(SLEEP_MICRO_SEC); + assert(data < QUEUE_SIZE - 1); + } +} + +int +main(void) +{ + srand(time(NULL)); + vuint64_t sz = bbq_spsc_memsize(QUEUE_SIZE); + g_bbq = create_shared_memory(sz); + assert(g_bbq); + vbool_t success = bbq_spsc_init(g_bbq, sz); + ASSERT(success && "BBQ init failed"); + for (vuint32_t i = 0; i < EPOCH_SIZE; i++) { + pid_t pid = fork(); + assert(pid >= 0); + if (pid != 0) { + usleep(rand() % 10000); + while (kill(pid, SIGTERM) < 0) {} + } else { + pthread_t t_writer, t_reader; + pthread_create(&t_writer, NULL, writer, NULL); + pthread_create(&t_reader, NULL, reader, NULL); + pthread_join(t_writer, NULL); + pthread_join(t_reader, NULL); + return 0; + } + } + check_space(); + return 0; +} +``` + + + +--- +# Macros + +| Macro | Description | +|---|---| +| [BBQ_BLOCK_NUM_LOG](bbq_spsc.h.md#macro-bbq_block_num_log) | Define this macro with `-DBBQ_BLOCK_NUM_LOG=N` to define the total number of blocks equals to `2^N` | +| [BBQ_ENTRY_SIZE_LOG](bbq_spsc.h.md#macro-bbq_entry_size_log) | Define this macro with `-BBQ_ENTRY_SIZE_LOG=N` to define an entry size equals to `2^N` | + +## Macro `BBQ_BLOCK_NUM_LOG` + + +_Define this macro with_ `-DBBQ_BLOCK_NUM_LOG=N` _to define the total number of blocks equals to_ `2^N` __ + + +> **Note:** default value is `3U` -> (`8` blocks) + + +## Macro `BBQ_ENTRY_SIZE_LOG` + + +_Define this macro with_ `-BBQ_ENTRY_SIZE_LOG=N` _to define an entry size equals to_ `2^N` __ + + +> **Note:** default value is `log2(sizeof(vuintptr_t))` + + +--- +# Functions + +| Function | Description | +|---|---| +| [bbq_spsc_enqueue](bbq_spsc.h.md#function-bbq_spsc_enqueue) | Enqueues one or more entries. | +| [bbq_spsc_dequeue](bbq_spsc.h.md#function-bbq_spsc_dequeue) | Dequeues one or more entries. | +| [bbq_spsc_memsize](bbq_spsc.h.md#function-bbq_spsc_memsize) | Calculates the size of bbq_spsc_t object based on the given capacity. | +| [bbq_spsc_init](bbq_spsc.h.md#function-bbq_spsc_init) | Initializes a bbq data structure. | + +## Function `bbq_spsc_enqueue` + +```c +static vuint32_t bbq_spsc_enqueue(bbq_spsc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +``` +_Enqueues one or more entries._ + + +Multiple entries can be enqueued if `src` points to an array. Use `count` to indicate how many entries should be enqueued, starting from `src`. + + + +**Parameters:** + +- `q`: address of bbq_spsc_t object. +- `buf`: address of the first entry. +- `count`: number of entries to enqueue. +- `wait`: true/false when set to true it waits (blocks) till space becomes available. Otherwise, it quits retrying. + + +**Returns:** number of enqueued entries. + + + +## Function `bbq_spsc_dequeue` + +```c +static vuint32_t bbq_spsc_dequeue(bbq_spsc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +``` +_Dequeues one or more entries._ + + +Multiple entries can be dequeued if `buf` points to an array. Use `count` to indicate how many entries should be dequeued. + + + +**Parameters:** + +- `q`: address of bbq_spsc_t object. +- `buf`: address of the first entry of the preallocated memory. +- `count`: number of entries to dequeue. +- `wait`: true/false. When set to true the API waits/blocks for entries to be available + + +**Returns:** number of dequeued entries. + + + +## Function `bbq_spsc_memsize` + +```c +static vsize_t bbq_spsc_memsize(vsize_t capacity) +``` +_Calculates the size of bbq_spsc_t object based on the given capacity._ + + + + +**Parameters:** + +- `capacity`: maximum number of entries that can fit in the queue. + + +**Returns:** size of bbq_spsc_t object with the given capacity. + + + +## Function `bbq_spsc_init` + +```c +static vbool_t bbq_spsc_init(bbq_spsc_t *q, vsize_t size) +``` +_Initializes a bbq data structure._ + + + + +**Parameters:** + +- `q`: address of bbq_spsc_t object. +- `size`: size of the given bbq_spsc_t object `q`. + + +**Returns:** true initialization succeeded. + +**Returns:** false initialization failed. + + + + +--- diff --git a/doc/api/vsync/queue/mpsc.h.md b/doc/api/vsync/queue/mpsc.h.md new file mode 100644 index 00000000..ee41b9ba --- /dev/null +++ b/doc/api/vsync/queue/mpsc.h.md @@ -0,0 +1,260 @@ +# [vsync](../README.md) / [queue](README.md) / mpsc.h +_Multi-producer single-consumer queue._ + +**Groups:** [Unbounded-Queue](GROUP_unbounded_queue.md) + +Enqueue operation is wait-free. + + +### Example: + + + +```c +#include +#include +#include +#include + +vmpsc_t g_queue; + +#define NUM_PRODUCERS 3U +#define NUM_PROD_ITEMS 10U +#define NUM_CONS_ITEMS (NUM_PRODUCERS * NUM_PROD_ITEMS) + +typedef struct data_s { + vuint32_t x; + vuint32_t y; +} data_t; + +void +destroy_cb(vmpsc_node_t *node, void *args) +{ + free(node->data); + free(node); + V_UNUSED(args); +} + +void * +consume(void *args) +{ + vmpsc_node_t *n = NULL; + data_t *d = NULL; + for (vuint32_t i = 0; i < NUM_CONS_ITEMS; i++) { + while (d = vmpsc_deq(&g_queue, &n), d == NULL) { + /* busy wait till an item is consumed. Repeat on empty! */ + } + ASSERT(d); + ASSERT(d->x == d->y); + printf("consumed item: {%u, %u}\n", d->x, d->y); + free(d); + /* n can be NULL even if d is not. */ + free(n); + } + V_UNUSED(args); + return NULL; +} + +void * +produce(void *args) +{ + vuint32_t tid = (vuint32_t)(vuintptr_t)args; + for (vuint32_t i = 0; i < NUM_PROD_ITEMS; i++) { + data_t *d = malloc(sizeof(*d)); + ASSERT(d); + d->x = d->y = (tid * NUM_PRODUCERS) + i; + vmpsc_node_t *n = malloc(sizeof(*n)); + ASSERT(n); + vmpsc_enq(&g_queue, n, d); + } + V_UNUSED(args); + return NULL; +} + +int +main(void) +{ + vmpsc_init(&g_queue); + pthread_t consumer; + pthread_t producer[NUM_PRODUCERS]; + + pthread_create(&consumer, NULL, consume, NULL); + for (vsize_t i = 0; i < NUM_PRODUCERS; i++) { + pthread_create(&producer[i], NULL, produce, (void *)(vuintptr_t)i); + } + for (vsize_t i = 0; i < NUM_PRODUCERS; i++) { + pthread_join(producer[i], NULL); + } + pthread_join(consumer, NULL); + vmpsc_destroy(&g_queue, destroy_cb, NULL); + return 0; +} +``` + + + + +### References: + [Non-intrusive MPSC node-based queue](https://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue) + +--- +# Functions + +| Function | Description | +|---|---| +| [vmpsc_init](mpsc.h.md#function-vmpsc_init) | Initializes the given queue object `q`. | +| [vmpsc_foreach](mpsc.h.md#function-vmpsc_foreach) | Calls the given `f` function on each enqueued node's data. | +| [vmpsc_destroy](mpsc.h.md#function-vmpsc_destroy) | Calls the given `f` function on each enqueued node. | +| [vmpsc_enq](mpsc.h.md#function-vmpsc_enq) | Enqueues the given node into the given queue. | +| [vmpsc_deq](mpsc.h.md#function-vmpsc_deq) | Dequeues a node and data object from the queue. | +| [vmpsc_empty](mpsc.h.md#function-vmpsc_empty) | Checks if the current queue is empty or not. | +| [vmpsc_length](mpsc.h.md#function-vmpsc_length) | Calculates how many nodes are enqueued in the queue. | + +## Function `vmpsc_init` + +```c +static void vmpsc_init(vmpsc_t *q) +``` +_Initializes the given queue object_ `q`_._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. + + +> **Note:** call only before threads start accessing the queue. + + +## Function `vmpsc_foreach` + +```c +static void vmpsc_foreach(const vmpsc_t *q, vmpsc_data_visit_t f, void *args) +``` +_Calls the given_ `f` _function on each enqueued node's data._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. +- `f`: function pointer of type vmpsc_data_visit_t. +- `args`: extra arguments of `f`. + + +> **Note:** Don't call while producer threads are active. + + +## Function `vmpsc_destroy` + +```c +static void vmpsc_destroy(vmpsc_t *q, vmpsc_visit_t f, void *args) +``` +_Calls the given_ `f` _function on each enqueued node._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. +- `f`: function pointer of type vmpsc_visit_t. +- `args`: extra arguments of `f`. + + +> **Note:** call only after all threads are done accessing the queue. + +> **Note:** use the callback function to destruct nodes and data objects. + + +## Function `vmpsc_enq` + +```c +static void vmpsc_enq(vmpsc_t *q, vmpsc_node_t *node, void *data) +``` +_Enqueues the given node into the given queue._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. +- `node`: address of vmpsc_node_t object to enqueue. +- `data`: address of data object to be associated with the given node. + + + + +## Function `vmpsc_deq` + +```c +static void* vmpsc_deq(vmpsc_t *q, vmpsc_node_t **discarded_node) +``` +_Dequeues a node and data object from the queue._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. +- `discarded_node`: output parameter. Address of vmpsc_node_t sentinel object to destroy, or NULL. + + +**Returns:** void* address of dequeued data object. + +**Returns:** NULL when the queue is empty. + +> **Note:** on each successful dequeue there will be a sentinel node object kicked out of the queue. + + +## Function `vmpsc_empty` + +```c +static vbool_t vmpsc_empty(vmpsc_t *q) +``` +_Checks if the current queue is empty or not._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. + + +**Returns:** true the queue is empty. + +**Returns:** false the queue is not empty. + +> **Note:** to be called only by the consumer. + + +## Function `vmpsc_length` + +```c +static vsize_t vmpsc_length(vmpsc_t *q) +``` +_Calculates how many nodes are enqueued in the queue._ + + + + +**Parameters:** + +- `q`: address of vmpsc_t object. + + +**Returns:** vsize_t number of available elements in the queue. + +> **Note:** to be called only by the consumer. + +> **Note:** only non-sentinel nodes are counted. + + + +--- diff --git a/examples/eg_bbq_mpmc.c b/examples/eg_bbq_mpmc.c new file mode 100644 index 00000000..9ed3b32b --- /dev/null +++ b/examples/eg_bbq_mpmc.c @@ -0,0 +1,89 @@ +#include +#include +#include +#include + +#define BUFFER_ENTRY_NUM 4096U +#define NUM 10U +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL +#define NUM_WRITER 4U +#define NUM_READER 5U + +#define NON_BLOCKING false +#define BLOCKING true + +bbq_mpmc_t *g_bbq_mpmc = NULL; + +void * +writer(void *arg) +{ + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM * NUM_READER; + + while (rest) { + vuint32_t count = rest < ENQUEUE_BATCH ? rest : ENQUEUE_BATCH; + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ++ptr; + } + rest -= bbq_mpmc_enqueue(g_bbq_mpmc, buf, count, BLOCKING); + } + + (void)arg; + return NULL; +} + +void * +reader(void *arg) +{ + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t rest = NUM * NUM_WRITER; + + while (rest) { + vuint32_t batch_size = rest < DEQUEUE_BATCH ? rest : DEQUEUE_BATCH; + vuint32_t count = + bbq_mpmc_dequeue(g_bbq_mpmc, buf, batch_size, NON_BLOCKING); + + for (vuint32_t i = 0; i < count; i++) { + ASSERT(buf[i] > 0); + printf("dequeue item %lu\n", buf[i]); + } + rest -= count; + } + (void)arg; + return NULL; +} + +int +main(void) +{ + pthread_t t_writer[NUM_WRITER]; + pthread_t t_reader[NUM_READER]; + + // allocate + vsize_t sz = bbq_mpmc_memsize(BUFFER_ENTRY_NUM); + g_bbq_mpmc = (bbq_mpmc_t *)malloc(sz); + ASSERT(g_bbq_mpmc); + + // init + vbool_t success = bbq_mpmc_init(g_bbq_mpmc, sz); + ASSERT(success); + + for (vsize_t i = 0; i < NUM_WRITER; i++) { + pthread_create(&t_writer[i], NULL, writer, NULL); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_create(&t_reader[i], NULL, reader, NULL); + } + for (vsize_t i = 0; i < NUM_WRITER; i++) { + pthread_join(t_writer[i], NULL); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_join(t_reader[i], NULL); + } + + // deallocate + free(g_bbq_mpmc); + return 0; +} diff --git a/examples/eg_bbq_spsc.c b/examples/eg_bbq_spsc.c new file mode 100644 index 00000000..f27077b9 --- /dev/null +++ b/examples/eg_bbq_spsc.c @@ -0,0 +1,79 @@ +#include +#include +#include +#include + +#define BUFFER_ENTRY_NUM 4096U +#define NUM 10U +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL + +#define NON_BLOCKING false +#define BLOCKING true + +bbq_spsc_t *g_bbq_spsc = NULL; + +void * +writer(void *arg) +{ + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + + while (rest) { + vuint32_t count = rest < ENQUEUE_BATCH ? rest : ENQUEUE_BATCH; + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ptr++; + } + rest -= bbq_spsc_enqueue(g_bbq_spsc, buf, count, BLOCKING); + } + + (void)arg; + return NULL; +} + +void * +reader(void *arg) +{ + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + + while (rest) { + vuint32_t count = + bbq_spsc_dequeue(g_bbq_spsc, buf, DEQUEUE_BATCH, NON_BLOCKING); + + for (vuint32_t i = 0; i < count; i++) { + ASSERT(buf[i] == ptr++); + printf("dequeue item %lu\n", buf[i]); + } + rest -= count; + } + (void)arg; + return NULL; +} + +int +main(void) +{ + pthread_t t_writer; + pthread_t t_reader; + + // allocate + vsize_t sz = bbq_spsc_memsize(BUFFER_ENTRY_NUM); + g_bbq_spsc = (bbq_spsc_t *)malloc(sz); + ASSERT(g_bbq_spsc); + + // init + vbool_t success = bbq_spsc_init(g_bbq_spsc, sz); + ASSERT(success); + + pthread_create(&t_writer, NULL, writer, NULL); + pthread_create(&t_reader, NULL, reader, NULL); + pthread_join(t_reader, NULL); + pthread_join(t_writer, NULL); + + // deallocate + free(g_bbq_spsc); + return 0; +} diff --git a/examples/eg_bbq_spsc_m_proc.c b/examples/eg_bbq_spsc_m_proc.c new file mode 100644 index 00000000..7c6a666c --- /dev/null +++ b/examples/eg_bbq_spsc_m_proc.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define QUEUE_SIZE 4095U +#define EPOCH_SIZE 100U +#define NUM_ENTRIES 1U +#define SLEEP_MICRO_SEC 10 +#define NON_BLOCKING false +#define BLOCKING true + +#include +#include +#include + +bbq_spsc_t *g_bbq; + +void * +create_shared_memory(vsize_t size) +{ + int protection = PROT_READ | PROT_WRITE; + int visibility = MAP_SHARED | MAP_ANONYMOUS; + return mmap(NULL, size, protection, visibility, -1, 0); +} + +void +check_space(void) +{ + vuint64_t data = 0; + vuint64_t count = 0; + + while (bbq_spsc_dequeue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + + while (bbq_spsc_enqueue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) { + count++; + } + while (bbq_spsc_dequeue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + + printf( + "Current size of the queue is %ld after %d crashes, shouldn't be " + "zero.\n", + count, EPOCH_SIZE); + assert(count != 0); +} + +void * +writer(void *arg) +{ + V_UNUSED(arg); + vuint64_t data = 0; + while (true) { + data = (data + 1) % (QUEUE_SIZE - 1); + while (!bbq_spsc_enqueue(g_bbq, &data, NUM_ENTRIES, NON_BLOCKING)) {} + usleep(SLEEP_MICRO_SEC); + } +} + +void * +reader(void *arg) +{ + V_UNUSED(arg); + vuint64_t data; + while (true) { + while (!bbq_spsc_dequeue(g_bbq, &data, 1, NON_BLOCKING)) {} + usleep(SLEEP_MICRO_SEC); + assert(data < QUEUE_SIZE - 1); + } +} + +int +main(void) +{ + srand(time(NULL)); + vuint64_t sz = bbq_spsc_memsize(QUEUE_SIZE); + g_bbq = create_shared_memory(sz); + assert(g_bbq); + vbool_t success = bbq_spsc_init(g_bbq, sz); + ASSERT(success && "BBQ init failed"); + for (vuint32_t i = 0; i < EPOCH_SIZE; i++) { + pid_t pid = fork(); + assert(pid >= 0); + if (pid != 0) { + usleep(rand() % 10000); + while (kill(pid, SIGTERM) < 0) {} + } else { + pthread_t t_writer, t_reader; + pthread_create(&t_writer, NULL, writer, NULL); + pthread_create(&t_reader, NULL, reader, NULL); + pthread_join(t_writer, NULL); + pthread_join(t_reader, NULL); + return 0; + } + } + check_space(); + return 0; +} diff --git a/examples/eg_mpsc.c b/examples/eg_mpsc.c new file mode 100644 index 00000000..89b40336 --- /dev/null +++ b/examples/eg_mpsc.c @@ -0,0 +1,78 @@ +#include +#include +#include +#include + +vmpsc_t g_queue; + +#define NUM_PRODUCERS 3U +#define NUM_PROD_ITEMS 10U +#define NUM_CONS_ITEMS (NUM_PRODUCERS * NUM_PROD_ITEMS) + +typedef struct data_s { + vuint32_t x; + vuint32_t y; +} data_t; + +void +destroy_cb(vmpsc_node_t *node, void *args) +{ + free(node->data); + free(node); + V_UNUSED(args); +} + +void * +consume(void *args) +{ + vmpsc_node_t *n = NULL; + data_t *d = NULL; + for (vuint32_t i = 0; i < NUM_CONS_ITEMS; i++) { + while (d = vmpsc_deq(&g_queue, &n), d == NULL) { + /* busy wait till an item is consumed. Repeat on empty! */ + } + ASSERT(d); + ASSERT(d->x == d->y); + printf("consumed item: {%u, %u}\n", d->x, d->y); + free(d); + /* n can be NULL even if d is not. */ + free(n); + } + V_UNUSED(args); + return NULL; +} + +void * +produce(void *args) +{ + vuint32_t tid = (vuint32_t)(vuintptr_t)args; + for (vuint32_t i = 0; i < NUM_PROD_ITEMS; i++) { + data_t *d = malloc(sizeof(*d)); + ASSERT(d); + d->x = d->y = (tid * NUM_PRODUCERS) + i; + vmpsc_node_t *n = malloc(sizeof(*n)); + ASSERT(n); + vmpsc_enq(&g_queue, n, d); + } + V_UNUSED(args); + return NULL; +} + +int +main(void) +{ + vmpsc_init(&g_queue); + pthread_t consumer; + pthread_t producer[NUM_PRODUCERS]; + + pthread_create(&consumer, NULL, consume, NULL); + for (vsize_t i = 0; i < NUM_PRODUCERS; i++) { + pthread_create(&producer[i], NULL, produce, (void *)(vuintptr_t)i); + } + for (vsize_t i = 0; i < NUM_PRODUCERS; i++) { + pthread_join(producer[i], NULL); + } + pthread_join(consumer, NULL); + vmpsc_destroy(&g_queue, destroy_cb, NULL); + return 0; +} diff --git a/include/vsync/doc.h b/include/vsync/doc.h index daf5ea72..613a0b32 100644 --- a/include/vsync/doc.h +++ b/include/vsync/doc.h @@ -1,9 +1,8 @@ /* - * Copyright (C) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + * Copyright (C) Huawei Technologies Co., Ltd. 2025-2026. All rights reserved. * SPDX-License-Identifier: MIT - * Description: VSync API documention - * Author: Huawei Dresden Research Center */ + #ifndef VSYNC_DOC_H #define VSYNC_DOC_H /******************************************************************************* diff --git a/include/vsync/pool/cached_pool.h b/include/vsync/pool/cached_pool.h index 7e284ca6..e5967279 100644 --- a/include/vsync/pool/cached_pool.h +++ b/include/vsync/pool/cached_pool.h @@ -1,5 +1,5 @@ /* - * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * Copyright (C) Huawei Technologies Co., Ltd. 2025-2026. All rights reserved. * SPDX-License-Identifier: MIT */ @@ -193,7 +193,7 @@ _cached_pool_vunit_find(cached_pool_t *a, vuint32_t id) /** * Calculate the needed memory space for creating a pool * - * @param thread_num maxinum thread number + * @param thread_num maximum thread number * @param entry_num minimal number of entires * @param entry_size size of each entry * @@ -211,7 +211,7 @@ _cached_pool_vunit_find(cached_pool_t *a, vuint32_t id) * Make sure the buffer has enough size (calculated by cached_pool_memsize) * * @param buf pointer to the buffer - * @param thread_num maxinum thread number + * @param thread_num maximum thread number * @param entry_num minimal number of entires * @param entry_size size of each entry * diff --git a/include/vsync/queue/bbq_mpmc.h b/include/vsync/queue/bbq_mpmc.h new file mode 100644 index 00000000..4253bf6b --- /dev/null +++ b/include/vsync/queue/bbq_mpmc.h @@ -0,0 +1,365 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_MPMC_H +#define VSYNC_BBQ_MPMC_H +/******************************************************************************* + * @file bbq_mpmc.h + * @brief Block-based Bounded Queue multi-producer/multi-consumer + * + * A highly performant bounded queue that splits the buffer in multiple blocks. + * + * ### Remarks: + * + * In this implementation, values have the fixed size (pointer size). + * This implementation does not support `DROP_OLD` mode as described in the + * original paper. + * + * @cite [BBQ: A Block-based Bounded Queue for Exchanging Data and + * Profiling](https://www.usenix.org/conference/atc22/presentation/wang-jiawei) + * + * @example + * @include eg_bbq_mpmc.c + * + ******************************************************************************/ + +#include +#include +#include +#include +#include + +/** + * @def BBQ_BLOCK_NUM_LOG + * + * Define this macro with `-DBBQ_BLOCK_NUM_LOG=N` + * to define the total number of blocks equals to `2^N` + * + * @note default value is `3U` -> (`8` blocks) + * + */ +#ifndef BBQ_BLOCK_NUM_LOG + #define BBQ_BLOCK_NUM_LOG 3U +#endif + +/** + * @def BBQ_ENTRY_SIZE_LOG + * + * Define this macro with `-BBQ_ENTRY_SIZE_LOG=N` to + * define an entry size equals to `2^N` + * + * @note default value is `log2(sizeof(vuintptr_t))` + * + */ +#ifndef BBQ_ENTRY_SIZE_LOG + #define BBQ_ENTRY_SIZE_LOG v_log2(sizeof(vuintptr_t)) +#endif + +#include + +typedef struct bbq_mpmc_block_s { + vatomic64_t allocated VSYNC_CACHEALIGN; + vatomic64_t committed VSYNC_CACHEALIGN; + vatomic64_t reserved VSYNC_CACHEALIGN; + vatomic64_t consumed VSYNC_CACHEALIGN; + vuint8_t entry[] VSYNC_CACHEALIGN; +} bbq_mpmc_block_t; + +typedef struct bbq_mpmc_s { + bbq_config_t config VSYNC_CACHEALIGN; + vatomic64_t widx VSYNC_CACHEALIGN; + vatomic64_t ridx VSYNC_CACHEALIGN; + vuint8_t blk[] VSYNC_CACHEALIGN; +} bbq_mpmc_t; + +/* block cursor init value */ +#define BBQ_BLOCK_MPMC_INIT_VALUE BBQ_BLOCK_INIT_VALUE(bbq_mpmc_block_t) + +/* Note:The following macros are used inside bbq/common.h in BBQ_COUNT + * definition */ +#define BBQ_MPMC_WRITE_PROD(k, v) (vatomic64_write(&(k), v)) +#define BBQ_MPMC_WRITE_CONS(k, v) (vatomic64_write(&(k), v)) +#define BBQ_MPMC_READ_PROD(k) (vatomic64_read(&(k))) +#define BBQ_MPMC_READ_CONS(k) (vatomic64_read(&(k))) +#define BBQ_MPMC_COUNT(q) BBQ_COUNT(q, mpmc, MPMC) + +static inline vbool_t _bbq_mpmc_enqueue(struct bbq_mpmc_s *q, vuintptr_t **buf, + vuint32_t *count); +static inline vbool_t _bbq_mpmc_dequeue(bbq_mpmc_t *q, vuintptr_t **buf, + vuint32_t *count); +static inline void _bbq_mpmc_block_init(bbq_mpmc_block_t *blk, vsize_t idx, + vuint16_t block_size); + +/** + * Enqueues one or more entries. + * + * Multiple entries can be enqueued if `src` points to an array. Use `count` to + * indicate how many entries should be enqueued, starting from `src`. + * + * @param q address of `bbq_mpmc_t` object. + * @param src pointer to first entry. + * @param count number of entries to enqueue. + * @param wait should wait for space to be available. + * + * @return number of enqueued entries. + */ +static inline vuint32_t +bbq_mpmc_enqueue(bbq_mpmc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +{ + vuint32_t rest = count; + vuintptr_t *rest_buf = buf; + vbool_t retry; + + /* the following is equivalent to + * while(bbq_mpmc_enqueue_internal(q, &rest_buf, &rest) || (wait && rest)); + */ + do { + retry = _bbq_mpmc_enqueue(q, &rest_buf, &rest); + + /* Help model checker in case the queue is empty + * The condition to leave the loop is + * retry' == true or rest' == 0 + * rest' == 0 => rest # 0 /\ retry' = true + * + * It is sufficient to observe retry only + * + */ + await_while (!retry && wait && rest) + retry = _bbq_mpmc_enqueue(q, &rest_buf, &rest); + } while (retry); + + return count - rest; +} + +/** + * Dequeues one or more entries. + * + * Multiple entries can be dequeued if `src` points to an array. Use `count` to + * indicate how many entries should be dequeued. + * + * @param q address of `bbq_mpmc_t` object. + * @param src pointer to preallocated memory for the first entry. + * @param count number of entries to dequeue. + * @param wait should wait for entries to be available. + * + * @return number of dequeued entries. + */ +static inline vuint32_t +bbq_mpmc_dequeue(bbq_mpmc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +{ + vuint32_t rest = count; + vuintptr_t *rest_buf = buf; + vbool_t retry; + + /* the following is equivalent to + * while(bbq_mpmc_dequeue_internal(q, &rest_buf, &rest) || (wait && rest)); + */ + do { + retry = _bbq_mpmc_dequeue(q, &rest_buf, &rest); + + /* Help model checker in case the queue is empty + * The condition to leave the loop is + * retry' == true or rest' == 0 + * rest' == 0 => rest # 0 /\ retry' = true + * + * It is sufficient to observe retry only + * + */ + await_while (!retry && wait && rest) + retry = _bbq_mpmc_dequeue(q, &rest_buf, &rest); + } while (retry); + + return count - rest; +} + +/** + * Calculates the size of the bbq queue. + * + * @param capacity maximum number of entries that can fit in the queue. + * @return size to be allocated in bytes. + */ +static inline vsize_t +bbq_mpmc_memsize(vsize_t capacity) +{ + vsize_t cnt_each_blk = (capacity) >> BBQ_BLOCK_NUM_LOG; + if (cnt_each_blk == 0) { + cnt_each_blk = 1; + } + vsize_t mem_each_blk = + sizeof(bbq_mpmc_block_t) + (cnt_each_blk << BBQ_ENTRY_SIZE_LOG); + vsize_t mem_each_blk_log = + v_pow2_round_down(mem_each_blk * 2 - 1); /* align up */ + vsize_t mem_buf = + sizeof(bbq_mpmc_t) + (mem_each_blk_log << BBQ_BLOCK_NUM_LOG); + return mem_buf; +} +/** + * Initializes a bbq data structure. + * + * @param q pointer to bbq data structure. + * @param size number of bytes allocated for bbq data structure. + * @return true initialization succeeded. + * @return false initialization failed. + */ +static inline vbool_t +bbq_mpmc_init(bbq_mpmc_t *q, vsize_t size) +{ + if (unlikely(q == NULL) || unlikely(BBQ_ENTRY_SIZE < BBQ_MIN_ENTRY_SIZE) || + unlikely(BBQ_ENTRY_SIZE > BBQ_MAX_ENTRY_SIZE) || + unlikely(BBQ_BLOCK_NUM_LOG < BBQ_MIN_BLOCK_NUM_LOG) || + unlikely(BBQ_BLOCK_NUM_LOG > BBQ_MAX_BLOCK_NUM_LOG)) { + return false; + } + + vsize_t blks_total_size = (size) - sizeof(bbq_mpmc_t); + vsize_t blk_size = v_pow2_round_down(blks_total_size >> BBQ_BLOCK_NUM_LOG); + if (unlikely(blk_size <= BBQ_BLOCK_INIT_VALUE(bbq_mpmc_block_t))) { + return false; + } + vsize_t blk_size_log = v_log2(blk_size); + BBQ_BLK_SZ_VERIFICATION(mpmc); + if (unlikely(blk_size_log < BBQ_MIN_BLOCK_SIZE_LOG) || + unlikely(blk_size_log >= BBQ_MAX_BLOCK_SIZE_LOG)) { + return false; + } + (q)->config.blk_size = blk_size; + (q)->config.blk_size_log = blk_size_log; + BBQ_MPMC_WRITE_PROD((q)->widx, 0); + BBQ_MPMC_WRITE_CONS((q)->ridx, 0); + for (vsize_t i = 0; i < (1UL << BBQ_BLOCK_NUM_LOG); i++) { + _bbq_mpmc_block_init( + (bbq_mpmc_block_t *)((q)->blk + (i << blk_size_log)), i, blk_size); + } + return true; +} + +static inline void +_bbq_mpmc_block_init(bbq_mpmc_block_t *blk, vsize_t idx, vuint16_t block_size) +{ + /* if it is not the first block, set to invalid state */ + vuint16_t init_value = likely(idx) ? block_size : BBQ_BLOCK_MPMC_INIT_VALUE; + vatomic64_write(&blk->allocated, init_value); + vatomic64_write(&blk->committed, init_value); + vatomic64_write(&blk->reserved, init_value); + vatomic64_write(&blk->consumed, init_value); +} + +/* return means retry */ +static inline vbool_t +_bbq_mpmc_enqueue(bbq_mpmc_t *q, vuintptr_t **buf, vuint32_t *count) +{ + if (*count == 0) { + return false; + } + /* get the address of the alloc block */ + vuint64_t widx = vatomic64_read(&q->widx); + vuint16_t block_idx = BBQ_GLOBAL_IDX(widx); + bbq_mpmc_block_t *blk = BBQ_GET_BLOCK(q, block_idx); + /* precheck once */ + vuint16_t block_size = q->config.blk_size; + vuint64_t allocated = vatomic64_read(&blk->allocated); + vuint64_t allocated_space = BBQ_LOCAL_IDX(allocated); + vsize_t entry_total_size = (*count) << BBQ_ENTRY_SIZE_LOG; + /* if out of bound, we don't add the space, but help to move the block */ + if (likely(allocated_space < block_size)) { + /* update the allocated index using FAA */ + vuint64_t old_allocated = + vatomic64_get_add(&blk->allocated, entry_total_size); + /* we have some space */ + vuint64_t old_local_space = BBQ_LOCAL_IDX(old_allocated); + if (likely(old_local_space < block_size)) { + vuint16_t space = + VMIN(entry_total_size, block_size - old_local_space); + void *entry = BBQ_GET_ENTRY(blk, old_local_space); + int r = memcpy_s(entry, space, *buf, space); + BUG_ON(r != 0); + vatomic64_add(&blk->committed, space); + vuint16_t offset = space >> BBQ_ENTRY_SIZE_LOG; + *buf += offset; + *count -= offset; + return true; + } + } + /* slow path, all writers help to move to next block */ + bbq_mpmc_block_t *nblk = BBQ_GET_NEXT_BLOCK(q, block_idx); + vuint64_t global_vsn = BBQ_GLOBAL_VSN(widx); + if (unlikely( + !BBQ_BLOCK_FULLY_CONSUMED_WITH_VSN(nblk, block_size, global_vsn))) { + return false; + } + /* reset cursor and advance block */ + bbq_reset_block_cursor_heavy(&nblk->committed, global_vsn + 1, + BBQ_BLOCK_MPMC_INIT_VALUE); + bbq_reset_block_cursor_heavy(&nblk->allocated, global_vsn + 1, + BBQ_BLOCK_MPMC_INIT_VALUE); + BBQ_ADVANCE_HEAD(&q->widx, widx, widx + 1); + return true; +} +/* return means retry */ +static inline vbool_t +_bbq_mpmc_dequeue(bbq_mpmc_t *q, vuintptr_t **buf, vuint32_t *count) +{ + if (*count == 0) { + return false; + } + /* get the address of the occupy block */ + vuint64_t ridx = vatomic64_read(&q->ridx); + vuint16_t block_idx = BBQ_GLOBAL_IDX(ridx); + bbq_mpmc_block_t *blk = BBQ_GET_BLOCK(q, block_idx); + /* check if the block is fully reserved */ + vuint16_t block_size = q->config.blk_size; + vuint64_t reserved = vatomic64_read(&blk->reserved); + vuint64_t reserved_space = BBQ_LOCAL_IDX(reserved); + if (likely(reserved_space < block_size)) { + vuint64_t committed = vatomic64_read(&blk->committed); + /* check if we have an entry to occupy */ + vuint64_t committed_space = BBQ_LOCAL_IDX(committed); + if (unlikely(reserved_space >= committed_space)) { + /* Note: the version is strictly monotic and may not wrap. */ + ASSERT(reserved <= committed && "reserved must be <= committed"); + return false; + } + vuint16_t entry_total_size = VMIN((*count) << BBQ_ENTRY_SIZE_LOG, + committed_space - reserved_space); + if (unlikely(committed_space != block_size)) { + vuint64_t allocated = vatomic64_read(&blk->allocated); + vuint64_t allocated_space = BBQ_LOCAL_IDX(allocated); + if (likely(allocated_space != committed_space)) { + return false; + } + } + if (vatomic64_cmpxchg(&blk->reserved, reserved, + reserved + entry_total_size) != reserved) { + return true; + } + /* we got the entry */ + void *entry = BBQ_GET_ENTRY(blk, BBQ_LOCAL_IDX(reserved)); + int r = memcpy_s(*buf, entry_total_size, entry, entry_total_size); + BUG_ON(r != 0); + /* consume after copy the data back */ + vatomic64_add(&blk->consumed, entry_total_size); + vuint16_t offset = entry_total_size >> BBQ_ENTRY_SIZE_LOG; + *buf += offset; + *count -= offset; + return true; + } + /* need to advance the block */ + bbq_mpmc_block_t *nblk = BBQ_GET_NEXT_BLOCK(q, block_idx); + /* r_head never pass the w_head and r_tail */ + vuint64_t next_consumer_vsn = BBQ_LOCAL_VSN(reserved) - (block_idx != 0); + vuint64_t next_producer_vsn = + BBQ_LOCAL_VSN(vatomic64_read(&nblk->committed)); + if (next_producer_vsn != next_consumer_vsn + 1) { + return false; + } + /* reset the cursor */ + bbq_reset_block_cursor_heavy(&nblk->consumed, next_consumer_vsn + 1, + BBQ_BLOCK_MPMC_INIT_VALUE); + bbq_reset_block_cursor_heavy(&nblk->reserved, next_consumer_vsn + 1, + BBQ_BLOCK_MPMC_INIT_VALUE); + BBQ_ADVANCE_HEAD(&q->ridx, ridx, ridx + 1); + return true; +} +#endif diff --git a/include/vsync/queue/bbq_spsc.h b/include/vsync/queue/bbq_spsc.h new file mode 100644 index 00000000..5b5bd236 --- /dev/null +++ b/include/vsync/queue/bbq_spsc.h @@ -0,0 +1,378 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_SPSC_H +#define VSYNC_BBQ_SPSC_H + +/******************************************************************************* + * @file bbq_spsc.h + * @brief Block-based Bounded Queue single-producer/single-consumer. + * @ingroup linearizable lock_free + * + * A highly performant bounded queue that splits the buffer in multiple blocks. + * + * + * ### Remarks: + * + * In this implementations, values have a fixed size equal to pointer size. + * This implementation does not support `DROP_OLD` mode as described in the + * original paper. + * + * @cite [BBQ: A Block-based Bounded Queue for Exchanging Data and + * Profiling](https://www.usenix.org/conference/atc22/presentation/wang-jiawei) + * + * @example + * ### Multi-threaded + * @include eg_bbq_spsc.c + * ### Multi-process + * @include eg_bbq_spsc_m_proc.c + * + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +/** + * @def BBQ_BLOCK_NUM_LOG + * + * Define this macro with `-DBBQ_BLOCK_NUM_LOG=N` + * to define the total number of blocks equals to `2^N` + * + * @note default value is `3U` -> (`8` blocks) + * + */ +#ifndef BBQ_BLOCK_NUM_LOG + #define BBQ_BLOCK_NUM_LOG 3U +#endif + +/** + * @def BBQ_ENTRY_SIZE_LOG + * + * Define this macro with `-BBQ_ENTRY_SIZE_LOG=N` to + * define an entry size equals to `2^N` + * + * @note default value is `log2(sizeof(vuintptr_t))` + * + */ +#ifndef BBQ_ENTRY_SIZE_LOG + #define BBQ_ENTRY_SIZE_LOG v_log2(sizeof(vuintptr_t)) +#endif + +#include + +typedef struct bbq_spsc_block_s { + vatomic64_t committed VSYNC_CACHEALIGN; + vatomic64_t consumed VSYNC_CACHEALIGN; + vuint8_t entry[] VSYNC_CACHEALIGN; +} bbq_spsc_block_t; + +typedef struct bbq_spsc_s { + bbq_config_t config VSYNC_CACHEALIGN; + vuint64_t widx VSYNC_CACHEALIGN; + vuint64_t ridx VSYNC_CACHEALIGN; + vuint8_t blk[] VSYNC_CACHEALIGN; +} bbq_spsc_t; + +/* block cursor init value */ +#define BBQ_BLOCK_SPSC_INIT_VALUE BBQ_BLOCK_INIT_VALUE(struct bbq_spsc_block_s) +/* Note:The following macros are used inside bbq/common.h in BBQ_COUNT + * definition */ +#define BBQ_SPSC_WRITE_PROD(k, v) ((k) = v) +#define BBQ_SPSC_WRITE_CONS(k, v) ((k) = v) +#define BBQ_SPSC_READ_PROD(k) (k) +#define BBQ_SPSC_READ_CONS(k) (k) +#define BBQ_SPSC_COUNT(q) BBQ_COUNT(q, spsc, SPSC) + +/* prototypes of internal functions */ +static inline vbool_t _bbq_spsc_enqueue(bbq_spsc_t *q, vuintptr_t **buf, + vuint32_t *count); +static inline vbool_t _bbq_spsc_dequeue(bbq_spsc_t *q, vuintptr_t **buf, + vuint32_t *count); +static inline void _bbq_spsc_block_init(bbq_spsc_block_t *blk, vsize_t idx, + vuint16_t block_size); +/** + * Enqueues one or more entries. + * + * Multiple entries can be enqueued if `src` points to an array. Use `count` to + * indicate how many entries should be enqueued, starting from `src`. + * + * @param q address of bbq_spsc_t object. + * @param buf address of the first entry. + * @param count number of entries to enqueue. + * @param wait true/false when set to true it waits (blocks) till space becomes + * available. Otherwise, it quits retrying. + * + * @return number of enqueued entries. + */ +static inline vuint32_t +bbq_spsc_enqueue(bbq_spsc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +{ + vuint32_t rest = count; + vuintptr_t *rest_buf = buf; + vbool_t retry; + + /* The following two are equivalent. + * Which one is better depends on the model checker. + */ +#if defined(VSYNC_VERIFICATION_DAT3M) + while (_bbq_spsc_enqueue(q, &rest_buf, &rest) || (wait && rest)) + ; +#else + do { + retry = _bbq_spsc_enqueue(q, &rest_buf, &rest); + + /* The condition to leave the loop is + * retry' == true or rest' == 0 + * rest' == 0 => rest # 0 /\ retry' = true + * + * It is sufficient to observe retry only + * + */ + await_while (!retry && wait && rest) { + retry = _bbq_spsc_enqueue(q, &rest_buf, &rest); + } + } while (retry); +#endif + + return count - rest; +} + +/** + * Dequeues one or more entries. + * + * Multiple entries can be dequeued if `buf` points to an array. Use `count` to + * indicate how many entries should be dequeued. + * + * @param q address of bbq_spsc_t object. + * @param buf address of the first entry of the preallocated memory. + * @param count number of entries to dequeue. + * @param wait true/false. When set to true the API waits/blocks for entries to + * be available + * + * @return number of dequeued entries. + */ +static inline vuint32_t +bbq_spsc_dequeue(bbq_spsc_t *q, vuintptr_t *buf, vuint32_t count, vbool_t wait) +{ + vuint32_t rest = count; + vuintptr_t *rest_buf = buf; + vbool_t retry; + + /* The following two are equivalent. + * Which one is better depends on the model checker. + */ +#if defined(VSYNC_VERIFICATION_DAT3M) + while (_bbq_spsc_dequeue(q, &rest_buf, &rest) || (wait && rest)) + ; +#else + do { + retry = _bbq_spsc_dequeue(q, &rest_buf, &rest); + + /* The condition to leave the loop is + * retry' == true or rest' == 0 + * rest' == 0 => rest # 0 /\ retry' = true + * + * It is sufficient to observe retry only + * + */ + await_while (!retry && wait && rest) { + retry = _bbq_spsc_dequeue(q, &rest_buf, &rest); + } + } while (retry); +#endif + + return count - rest; +} +/** + * Calculates the size of bbq_spsc_t object based on the given capacity. + * + * @param capacity maximum number of entries that can fit in the queue. + * @return size of bbq_spsc_t object with the given capacity. + */ +static inline vsize_t +bbq_spsc_memsize(vsize_t capacity) +{ + vsize_t cnt_each_blk = (capacity) >> BBQ_BLOCK_NUM_LOG; + if (cnt_each_blk == 0) { + cnt_each_blk = 1; + } + vsize_t mem_each_blk = + sizeof(bbq_spsc_block_t) + (cnt_each_blk << BBQ_ENTRY_SIZE_LOG); + + vsize_t mem_each_blk_log = + v_pow2_round_down(mem_each_blk * 2 - 1); /* align up */ + + vsize_t mem_buf = + sizeof(bbq_spsc_t) + (mem_each_blk_log << BBQ_BLOCK_NUM_LOG); + + return mem_buf; +} + +/** + * Initializes a bbq data structure. + * + * @param q address of bbq_spsc_t object. + * @param size size of the given bbq_spsc_t object `q`. + * + * @return true initialization succeeded. + * @return false initialization failed. + */ +static inline vbool_t +bbq_spsc_init(bbq_spsc_t *q, vsize_t size) +{ + // we shift vuint16_t by BBQ_ENTRY_SIZE_LOG, we need to make sure the + // behavior is defined + ASSERT( + BBQ_ENTRY_SIZE_LOG < 16U && + "must have width less than vuint16_t because to be able to shift it"); + + if (unlikely(q == NULL) || unlikely(BBQ_ENTRY_SIZE < BBQ_MIN_ENTRY_SIZE) || + unlikely(BBQ_ENTRY_SIZE > BBQ_MAX_ENTRY_SIZE) || + unlikely(BBQ_BLOCK_NUM_LOG < BBQ_MIN_BLOCK_NUM_LOG) || + unlikely(BBQ_BLOCK_NUM_LOG > BBQ_MAX_BLOCK_NUM_LOG)) { + return false; + } + + vsize_t blks_total_size = (size) - sizeof(bbq_spsc_t); + vsize_t blk_size = v_pow2_round_down(blks_total_size >> BBQ_BLOCK_NUM_LOG); + if (unlikely(blk_size <= BBQ_BLOCK_INIT_VALUE(struct bbq_spsc_block_s))) { + return false; + } + vsize_t blk_size_log = v_log2(blk_size); + BBQ_BLK_SZ_VERIFICATION(spsc); + if (unlikely(blk_size_log < BBQ_MIN_BLOCK_SIZE_LOG) || + unlikely(blk_size_log >= BBQ_MAX_BLOCK_SIZE_LOG)) { + return false; + } + (q)->config.blk_size = blk_size; + (q)->config.blk_size_log = blk_size_log; + BBQ_SPSC_WRITE_PROD((q)->widx, 0); + BBQ_SPSC_WRITE_CONS((q)->ridx, 0); + + for (vsize_t i = 0; i < (1UL << BBQ_BLOCK_NUM_LOG); i++) { + _bbq_spsc_block_init( + (bbq_spsc_block_t *)((q)->blk + (i << blk_size_log)), i, blk_size); + } + return true; +} + +static inline void +_bbq_spsc_block_init(bbq_spsc_block_t *blk, vsize_t idx, vuint16_t block_size) +{ + /* if it is not the first block, set to invalid state */ + vuint16_t init_value = likely(idx) ? block_size : BBQ_BLOCK_SPSC_INIT_VALUE; + vatomic64_write(&blk->committed, init_value); + vatomic64_write(&blk->consumed, init_value); +} + +/* return means retry */ +static inline vbool_t +_bbq_spsc_enqueue(bbq_spsc_t *q, vuintptr_t **buf, vuint32_t *count) +{ + if (*count == 0) { + return false; + } + + /* get the address of the alloc block */ + vuint64_t widx = q->widx; + vuint16_t block_idx = BBQ_GLOBAL_IDX(widx); + bbq_spsc_block_t *blk = BBQ_GET_BLOCK(q, block_idx); + /* precheck once */ + vuint16_t block_size = q->config.blk_size; + vuint64_t committed = vatomic64_read_rlx(&blk->committed); + vuint64_t committed_space = BBQ_LOCAL_IDX(committed); + vsize_t entry_total_size = (*count) << BBQ_ENTRY_SIZE_LOG; + /* if out of bound, we don't add the space, but help to move the block */ + if (likely(committed_space < block_size)) { + vuint16_t space = VMIN(entry_total_size, block_size - committed_space); + void *entry = BBQ_GET_ENTRY(blk, committed_space); + int r = memcpy_s(entry, space, *buf, space); + BUG_ON(r != 0); + vuint64_t new_committed = BBQ_LOCAL_COMPOSE(BBQ_LOCAL_VSN(committed), + committed_space + space); + vatomic64_write_rel(&blk->committed, new_committed); + vuint16_t offset = space >> BBQ_ENTRY_SIZE_LOG; + *buf += offset; + *count -= offset; + return true; + } + + /* slow path, all writers help to move to next block */ + bbq_spsc_block_t *nblk = BBQ_GET_NEXT_BLOCK(q, block_idx); + vuint64_t global_vsn = BBQ_GLOBAL_VSN(widx); + if (unlikely( + !BBQ_BLOCK_FULLY_CONSUMED_WITH_VSN(nblk, block_size, global_vsn))) { + return false; + } + + /* reset cursor and advance block */ + BBQ_RESET_BLOCK_CURSOR_LIGHT(&nblk->committed, global_vsn + 1, + BBQ_BLOCK_SPSC_INIT_VALUE); + q->widx++; + return true; +} + +/* return means retry */ +static inline vbool_t +_bbq_spsc_dequeue(bbq_spsc_t *q, vuintptr_t **buf, vuint32_t *count) +{ + if (*count == 0) { + return false; + } + + /* get the address of the occupy block */ + vuint64_t ridx = q->ridx; + vuint16_t block_idx = BBQ_GLOBAL_IDX(ridx); + bbq_spsc_block_t *blk = BBQ_GET_BLOCK(q, block_idx); + + /* check if the block is fully reserved */ + vuint16_t block_size = q->config.blk_size; + vuint64_t consumed = vatomic64_read_rlx(&blk->consumed); + vuint64_t consumed_space = BBQ_LOCAL_IDX(consumed); + if (likely(consumed_space < block_size)) { + vuint64_t committed = vatomic64_read_acq(&blk->committed); + /* check if we have an entry to occupy */ + vuint64_t committed_space = BBQ_LOCAL_IDX(committed); + if (unlikely(consumed_space >= committed_space)) { + ASSERT(consumed_space == committed_space && + "Consumed should be <= committed"); + return false; + } + + /* we got the entry */ + vuint16_t space = VMIN((*count) << BBQ_ENTRY_SIZE_LOG, + committed_space - consumed_space); + void *entry = BBQ_GET_ENTRY(blk, consumed_space); + int r = memcpy_s(*buf, space, entry, space); + BUG_ON(r != 0); + vuint64_t new_consumed = consumed + space; + vatomic64_write_rel(&blk->consumed, new_consumed); + vuint16_t offset = space >> BBQ_ENTRY_SIZE_LOG; + *buf += offset; + *count -= offset; + return true; + } + + /* need to advance the block */ + bbq_spsc_block_t *nblk = BBQ_GET_NEXT_BLOCK(q, block_idx); + vuint64_t global_vsn = BBQ_GLOBAL_VSN(ridx); + + /* r_head never pass the w_head and r_tail */ + vuint64_t next_block_vsn = + BBQ_LOCAL_VSN(vatomic64_read_rlx(&nblk->committed)); + if (unlikely(next_block_vsn != global_vsn + 1)) { + return false; + } + /* reset the cursor */ + BBQ_RESET_BLOCK_CURSOR_LIGHT(&nblk->consumed, global_vsn + 1, + BBQ_BLOCK_SPSC_INIT_VALUE); + q->ridx++; + return true; +} +#endif diff --git a/include/vsync/queue/internal/bbq/common.h b/include/vsync/queue/internal/bbq/common.h new file mode 100644 index 00000000..43a8dec5 --- /dev/null +++ b/include/vsync/queue/internal/bbq/common.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_COMMON_H +#define VSYNC_BBQ_COMMON_H + +#include +#include +#include +#include + +/* utils */ + +/* size related */ +/* minimum and maximum entry size, block size and block number */ +#define BBQ_ENTRY_SIZE (1ULL << BBQ_ENTRY_SIZE_LOG) +#define BBQ_MIN_BLOCK_NUM_LOG 0U +#define BBQ_MAX_BLOCK_NUM_LOG 16U +#define BBQ_MIN_ENTRY_SIZE 4U +#define BBQ_MAX_ENTRY_SIZE (1ULL << (BBQ_MAX_BLOCK_SIZE_LOG - 1)) +#define BBQ_MIN_BLOCK_SIZE_LOG 4U +#define BBQ_MAX_BLOCK_SIZE_LOG 16U + +/* offset and mask */ +/* global var related */ +#define BBQ_GLOBA_VERSION_BIT 44U /* 64 - 20 */ +#define BBQ_GLOBA_VERSION_MASK ((1ULL << BBQ_GLOBA_VERSION_BIT) - 1) +#define BBQ_GLOBAL_IDX(v) ((v) & ((1ULL << BBQ_BLOCK_NUM_LOG) - 1)) +#define BBQ_GLOBAL_VSN(v) (((v) >> BBQ_BLOCK_NUM_LOG) & BBQ_GLOBA_VERSION_MASK) +#define BBQ_GLOBAL_COMPOSE(h, l) (((h) << BBQ_BLOCK_NUM_LOG) | (l)) + +/* local var related */ +#define BBQ_LOCAL_SPACE_BIT 20U /* >16 to provent the FAA overflow */ +#define BBQ_LOCAL_SPACE_MASK ((1ULL << BBQ_LOCAL_SPACE_BIT) - 1) +#define BBQ_LOCAL_IDX(v) ((v)&BBQ_LOCAL_SPACE_MASK) +#define BBQ_LOCAL_VSN(v) ((v) >> BBQ_LOCAL_SPACE_BIT) +#define BBQ_LOCAL_COMPOSE(h, l) (((h) << BBQ_LOCAL_SPACE_BIT) | (l)) + +typedef struct bbq_config_s { + vuint16_t blk_size_log; /* total size of each block (in log) */ + vuint16_t blk_size; /* total size of each block */ +} bbq_config_t; + +#define BBQ_BLOCK_INIT_VALUE(S) \ + ((sizeof(S) / BBQ_ENTRY_SIZE + 1) * BBQ_ENTRY_SIZE) + +#ifdef VSYNC_VERIFICATION +static vuint32_t g_verify_bbq_count = 0; + #define BBQ_VERIFY_BLK_COUNT(count) g_verify_bbq_count = (count) + #define BBQ_BLK_SZ_VERIFICATION(name) \ + do { \ + ASSERT(g_verify_bbq_count != 0 && "must set BBQ_COUNT"); \ + vuint32_t count_per_blk = \ + (g_verify_bbq_count) >> BBQ_BLOCK_NUM_LOG; \ + blk_size = BBQ_BLOCK_INIT_VALUE(struct bbq_##name##_block_s) + \ + (count_per_blk << BBQ_ENTRY_SIZE_LOG); \ + } while (0) +#else + #define BBQ_VERIFY_BLK_COUNT(count) \ + do { \ + } while (0) + #define BBQ_BLK_SZ_VERIFICATION(name) \ + do { \ + } while (0) +#endif + +#define BBQ_GET_BLOCK(rb, idx) \ + (void *)((rb)->blk + ((idx) << (rb)->config.blk_size_log)) +#define BBQ_GET_NEXT_BLOCK(rb, idx) \ + (BBQ_GET_BLOCK(rb, BBQ_GLOBAL_IDX((idx) + 1))) +#define BBQ_GET_ENTRY(blk, offset) \ + ((void *)(((vuintptr_t)(blk)) + (vuintptr_t)(offset))) + +#define BBQ_BLOCK_FULLY_CONSUMED_WITH_VSN(blk, sz, vsn) \ + ({ \ + vuint64_t consumed = vatomic64_read_acq(&(blk)->consumed); \ + (BBQ_LOCAL_IDX(consumed) == (sz) && \ + BBQ_LOCAL_VSN(consumed) == (vsn)) || \ + BBQ_LOCAL_VSN(consumed) > (vsn); \ + }) + +#define BBQ_RESET_BLOCK_CURSOR_LIGHT(v, new_vsn, init_v) \ + vatomic64_write_rlx(v, BBQ_LOCAL_COMPOSE(new_vsn, init_v)) + +static inline void +bbq_reset_block_cursor_heavy(vatomic64_t *v, vuint64_t new_vsn, + vuintptr_t init_v) +{ + vuint64_t new_cursor = BBQ_LOCAL_COMPOSE(new_vsn, init_v); + vatomic64_max(v, new_cursor); +} + +#define BBQ_ADVANCE_HEAD(v, old, new) vatomic64_max(v, new) + +#define BBQ_COUNT(rb, name, name_uc) \ + ({ \ + vuint64_t ridx = BBQ_##name_uc##_READ_CONS((rb)->ridx); \ + struct bbq_##name##_block_s *rblk = \ + BBQ_GET_BLOCK(rb, BBQ_GLOBAL_IDX(ridx)); \ + vuintptr_t consumed_index = \ + BBQ_LOCAL_IDX(vatomic64_read_rlx(&rblk->consumed)); \ + \ + vuint64_t widx = BBQ_##name_uc##_READ_PROD((rb)->widx); \ + struct bbq_##name##_block_s *wblk = \ + BBQ_GET_BLOCK(rb, BBQ_GLOBAL_IDX(widx)); \ + vuintptr_t committed_index = \ + BBQ_LOCAL_IDX(vatomic64_read_rlx(&wblk->committed)); \ + \ + vuintptr_t block_size = (rb)->config.blk_size; \ + ((((widx - ridx) * \ + (block_size - BBQ_BLOCK_INIT_VALUE(struct bbq_##name##_block_s))) + \ + (committed_index - consumed_index)) >> \ + BBQ_ENTRY_SIZE_LOG); \ + }) + +#endif /* VSYNC_BBQ_COMMON_H */ diff --git a/include/vsync/queue/mpsc.h b/include/vsync/queue/mpsc.h new file mode 100644 index 00000000..ea225708 --- /dev/null +++ b/include/vsync/queue/mpsc.h @@ -0,0 +1,247 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of + * + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list + * + * of conditions and the following disclaimer in the documentation and/or + * other materials + * + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of Dmitry Vyukov. + */ +#ifndef VSYNC_MPSC_H +#define VSYNC_MPSC_H +/** + * @file mpsc.h + * @brief Multi-producer single-consumer queue. + * + * Enqueue operation is wait-free. + * + * @ingroup unbounded_queue + * + * @example + * @include eg_mpsc.c + * + * @cite [Non-intrusive MPSC node-based queue] + * (https://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue) + * + */ +#include +#include +#include + +typedef struct vmpsc_node_s { + void *data; /* address of data object associated with the node */ + vatomicptr_t + next; /* address of vmpsc_node_t object connected to this node */ +} vmpsc_node_t; + +typedef struct vmpsc_s { + vatomicptr_t tail; /* address of most recent node in the queue */ + vmpsc_node_t *head; /* address of least recent node in the queue */ + vmpsc_node_t sentinel; /* sentinel node */ +} vmpsc_t; + +typedef void (*vmpsc_visit_t)(vmpsc_node_t *node, void *args); +typedef vbool_t (*vmpsc_data_visit_t)(void *data, void *args); +/** + * Initializes the given queue object `q`. + * + * @param q address of vmpsc_t object. + * + * @note call only before threads start accessing the queue. + */ +static inline void +vmpsc_init(vmpsc_t *q) +{ + ASSERT(q); + vmpsc_node_t *sentinel = &q->sentinel; + sentinel->data = NULL; + vatomicptr_write_rlx(&sentinel->next, NULL); + /* both head and tail point to the sentinel */ + q->head = sentinel; + vatomicptr_write_rlx(&q->tail, sentinel); +} +/** + * Calls the given `f` function on each enqueued node's data. + * + * @param q address of vmpsc_t object. + * @param f function pointer of type vmpsc_data_visit_t. + * @param args extra arguments of `f`. + * + * @note Don't call while producer threads are active. + */ +static inline void +vmpsc_foreach(const vmpsc_t *q, vmpsc_data_visit_t f, void *args) +{ + vmpsc_node_t *cur = q->head; + vmpsc_node_t *next = NULL; + while (cur) { + /* the first node of the queue is always a sentinel */ + next = vatomicptr_read_rlx(&cur->next); + if (next) { + if (!f(next->data, args)) { + break; + }; + } + cur = next; + } +} +/** + * Calls the given `f` function on each enqueued node. + * + * @param q address of vmpsc_t object. + * @param f function pointer of type vmpsc_visit_t. + * @param args extra arguments of `f`. + * + * @note call only after all threads are done accessing the queue. + * @note use the callback function to destruct nodes and data objects. + */ +static inline void +vmpsc_destroy(vmpsc_t *q, vmpsc_visit_t f, void *args) +{ + vmpsc_node_t *cur = q->head; + vmpsc_node_t *next = NULL; + while (cur) { + next = vatomicptr_read_rlx(&cur->next); + if (cur != &q->sentinel) { + f(cur, args); + } + cur = next; + } +} +/** + * Enqueues the given node into the given queue. + * + * @param q address of vmpsc_t object. + * @param node address of vmpsc_node_t object to enqueue. + * @param data address of data object to be associated with the given node. + */ +static inline void +vmpsc_enq(vmpsc_t *q, vmpsc_node_t *node, void *data) +{ + vmpsc_node_t *tail = NULL; + ASSERT(q); + ASSERT(node); + ASSERT(data); + node->data = data; + vatomicptr_write_rlx(&node->next, NULL); + /* move the tail to node, and record the old tail */ + tail = vatomicptr_xchg(&q->tail, node); + ASSERT(tail); + /* connect the old tail to node */ + vatomicptr_write_rel(&tail->next, node); +} +/** + * Dequeues a node and data object from the queue. + * + * @param q address of vmpsc_t object. + * @param discarded_node output parameter. Address of vmpsc_node_t sentinel + * object to destroy, or NULL. + * + * @return void* address of dequeued data object. + * @return NULL when the queue is empty. + * + * @note on each successful dequeue there will be a sentinel node object kicked + * out of the queue. + */ +static inline void * +vmpsc_deq(vmpsc_t *q, vmpsc_node_t **discarded_node) +{ + void *entry = NULL; + vmpsc_node_t *head = q->head; + vmpsc_node_t *next = vatomicptr_read_acq(&head->next); + *discarded_node = NULL; + if (next == NULL) { + if (vatomicptr_read_rlx(&q->tail) == head) { + return NULL; + } + /* tail moved, a producer is mid-enq on wait for the entry to become + * available */ + next = vatomicptr_await_neq_acq(&head->next, NULL); + } + entry = next->data; + assert(entry); + next->data = NULL; + q->head = next; + + if (likely(head != &q->sentinel)) { + /* + * NOTE: Why is SMR not needed here? + * At this point `head->next` is not NULL. + * and in line vatomicptr_write_rel(&tail->next, node); of deq + * tail->next is NULL. hence tail != head. + * and it is safe to free head. + */ + *discarded_node = head; + } + return entry; +} +/** + * Checks if the current queue is empty or not. + * + * @param q address of vmpsc_t object. + * + * @return true the queue is empty. + * @return false the queue is not empty. + * + * @note to be called only by the consumer. + */ +static inline vbool_t +vmpsc_empty(vmpsc_t *q) +{ + vmpsc_node_t *head = q->head; + vmpsc_node_t *tail = vatomicptr_read(&q->tail); + vmpsc_node_t *next = vatomicptr_read(&tail->next); + return (next == NULL) && (head == tail); +} +/** + * Calculates how many nodes are enqueued in the queue. + * + * @param q address of vmpsc_t object. + * @return vsize_t number of available elements in the queue. + * + * @note to be called only by the consumer. + * @note only non-sentinel nodes are counted. + */ +static inline vsize_t +vmpsc_length(vmpsc_t *q) +{ + vmpsc_node_t *curr = q->head; + vmpsc_node_t *next = NULL; + vsize_t len = 0; + while (curr) { + // TODO: check next of tail! + next = vatomicptr_read(&curr->next); + if (next) { + len++; + } + curr = next; + } + return len; +} +#endif diff --git a/include/vsync/spinlock/doc.h b/include/vsync/spinlock/doc.h index 34958f03..c265f68b 100644 --- a/include/vsync/spinlock/doc.h +++ b/include/vsync/spinlock/doc.h @@ -1,7 +1,8 @@ /* - * Copyright (C) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. - * Author: Huawei Dresden Research Center + * Copyright (C) Huawei Technologies Co., Ltd. 2025-2026. All rights reserved. + * SPDX-License-Identifier: MIT */ + #ifndef VDOC_H #define VDOC_H /******************************************************************************* diff --git a/include/vsync/utils/internal/math.h b/include/vsync/utils/internal/math.h deleted file mode 100644 index 9aab48ba..00000000 --- a/include/vsync/utils/internal/math.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) Huawei Technologies Co., Ltd. 2024-2025. All rights reserved. - * SPDX-License-Identifier: MIT - */ - -#ifndef VSYNC_UTILS_INTERNAL_MATH_H -#define VSYNC_UTILS_INTERNAL_MATH_H -#include - -#ifdef VSYNC_VERIFICATION -static inline vuint32_t -v_log2(vuint32_t v) -{ - #define V_ARR_LEN 5U - const vuint32_t b[V_ARR_LEN] = {0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}; - const vuint32_t S[V_ARR_LEN] = {1, 2, 4, 8, 16}; - vsize_t i = 0; - register vuint32_t r = 0; // result of log2(v) will go here - for (i = V_ARR_LEN - 1; i < V_ARR_LEN; i--) { - if (v & b[i]) { - v >>= S[i]; - r |= S[i]; - } - } - #undef V_ARR_LEN - return r; -} -/** - * Returns the next power of two - * - * @param v value to round - * - * @see - * https://stackoverflow.com/questions/466204/rounding-up-to-next-power-of-2 - */ -static inline vuint32_t -v_pow2_round_up(vuint32_t v) -{ - v--; - /* this loop should shift and or for 1, 2, 4, 8, 16 */ - for (vuint32_t shift_by = 1; shift_by <= 16; shift_by *= 2) { - v |= v >> shift_by; - } - v++; - return v; -} - -static inline vuint32_t -v_pow2_round_down(vuint32_t v) -{ - /* this loop should shift and or for 1, 2, 4, 8, 16 */ - for (vuint32_t shift_by = 1; shift_by <= 16; shift_by *= 2) { - v |= v >> shift_by; - } - return v - (v >> 1); -} -#endif -#endif diff --git a/include/vsync/utils/math.h b/include/vsync/utils/math.h index 63492e17..74e640c1 100644 --- a/include/vsync/utils/math.h +++ b/include/vsync/utils/math.h @@ -1,5 +1,5 @@ /* - * Copyright (C) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + * Copyright (C) Huawei Technologies Co., Ltd. 2023-2026. All rights reserved. * SPDX-License-Identifier: MIT */ @@ -9,13 +9,9 @@ #include #include -#ifdef VSYNC_VERIFICATION - #include -#else - - /* We define a few constants to make the linter happy. */ - #define V_NUM_8 8U - #define V_NUM_32 32U +/* We define a few constants to make the linter happy. */ +#define V_NUM_8 8U +#define V_NUM_32 32U static inline vuint32_t v_log2(vuint32_t v) @@ -40,10 +36,8 @@ v_pow2_round_down(vuint32_t v) return 1U << ((V_NUM_32 - (vuint32_t)__builtin_clz(v)) - 1U); } - #undef V_NUM_8 - #undef V_NUM_32 - -#endif +#undef V_NUM_8 +#undef V_NUM_32 #ifndef V_IS_POWER_OF_TWO /** diff --git a/test/bbq/CMakeLists.txt b/test/bbq/CMakeLists.txt new file mode 100644 index 00000000..3085ecde --- /dev/null +++ b/test/bbq/CMakeLists.txt @@ -0,0 +1,15 @@ +target_include_directories( + vsync INTERFACE $ + $) +file(GLOB SRCS *.c) +foreach(SRC ${SRCS}) + + get_filename_component(TEST ${SRC} NAME_WE) + set(TEST test_${TEST}) + + add_executable(${TEST} ${SRC}) + target_link_libraries(${TEST} vsync pthread) + target_compile_options(${TEST} PRIVATE -O3) + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + set_tests_properties(${TEST} PROPERTIES COST 300) +endforeach() diff --git a/test/bbq/align_test.c b/test/bbq/align_test.c new file mode 100644 index 00000000..b30db56c --- /dev/null +++ b/test/bbq/align_test.c @@ -0,0 +1,88 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +uint32_t +log2_of(vuint32_t v) +{ +#if 0 + return ((unsigned)(8 * sizeof(unsigned long long) - __builtin_clzll((v)) - 1)); +#else + + const unsigned int b[] = {0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}; + const unsigned int S[] = {1, 2, 4, 8, 16}; + int i; + + register unsigned int r = 0; // result of log2(v) will go here + for (i = 4; i >= 0; i--) // unroll for speed... + { + if (v & b[i]) { + v >>= S[i]; + r |= S[i]; + } + } + return r; +#endif +} + +uint32_t +pow2_round_up(vuint32_t v) +{ +#if 1 + return v == 1 ? 1 : 1 << (32 - __builtin_clz(v - 1)); +#else + v--; + for (vuint32_t shift_by = 1; shift_by <= 16; shift_by *= 2) { + v |= v >> shift_by; + } + return v; +#endif +} + +uint32_t +pow2_round_down(vuint32_t v) +{ +#if 0 + return (1U << (32U - __builtin_clz((vuint32_t)(v)) - 1U)); +#else + for (vuint32_t shift_by = 1; shift_by <= 16; shift_by *= 2) { + v |= v >> shift_by; + } + return v - (v >> 1); +#endif +} + +#define bbq_align_down_with_power2(a) (pow2_round_down(a)) + +#define bbq_log2(X) (log2_of(X)) + +int +main(void) +{ + // 2^f2(x) = f1(x)，f1 is the align down and f2 is log2 + // for example, f2(1023) = 9, f2(1024) = 10 + // Jiawei Wang(84201116)2022-12-15 08:11 + // f1(1023) = 512, f1(1024) = 1024 + + ASSERT(bbq_align_down_with_power2(1023) == 512); + ASSERT(pow2_round_up(1023) == 1024); + + ASSERT(bbq_align_down_with_power2(1024) == 1024); + ASSERT(pow2_round_up(1024) == 1024); + + ASSERT(bbq_align_down_with_power2(1023) == 512); + ASSERT(pow2_round_up(1023) == 1024); + + ASSERT(bbq_align_down_with_power2(1023) == 512); + ASSERT(pow2_round_up(1023) == 1024); + + ASSERT(bbq_log2(1023) == 9); + ASSERT(bbq_log2(1024) == 10); + + return 0; +} diff --git a/test/bbq/bbq_mpmc.c b/test/bbq/bbq_mpmc.c new file mode 100644 index 00000000..319fb4ac --- /dev/null +++ b/test/bbq/bbq_mpmc.c @@ -0,0 +1,102 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM 2000000UL +#define NUM_WRITER 4 +#define NUM_READER 5 + +#define BUFFER_ENTRY_NUM 4096 +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL + +#include + +struct bbq_mpmc_s *rb; + +void * +writer(void *arg) +{ + vuint64_t *id = (vuint64_t *)arg; + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM * NUM_READER; + while (rest) { + vuint32_t count = VMIN(rest, ENQUEUE_BATCH); + ASSERT(count <= ENQUEUE_BATCH); + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ((*id) << 30) | (ptr++); + } + rest -= bbq_mpmc_enqueue(rb, buf, count, true); + } + free(arg); + return NULL; +} + +void * +reader(void *arg) +{ + V_UNUSED(arg); + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t last[NUM_WRITER] = {0}; + vuint64_t rest = NUM * NUM_WRITER; + while (rest) { + vuint32_t count = VMIN(rest, DEQUEUE_BATCH); + count = bbq_mpmc_dequeue(rb, buf, count, false); + for (vuint32_t i = 0; i < count; i++) { + vuint64_t id = buf[i] >> 30; + vuint64_t data = buf[i] & ((1ULL << 30) - 1); + ASSERT(id < NUM_WRITER); + ASSERT(last[id] <= data); + last[id] = data; + } + rest -= count; + } + return NULL; +} + +int +main(void) +{ + vsize_t sz = bbq_mpmc_memsize(BUFFER_ENTRY_NUM); + rb = (bbq_mpmc_t *)malloc(sz); + if (rb == NULL) { + perror("fail to create the ring buffer"); + abort(); + } + BBQ_VERIFY_BLK_COUNT(BUFFER_ENTRY_NUM); + vbool_t success = bbq_mpmc_init(rb, sz); + ASSERT(success); + ASSERT(BBQ_MPMC_COUNT(rb) == 0); + + pthread_t t1[NUM_WRITER], t2[NUM_READER]; + for (vsize_t i = 0; i < NUM_WRITER; i++) { + vuint64_t *arg = malloc(sizeof(*arg)); + *arg = i; + pthread_create(&t1[i], NULL, writer, arg); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_create(&t2[i], NULL, reader, NULL); + } + + for (vsize_t i = 0; i < NUM_WRITER; i++) { + pthread_join(t1[i], NULL); + } + for (vsize_t i = 0; i < NUM_READER; i++) { + pthread_join(t2[i], NULL); + } + + ASSERT(BBQ_MPMC_COUNT(rb) == 0); + free(rb); + return 0; +} diff --git a/test/bbq/bbq_spsc.c b/test/bbq/bbq_spsc.c new file mode 100644 index 00000000..fee9e673 --- /dev/null +++ b/test/bbq/bbq_spsc.c @@ -0,0 +1,85 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM 200000000UL +#define BUFFER_ENTRY_NUM 4096 +#define ENQUEUE_BATCH 5UL +#define DEQUEUE_BATCH 4UL + +#include + +bbq_spsc_t *rb; + +void * +writer(void *arg) +{ + V_UNUSED(arg); + vuintptr_t buf[ENQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + while (rest) { + vuint32_t count = VMIN(rest, ENQUEUE_BATCH); + ASSERT(count <= ENQUEUE_BATCH); + for (vuint32_t i = 0; i < count; i++) { + buf[i] = ptr++; + } + rest -= bbq_spsc_enqueue(rb, buf, count, true); + } + return NULL; +} + +void * +reader(void *arg) +{ + V_UNUSED(arg); + vuintptr_t buf[DEQUEUE_BATCH] = {0}; + vuint64_t ptr = 0; + vuint64_t rest = NUM; + while (rest) { + vuint32_t count = bbq_spsc_dequeue(rb, buf, DEQUEUE_BATCH, false); + for (vuint32_t i = 0; i < count; i++) { + ASSERT(buf[i] == ptr++); + } + rest -= count; + } + return NULL; +} + +int +main(void) +{ + vuint32_t sz = bbq_spsc_memsize(BUFFER_ENTRY_NUM); + rb = (struct bbq_spsc_s *)malloc(sz); + if (rb == NULL) { + perror("fail to create the ring buffer"); + abort(); + } + BBQ_VERIFY_BLK_COUNT(BUFFER_ENTRY_NUM); + + vbool_t success = bbq_spsc_init(rb, sz); + ASSERT(success); + ASSERT(BBQ_SPSC_COUNT(rb) == 0); + + pthread_t t1, t2; + pthread_create(&t1, NULL, writer, NULL); + pthread_create(&t2, NULL, reader, NULL); + + pthread_join(t1, NULL); + pthread_join(t2, NULL); + + ASSERT(BBQ_SPSC_COUNT(rb) == 0); + free(rb); + return 0; +} diff --git a/test/bbq/include/test/bbq/bbq_test_helper.h b/test/bbq/include/test/bbq/bbq_test_helper.h new file mode 100644 index 00000000..b6e64446 --- /dev/null +++ b/test/bbq/include/test/bbq/bbq_test_helper.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_TEST_HELPER +#define VSYNC_BBQ_TEST_HELPER + +#if defined(VSYNC_VERIFICATION) + + #include + +/* TODO: linters will complain about this even if VSYNC_VERIFICATION is not + * defined! */ + +static inline int +memcpy_s(void *dst, vsize_t dstsz, void *src, vsize_t srcsz) +{ + ASSERT(srcsz <= dstsz); + vuintptr_t *d = (vuintptr_t *)dst; + vuintptr_t *s = (vuintptr_t *)src; + for (vuint64_t i = 0; i < dstsz / sizeof(vuintptr_t); i++) { + d[i] = s[i]; + } + return 0; +} +#endif + +#endif diff --git a/test/bbq/include/test/bbq/debug.h b/test/bbq/include/test/bbq/debug.h new file mode 100644 index 00000000..e9c04999 --- /dev/null +++ b/test/bbq/include/test/bbq/debug.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_DEBUG_H +#define VSYNC_BBQ_DEBUG_H + +/* debug function */ +static inline void +bbq_spsc_debug(struct bbq_spsc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_SPSC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(rb->widx); + vuint64_t write_version = BBQ_GLOBAL_VSN(rb->widx); + vuint64_t read_index = BBQ_GLOBAL_IDX(rb->ridx); + vuint64_t read_version = BBQ_GLOBAL_VSN(rb->ridx); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_SPSC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_spsc_block_s *blk = (struct bbq_spsc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("committed: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", committed_version, committed_space); + printf("consumed: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_mpsc_debug(struct bbq_mpsc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_MPSC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->widx)); + vuint64_t write_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->widx)); + vuint64_t read_index = BBQ_GLOBAL_IDX(rb->ridx); + vuint64_t read_version = BBQ_GLOBAL_VSN(rb->ridx); + + DBG("------------------------------------------------------------------------\n"); + DBG("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + DBG("Block number: %" VUINT16_FORMAT "\n", block_number); + DBG("Block init value: %" VUINT16_FORMAT "\n", BBQ_BLOCK_MPSC_INIT_VALUE); + DBG("Block size: %" VUINT16_FORMAT "\n", block_size); + DBG("Entry size: %" VUINT64_FORMAT "\n", (vuint64_t) BBQ_ENTRY_SIZE); + DBG("Data space: %" VUINT16_FORMAT "\n", data_space); + DBG("widx: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", write_version, write_index); + DBG("ridx: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_mpsc_block_s *blk = (struct bbq_mpsc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t allocated_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->allocated)); + vuint64_t allocated_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->allocated)); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("allocated: (%ld,\t%ld)\n", allocated_version, allocated_space); + printf("committed: (%ld,\t%ld)\n", committed_version, committed_space); + printf("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_spmc_debug(struct bbq_spmc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_SPMC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(rb->widx); + vuint64_t write_version = BBQ_GLOBAL_VSN(rb->widx); + vuint64_t read_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->ridx)); + vuint64_t read_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->ridx)); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_SPMC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_spmc_block_s *blk = (struct bbq_spmc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t reserved_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->reserved)); + vuint64_t reserved_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->reserved)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("committed: (%ld,\t%ld)\n", committed_version, committed_space); + printf("reserved: (%ld,\t%ld)\n", reserved_version, reserved_space); + printf("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_mpmc_debug(struct bbq_mpmc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_MPMC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->widx)); + vuint64_t write_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->widx)); + vuint64_t read_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->ridx)); + vuint64_t read_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->ridx)); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_MPMC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_mpmc_block_s *blk = (struct bbq_mpmc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t allocated_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->allocated)); + vuint64_t allocated_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->allocated)); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t reserved_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->reserved)); + vuint64_t reserved_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->reserved)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + DBG("allocated: (%ld,\t%ld)\n", allocated_version, allocated_space); + DBG("committed: (%ld,\t%ld)\n", committed_version, committed_space); + DBG("reserved: (%ld,\t%ld)\n", reserved_version, reserved_space); + DBG("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +#endif diff --git a/test/include/test/bbq/bbq_test_helper.h b/test/include/test/bbq/bbq_test_helper.h new file mode 100644 index 00000000..b6e64446 --- /dev/null +++ b/test/include/test/bbq/bbq_test_helper.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_TEST_HELPER +#define VSYNC_BBQ_TEST_HELPER + +#if defined(VSYNC_VERIFICATION) + + #include + +/* TODO: linters will complain about this even if VSYNC_VERIFICATION is not + * defined! */ + +static inline int +memcpy_s(void *dst, vsize_t dstsz, void *src, vsize_t srcsz) +{ + ASSERT(srcsz <= dstsz); + vuintptr_t *d = (vuintptr_t *)dst; + vuintptr_t *s = (vuintptr_t *)src; + for (vuint64_t i = 0; i < dstsz / sizeof(vuintptr_t); i++) { + d[i] = s[i]; + } + return 0; +} +#endif + +#endif diff --git a/test/include/test/bbq/debug.h b/test/include/test/bbq/debug.h new file mode 100644 index 00000000..e9c04999 --- /dev/null +++ b/test/include/test/bbq/debug.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BBQ_DEBUG_H +#define VSYNC_BBQ_DEBUG_H + +/* debug function */ +static inline void +bbq_spsc_debug(struct bbq_spsc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_SPSC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(rb->widx); + vuint64_t write_version = BBQ_GLOBAL_VSN(rb->widx); + vuint64_t read_index = BBQ_GLOBAL_IDX(rb->ridx); + vuint64_t read_version = BBQ_GLOBAL_VSN(rb->ridx); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_SPSC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_spsc_block_s *blk = (struct bbq_spsc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("committed: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", committed_version, committed_space); + printf("consumed: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_mpsc_debug(struct bbq_mpsc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_MPSC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->widx)); + vuint64_t write_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->widx)); + vuint64_t read_index = BBQ_GLOBAL_IDX(rb->ridx); + vuint64_t read_version = BBQ_GLOBAL_VSN(rb->ridx); + + DBG("------------------------------------------------------------------------\n"); + DBG("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + DBG("Block number: %" VUINT16_FORMAT "\n", block_number); + DBG("Block init value: %" VUINT16_FORMAT "\n", BBQ_BLOCK_MPSC_INIT_VALUE); + DBG("Block size: %" VUINT16_FORMAT "\n", block_size); + DBG("Entry size: %" VUINT64_FORMAT "\n", (vuint64_t) BBQ_ENTRY_SIZE); + DBG("Data space: %" VUINT16_FORMAT "\n", data_space); + DBG("widx: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", write_version, write_index); + DBG("ridx: (%" VUINT64_FORMAT ",\t%" VUINT64_FORMAT ")\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_mpsc_block_s *blk = (struct bbq_mpsc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t allocated_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->allocated)); + vuint64_t allocated_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->allocated)); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("allocated: (%ld,\t%ld)\n", allocated_version, allocated_space); + printf("committed: (%ld,\t%ld)\n", committed_version, committed_space); + printf("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_spmc_debug(struct bbq_spmc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_SPMC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(rb->widx); + vuint64_t write_version = BBQ_GLOBAL_VSN(rb->widx); + vuint64_t read_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->ridx)); + vuint64_t read_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->ridx)); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_SPMC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_spmc_block_s *blk = (struct bbq_spmc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t reserved_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->reserved)); + vuint64_t reserved_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->reserved)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + printf("committed: (%ld,\t%ld)\n", committed_version, committed_space); + printf("reserved: (%ld,\t%ld)\n", reserved_version, reserved_space); + printf("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +static inline void +bbq_mpmc_debug(struct bbq_mpmc_s *rb) +{ + (void)rb; +#if 0 + vuint16_t block_number = 1U << BBQ_BLOCK_NUM_LOG; + vuint16_t block_size = rb->config.blk_size; + vuint16_t data_space = block_size - BBQ_BLOCK_MPMC_INIT_VALUE; + vuint64_t write_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->widx)); + vuint64_t write_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->widx)); + vuint64_t read_index = BBQ_GLOBAL_IDX(vatomic64_read(&rb->ridx)); + vuint64_t read_version = BBQ_GLOBAL_VSN(vatomic64_read(&rb->ridx)); + + printf("------------------------------------------------------------------------\n"); + printf("Drop old mode: %d\n", BBQ_MODE == BBQ_DROP_OLD); + printf("Block number: %d\n", block_number); + printf("Block init value: %ld\n", BBQ_BLOCK_MPMC_INIT_VALUE); + printf("Block size: %d\n", block_size); + printf("Entry size: %ld\n", BBQ_ENTRY_SIZE); + printf("Data space: %d\n", data_space); + printf("widx: (%ld,\t%ld)\n", write_version, write_index); + printf("ridx: (%ld,\t%ld)\n", read_version, read_index); + + for (int i = 0; i < block_number; i++) { + printf("------------------------------------------------\n"); + + struct bbq_mpmc_block_s *blk = (struct bbq_mpmc_block_s *) (rb->blk + i * block_size); + printf("[Block %d] %p \n", i, (void *) blk); + + vuint64_t allocated_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->allocated)); + vuint64_t allocated_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->allocated)); + + vuint64_t committed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->committed)); + vuint64_t committed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->committed)); + + vuint64_t reserved_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->reserved)); + vuint64_t reserved_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->reserved)); + + vuint64_t consumed_space = BBQ_LOCAL_IDX(vatomic64_read(&blk->consumed)); + vuint64_t consumed_version = BBQ_LOCAL_VSN(vatomic64_read(&blk->consumed)); + + DBG("allocated: (%ld,\t%ld)\n", allocated_version, allocated_space); + DBG("committed: (%ld,\t%ld)\n", committed_version, committed_space); + DBG("reserved: (%ld,\t%ld)\n", reserved_version, reserved_space); + DBG("consumed: (%ld,\t%ld)\n", consumed_version, consumed_space); + } +#endif +} + +#endif diff --git a/test/include/test/impsc.h b/test/include/test/impsc.h new file mode 100644 index 00000000..8571c26a --- /dev/null +++ b/test/include/test/impsc.h @@ -0,0 +1,227 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_IMPSC_H +#define VSYNC_IMPSC_H + +#include +#include +#include +#include + +#define NTRACES (NTHREADS + 1U) +#define TRACE_LEN 10U + +trace_t g_deq_trace; /* one consumer, one trace */ +trace_t g_enq_trace[NTRACES]; /* traces of producers */ +trace_t g_final_state; /* final state of the queue */ + +static inline vbool_t +collect_cb(void *data, void *args) +{ + ASSERT(data); + trace_add(&g_final_state, (vuintptr_t)data); + V_UNUSED(args); + return true; +} +static inline void +dump_traces(trace_t *total_enq) +{ + trace_print(total_enq, "total_enq"); + trace_print(&g_final_state, "g_final_state"); + trace_print(&g_deq_trace, "g_deq_trace"); +} + +static inline void +init_traces(void) +{ + trace_init(&g_deq_trace, TRACE_LEN); + trace_init(&g_final_state, TRACE_LEN); + for (vsize_t i = 0; i < NTRACES; i++) { + trace_init(&g_enq_trace[i], TRACE_LEN); + } +} +static inline void +destroy_traces(void) +{ + trace_destroy(&g_deq_trace); + trace_destroy(&g_final_state); + for (vsize_t i = 0; i < NTRACES; i++) { + trace_destroy(&g_enq_trace[i]); + } +} +static inline void +print_cb(vuintptr_t k) +{ + fprintf(stderr, "%lu\n", k); +} +static inline void +verify_traces(void) +{ + trace_t total_enq; + trace_init(&total_enq, TRACE_LEN); + /* verify */ + for (vsize_t i = 0; i < NTRACES; i++) { + trace_merge_into(&total_enq, &g_enq_trace[i]); + } + + vbool_t subtrace = trace_is_subtrace(&total_enq, &g_deq_trace, NULL); + if (!subtrace) { + dump_traces(&total_enq); + } + ASSERT(subtrace && + "Dequeued set of elements is not a subset of enqueued ones."); + subtrace = trace_is_subtrace(&total_enq, &g_final_state, NULL); + if (!subtrace) { + dump_traces(&total_enq); + } + + ASSERT(subtrace && + "Remaining set of elements is not a subset of enqueued ones."); + trace_merge_into(&g_final_state, &g_deq_trace); + vbool_t equal = trace_are_eq(&g_final_state, &total_enq, NULL); + if (!equal) { + dump_traces(&total_enq); + } + ASSERT(equal && + "The set of dequeued and remaining elements is not equal to " + "enqueued ones."); + trace_destroy(&total_enq); +} + +#ifdef MPSC + #include + +vmpsc_t queue; + +void +init(void) +{ + vmpsc_init(&queue); + init_traces(); +} + +vuintptr_t +deq(void) +{ + vmpsc_node_t *node = NULL; + vuintptr_t v = (vuintptr_t)vmpsc_deq(&queue, &node); + if (v) { + trace_add(&g_deq_trace, v); + free(node); + } + return v; +} + +void +enq(vsize_t tid, vuintptr_t v) +{ + vmpsc_node_t *node = (vmpsc_node_t *)malloc(sizeof(vmpsc_node_t)); + ASSERT(node); + ASSERT(tid < NTRACES); + trace_add(&g_enq_trace[tid], v); + return vmpsc_enq(&queue, node, (void *)v); +} + +vbool_t +is_empty(void) +{ + return vmpsc_empty(&queue); +} + +vsize_t +get_len(void) +{ + return vmpsc_length(&queue); +} + +void +des_cb(vmpsc_node_t *node, void *args) +{ + free(node); + V_UNUSED(args); +} +void +fini(void) +{ + vmpsc_foreach(&queue, collect_cb, NULL); + verify_traces(); + vmpsc_destroy(&queue, des_cb, NULL); + destroy_traces(); +} +#elif LEVEL_QUEUE + + #include + #include + +void * +lvl_queue_alloc_cb(vsize_t sz, void *arg) +{ + V_UNUSED(arg); + #if defined(VSYNC_VERIFICATION) + return vmem_malloc(sz); + #else + return vmem_aligned_malloc(VSYNC_CACHELINE_SIZE, sz); + #endif +} + +vmem_lib_t g_mem_lib = {.free_fun = vmem_free_cb, + .malloc_fun = lvl_queue_alloc_cb, + .arg = NULL}; +level_queue_t q; + +void +init(void) +{ + level_queue_init(&q, &g_mem_lib); + init_traces(); +} + +vuintptr_t +deq(void) +{ + vuintptr_t v = (vuintptr_t)level_queue_dequeue(&q); + if (v) { + trace_add(&g_deq_trace, v); + } + return v; +} + +level_queue_prod_t g_prods[NTHREADS]; +void +enq(vsize_t tid, vuintptr_t v) +{ + ASSERT(tid < NTHREADS); + if (g_prods[tid] == NULL) { + level_queue_prod_init(&q, &g_prods[tid]); + } + level_queue_enqueue(&q, &g_prods[tid], (void *)v); + ASSERT(tid < NTRACES); + trace_add(&g_enq_trace[tid], v); +} + +vbool_t +is_empty(void) +{ + return level_queue_empty(&q); +} + +vsize_t +get_len(void) +{ + return level_queue_length(&q); +} + +void +fini(void) +{ + level_queue_foreach(&q, collect_cb, NULL); + verify_traces(); + level_queue_destroy(&q); + destroy_traces(); +} + +#endif +#endif diff --git a/test/include/test/trace_manager.h b/test/include/test/trace_manager.h index 0ec5c316..2f5d915c 100644 --- a/test/include/test/trace_manager.h +++ b/test/include/test/trace_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + * Copyright (C) Huawei Technologies Co., Ltd. 2023-2026. All rights reserved. * SPDX-License-Identifier: MIT */ @@ -14,7 +14,7 @@ #include #include -#define VTRACE_EXTEND_FACTOR 2 +#define VTRACE_EXTEND_FACTOR 2U typedef struct trace_unit_s { vuintptr_t key; @@ -30,6 +30,81 @@ typedef struct trace_s { typedef vbool_t (*trace_verify_unit)(trace_unit_t *unit); +#if defined(VSYNC_VERIFICATION_DAT3M) +/* DAT3M cannot handle this at the moment! */ +static inline void +trace_init(trace_t *trace, vsize_t capacity) +{ + V_UNUSED(trace, capacity); +} +static inline void +trace_reset(trace_t *trace) +{ + V_UNUSED(trace); +} +static inline vsize_t +trace_get_length(trace_t *trace) +{ + V_UNUSED(trace); + return 0; +} +static inline vbool_t +trace_is_subtrace(trace_t *super_trace, trace_t *sub_trace, + void (*print)(vuintptr_t k)) +{ + V_UNUSED(super_trace, sub_trace, print); + return true; +} +static inline vbool_t +trace_are_eq(trace_t *trace_a, trace_t *trace_b, void (*print)(vuintptr_t k)) +{ + V_UNUSED(trace_a, trace_b, print); + return true; +} +static inline vbool_t +trace_verify(trace_t *trace, trace_verify_unit verify_fun) +{ + V_UNUSED(trace, verify_fun); + return true; +} +static inline void +trace_print(trace_t *trace, char *trace_name) +{ + V_UNUSED(trace, trace_name); +} +static inline void +trace_subtract_from(trace_t *trace_container, trace_t *trace) +{ + V_UNUSED(trace_container, trace); +} +static inline void +trace_merge_into(trace_t *trace_container, trace_t *trace) +{ + V_UNUSED(trace_container, trace); +} +static inline void +trace_add(trace_t *trace, vuintptr_t key) +{ + V_UNUSED(trace, key); +} +static inline vbool_t +trace_find_unit_idx(trace_t *trace, vuintptr_t key, vsize_t *out_idx) +{ + V_UNUSED(trace, key, out_idx); + return true; +} +static inline void +trace_destroy(trace_t *trace) +{ + V_UNUSED(trace); +} +static inline void +trace_extend(trace_t *trace) +{ + V_UNUSED(trace); +} +#else + static inline void trace_init(trace_t *trace, vsize_t capacity) { @@ -307,11 +382,12 @@ trace_is_subtrace(trace_t *super_trace, trace_t *sub_trace, ASSERT(unit_a->key == unit_b->key); - if (unit_a->count != unit_b->count) { + if (unit_a->count > unit_b->count) { if (print) { - printf("key[%" VUINTPTR_FORMAT - "] count is different %zu != %zu\n", - unit_a->key, unit_a->count, unit_b->count); + printf( + "key[%" VUINTPTR_FORMAT + "] count in subtrace %zu > count in super-trace %zu\n", + unit_a->key, unit_a->count, unit_b->count); print(unit_a->key); } return false; @@ -328,3 +404,4 @@ trace_is_subtrace(trace_t *super_trace, trace_t *sub_trace, return true; } #endif +#endif diff --git a/test/mpsc/CMakeLists.txt b/test/mpsc/CMakeLists.txt new file mode 100644 index 00000000..5507a42d --- /dev/null +++ b/test/mpsc/CMakeLists.txt @@ -0,0 +1,45 @@ +target_include_directories( + vsync INTERFACE $ + $) + +file(GLOB TEST_FILES *.c) + +# find the number of processors +ProcessorCount(PCOUNT) + +if(${LIBVSYNC_CROSS_TESTS}) + # QEMU might be too slow, we cannot really stress test + set(NUM_THREADS 4) + set(ITERATIONS 10) +else() + math(EXPR NUM_THREADS "${PCOUNT}*2") + set(ITERATIONS 10000) +endif() + +set(TEST_DEFS -D_GNU_SOURCE NTHREADS=${NUM_THREADS} IT=${ITERATIONS} + DBG_ENABLE_COL_ALL VSYNC_ENABLE_DEBUG) + +if(NOT DEFINED ALGOS) + set(ALGOS MPSC) +endif() + +foreach(test_path IN ITEMS ${TEST_FILES}) + # extract test_name with extension + get_filename_component(test_name ${test_path} NAME) + + # name without extension + get_filename_component(test_case ${test_path} NAME_WE) + + foreach(algo IN ITEMS ${ALGOS}) + + set(TEST test_${test_case}_${algo}) + string(TOLOWER ${TEST} TEST) + add_executable(${TEST} ${test_name}) + target_link_libraries(${TEST} vsync pthread) + target_compile_definitions(${TEST} PUBLIC ${algo} ${TEST_DEFS}) + + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + + set_tests_properties(${TEST} PROPERTIES TIMEOUT 300 COST 60) + endforeach() +endforeach() diff --git a/test/mpsc/include/test/impsc.h b/test/mpsc/include/test/impsc.h new file mode 100644 index 00000000..8571c26a --- /dev/null +++ b/test/mpsc/include/test/impsc.h @@ -0,0 +1,227 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_IMPSC_H +#define VSYNC_IMPSC_H + +#include +#include +#include +#include + +#define NTRACES (NTHREADS + 1U) +#define TRACE_LEN 10U + +trace_t g_deq_trace; /* one consumer, one trace */ +trace_t g_enq_trace[NTRACES]; /* traces of producers */ +trace_t g_final_state; /* final state of the queue */ + +static inline vbool_t +collect_cb(void *data, void *args) +{ + ASSERT(data); + trace_add(&g_final_state, (vuintptr_t)data); + V_UNUSED(args); + return true; +} +static inline void +dump_traces(trace_t *total_enq) +{ + trace_print(total_enq, "total_enq"); + trace_print(&g_final_state, "g_final_state"); + trace_print(&g_deq_trace, "g_deq_trace"); +} + +static inline void +init_traces(void) +{ + trace_init(&g_deq_trace, TRACE_LEN); + trace_init(&g_final_state, TRACE_LEN); + for (vsize_t i = 0; i < NTRACES; i++) { + trace_init(&g_enq_trace[i], TRACE_LEN); + } +} +static inline void +destroy_traces(void) +{ + trace_destroy(&g_deq_trace); + trace_destroy(&g_final_state); + for (vsize_t i = 0; i < NTRACES; i++) { + trace_destroy(&g_enq_trace[i]); + } +} +static inline void +print_cb(vuintptr_t k) +{ + fprintf(stderr, "%lu\n", k); +} +static inline void +verify_traces(void) +{ + trace_t total_enq; + trace_init(&total_enq, TRACE_LEN); + /* verify */ + for (vsize_t i = 0; i < NTRACES; i++) { + trace_merge_into(&total_enq, &g_enq_trace[i]); + } + + vbool_t subtrace = trace_is_subtrace(&total_enq, &g_deq_trace, NULL); + if (!subtrace) { + dump_traces(&total_enq); + } + ASSERT(subtrace && + "Dequeued set of elements is not a subset of enqueued ones."); + subtrace = trace_is_subtrace(&total_enq, &g_final_state, NULL); + if (!subtrace) { + dump_traces(&total_enq); + } + + ASSERT(subtrace && + "Remaining set of elements is not a subset of enqueued ones."); + trace_merge_into(&g_final_state, &g_deq_trace); + vbool_t equal = trace_are_eq(&g_final_state, &total_enq, NULL); + if (!equal) { + dump_traces(&total_enq); + } + ASSERT(equal && + "The set of dequeued and remaining elements is not equal to " + "enqueued ones."); + trace_destroy(&total_enq); +} + +#ifdef MPSC + #include + +vmpsc_t queue; + +void +init(void) +{ + vmpsc_init(&queue); + init_traces(); +} + +vuintptr_t +deq(void) +{ + vmpsc_node_t *node = NULL; + vuintptr_t v = (vuintptr_t)vmpsc_deq(&queue, &node); + if (v) { + trace_add(&g_deq_trace, v); + free(node); + } + return v; +} + +void +enq(vsize_t tid, vuintptr_t v) +{ + vmpsc_node_t *node = (vmpsc_node_t *)malloc(sizeof(vmpsc_node_t)); + ASSERT(node); + ASSERT(tid < NTRACES); + trace_add(&g_enq_trace[tid], v); + return vmpsc_enq(&queue, node, (void *)v); +} + +vbool_t +is_empty(void) +{ + return vmpsc_empty(&queue); +} + +vsize_t +get_len(void) +{ + return vmpsc_length(&queue); +} + +void +des_cb(vmpsc_node_t *node, void *args) +{ + free(node); + V_UNUSED(args); +} +void +fini(void) +{ + vmpsc_foreach(&queue, collect_cb, NULL); + verify_traces(); + vmpsc_destroy(&queue, des_cb, NULL); + destroy_traces(); +} +#elif LEVEL_QUEUE + + #include + #include + +void * +lvl_queue_alloc_cb(vsize_t sz, void *arg) +{ + V_UNUSED(arg); + #if defined(VSYNC_VERIFICATION) + return vmem_malloc(sz); + #else + return vmem_aligned_malloc(VSYNC_CACHELINE_SIZE, sz); + #endif +} + +vmem_lib_t g_mem_lib = {.free_fun = vmem_free_cb, + .malloc_fun = lvl_queue_alloc_cb, + .arg = NULL}; +level_queue_t q; + +void +init(void) +{ + level_queue_init(&q, &g_mem_lib); + init_traces(); +} + +vuintptr_t +deq(void) +{ + vuintptr_t v = (vuintptr_t)level_queue_dequeue(&q); + if (v) { + trace_add(&g_deq_trace, v); + } + return v; +} + +level_queue_prod_t g_prods[NTHREADS]; +void +enq(vsize_t tid, vuintptr_t v) +{ + ASSERT(tid < NTHREADS); + if (g_prods[tid] == NULL) { + level_queue_prod_init(&q, &g_prods[tid]); + } + level_queue_enqueue(&q, &g_prods[tid], (void *)v); + ASSERT(tid < NTRACES); + trace_add(&g_enq_trace[tid], v); +} + +vbool_t +is_empty(void) +{ + return level_queue_empty(&q); +} + +vsize_t +get_len(void) +{ + return level_queue_length(&q); +} + +void +fini(void) +{ + level_queue_foreach(&q, collect_cb, NULL); + verify_traces(); + level_queue_destroy(&q); + destroy_traces(); +} + +#endif +#endif diff --git a/test/mpsc/mt_fifo.c b/test/mpsc/mt_fifo.c new file mode 100644 index 00000000..91d26d5c --- /dev/null +++ b/test/mpsc/mt_fifo.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#if defined(VSYNC_ADDRESS_SANITIZER) + #undef NTHREADS + #undef IT + #define NTHREADS 4U + #define IT 128U +#endif +#include +#include +#include +#define CONSUMER_DELAY_MICRO_SEC 200U +#define CONSUME_COUNT (IT * NTHREADS) +void +consume(void) +{ + usleep(CONSUMER_DELAY_MICRO_SEC); + vuintptr_t last = 0; + vuintptr_t v = 0; + for (vsize_t i = 0; i < CONSUME_COUNT; i++) { + v = deq(); + if (v != 0 && v != IT + 1) { + if (last != 0) { + ASSERT(v == last + 1); + } + last = v; + } + } +} +void +produce(vsize_t tid) +{ + vuintptr_t v = 0; + for (vsize_t i = 1; i <= IT; i++) { + if (tid == 1) { + v = i; + } else { + v = IT + 1; + } + enq(tid, v); + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == 0) { + consume(); + } else { + produce(tid); + } + return NULL; +} + +int +main(void) +{ + init(); + launch_threads(NTHREADS, run); + fini(); + return 0; +} diff --git a/test/mpsc/mt_sanity.c b/test/mpsc/mt_sanity.c new file mode 100644 index 00000000..278a8879 --- /dev/null +++ b/test/mpsc/mt_sanity.c @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#if defined(VSYNC_ADDRESS_SANITIZER) + #undef NTHREADS + #undef IT + #define NTHREADS 4U + #define IT 128U +#endif +#include +#include +#include +#define CONSUMER_DELAY_MICRO_SEC 200U +void +consume(void) +{ + usleep(CONSUMER_DELAY_MICRO_SEC); + for (vsize_t i = 0; i < IT; i++) { + deq(); + } +} +void +produce(vsize_t tid) +{ + vuintptr_t v = 0; + for (vsize_t i = 1; i <= IT; i++) { + v = (NTHREADS * tid) + i; + enq(tid, v); + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == 0) { + consume(); + } else { + produce(tid); + } + return NULL; +} + +int +main(void) +{ + init(); + launch_threads(NTHREADS, run); + fini(); + return 0; +} diff --git a/test/mpsc/spsc_sanity.c b/test/mpsc/spsc_sanity.c new file mode 100644 index 00000000..787591e3 --- /dev/null +++ b/test/mpsc/spsc_sanity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#undef NTHREADS +#define NTHREADS 2U +/** + * @file spsc_sanity.c + * @brief assert FIFO order of single producer, single consumer + * + */ +#include +#include +#include + +#if defined(VSYNC_ADDRESS_SANITIZER) + #define NUM_ENTRIES 10000U +#else + #define NUM_ENTRIES 100000U +#endif + +void +consume(void) +{ + vuintptr_t v = 0; + for (vuintptr_t i = 1; i <= NUM_ENTRIES; i++) { + while (v = deq(), !v) { + /* ignore empty dequeues */ + } + ASSERT(v == i); + } +} +void +produce(vsize_t tid) +{ + for (vuintptr_t i = 1; i <= NUM_ENTRIES; i++) { + enq(tid, i); + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == 0) { + consume(); + } else if (tid == 1) { + produce(tid); + } + return NULL; +} + +int +main(void) +{ + init(); + launch_threads(NTHREADS, run); + fini(); + return 0; +} diff --git a/test/mpsc/ut_sanity.c b/test/mpsc/ut_sanity.c new file mode 100644 index 00000000..b03acff9 --- /dev/null +++ b/test/mpsc/ut_sanity.c @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include + +void +ut_empty(void) +{ + ASSERT(is_empty()); + ASSERT(get_len() == 0); + for (vsize_t i = 0; i < 10; i++) { + ASSERT(deq() == 0); + } + ASSERT(is_empty()); + ASSERT(get_len() == 0); +} + +void +ut_len(void) +{ + const vsize_t count = 10000; + for (vsize_t i = 1; i <= count; i++) { + enq(0, (vuintptr_t)i); + ASSERT(get_len() == i); + } + ASSERT(!is_empty()); + ASSERT(get_len() == count); + for (vsize_t i = 1; i <= count; i++) { + ASSERT(deq() == i); + if (get_len() != count - i) { + fprintf(stderr, "%zu %zu\n", get_len(), count - i); + } + ASSERT(get_len() == (count - i)); + } + ASSERT(is_empty()); +} + +int +main(void) +{ + init(); + ut_empty(); + ut_len(); + fini(); + return 0; +} diff --git a/verify/bbq/CMakeLists.txt b/verify/bbq/CMakeLists.txt new file mode 100644 index 00000000..04bcf557 --- /dev/null +++ b/verify/bbq/CMakeLists.txt @@ -0,0 +1,31 @@ +file(GLOB SRCS *.c) +foreach(SRC ${SRCS}) + get_filename_component(TEST ${SRC} NAME_WE) + + add_executable(${TEST} ${SRC}) + target_link_libraries(${TEST} vsync pthread) + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + + add_vsyncer_check( + TARGET check_${TEST} + SOURCE ${SRC} + TIMEOUT 1200 + DEPENDENCIES vsync) +endforeach() + +# Enable some tests with VMM and set extra options for Dartagnan +set(DAT3M_BOUND_bbq_spsc 2) + +foreach(SRC ${SRCS}) + get_filename_component(TEST ${SRC} NAME_WE) + + if(${DAT3M_BOUND_${TEST}}) + add_vsyncer_check( + TARGET check_${TEST} + SOURCE ${SRC} + MEMORY_MODELS vmm USE_DAT3M DARTAGNAN_OPTIONS + --bound=${DAT3M_BOUND_${TEST}} + TIMEOUT 700 + DEPENDENCIES vsync) + endif() +endforeach() diff --git a/verify/bbq/bbq_mpsc.c b/verify/bbq/bbq_mpsc.c new file mode 100644 index 00000000..eafa08cd --- /dev/null +++ b/verify/bbq/bbq_mpsc.c @@ -0,0 +1,102 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +/* small enough such that producer may block with queue full */ +#define BBQ_BLOCK_NUM_LOG 1U +#define BUFFER_ENTRY_NUM 2 +#define NUM_WRITER 2 +#define NUM_READER 1 + +#include + +struct bbq_mpmc_s *q; + +static void * +writer(void *arg) +{ + vuint64_t *id = (vuint64_t *)arg; + vuint64_t count = (*id) + 1; + vuintptr_t ptr = 1; + while (count) { + vuint32_t r = bbq_mpmc_enqueue(q, &ptr, 1, true); + if (r) { + count--; + ptr++; + } + } + free(arg); + return NULL; +} + +static void * +reader(void *arg) +{ + vuint64_t count = 0; + vuintptr_t exp = 1; + + while (count < 1) { + vuintptr_t ptr; + vuint32_t r = bbq_mpmc_dequeue(q, &ptr, 1, true); + if (r) { + ASSERT(ptr == 1); + count++; + exp++; + } + } + free(arg); + return NULL; +} + +#define SIZE \ + ((sizeof(struct bbq_mpmc_s)) + \ + (BUFFER_ENTRY_NUM * \ + (sizeof(struct bbq_mpmc_block_s) + sizeof(vuintptr_t)))) + +int +main(void) +{ + ASSERT((SIZE) <= 2048); + + vuint32_t sz = bbq_mpmc_memsize(BUFFER_ENTRY_NUM); + q = (struct bbq_mpmc_s *)malloc(sz); + ASSERT(q != NULL && "failed to create the ring buffer"); + BBQ_VERIFY_BLK_COUNT(BUFFER_ENTRY_NUM); + vbool_t success = bbq_mpmc_init(q, sz); + ASSERT(success); + ASSERT(BBQ_MPMC_COUNT(q) == 0); + + pthread_t t1[NUM_WRITER], t2[NUM_READER]; + + for (int i = 0; i < NUM_WRITER; i++) { + vuint64_t *arg = malloc(sizeof(*arg)); + *arg = i; + pthread_create(&t1[i], NULL, writer, arg); + } + + for (int i = 0; i < NUM_READER; i++) { + vuint64_t *arg = malloc(sizeof(*arg)); + *arg = i; + pthread_create(&t2[i], NULL, reader, arg); + } + + for (int i = 0; i < NUM_WRITER; i++) + pthread_join(t1[i], NULL); + for (int i = 0; i < NUM_READER; i++) + pthread_join(t2[i], NULL); + + free(q); + + return 0; +} diff --git a/verify/bbq/bbq_spmc.c b/verify/bbq/bbq_spmc.c new file mode 100644 index 00000000..0ae22c64 --- /dev/null +++ b/verify/bbq/bbq_spmc.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#include +#include +#include +#include +#include +#include + +/* small enough such that producer may block with queue full */ +#define BBQ_BLOCK_NUM_LOG 1U +#define BUFFER_ENTRY_NUM 2 +#define NUM_WRITER 1 +#define NUM_READER 2 + +#include + +struct bbq_mpmc_s *q; + +static void * +writer(void *arg) +{ + vuint64_t count = 3; + vuintptr_t ptr = 1; + while (count) { + vuint32_t r = bbq_mpmc_enqueue(q, &ptr, 1, true); + if (r) { + count--; + ptr++; + } + } + free(arg); + return NULL; +} + +static void * +reader(void *arg) +{ + vuint64_t count = 1; + while (count) { + vuintptr_t ptr; + vuint32_t r = bbq_mpmc_dequeue(q, &ptr, 1, true); + if (r) { + ASSERT(ptr == 1 || ptr == 2); + count--; + } + } + free(arg); + return NULL; +} + +#define SIZE \ + ((sizeof(struct bbq_mpmc_s)) + \ + (BUFFER_ENTRY_NUM * \ + (sizeof(struct bbq_mpmc_block_s) + sizeof(vuintptr_t)))) + +int +main(void) +{ + ASSERT((SIZE) <= 2048); + vuint32_t sz = bbq_mpmc_memsize(BUFFER_ENTRY_NUM); + q = (struct bbq_mpmc_s *)malloc(sz); + ASSERT(q != NULL && "failed to create the ring buffer"); + BBQ_VERIFY_BLK_COUNT(BUFFER_ENTRY_NUM); + vbool_t success = bbq_mpmc_init(q, sz); + ASSERT(success); + ASSERT(BBQ_MPMC_COUNT(q) == 0); + + pthread_t t1[NUM_WRITER], t2[NUM_READER]; + + for (int i = 0; i < NUM_WRITER; i++) { + vuint64_t *arg = malloc(sizeof(*arg)); + *arg = i; + pthread_create(&t1[i], NULL, writer, arg); + } + + for (int i = 0; i < NUM_READER; i++) { + vuint64_t *arg = malloc(sizeof(*arg)); + *arg = i; + pthread_create(&t2[i], NULL, reader, arg); + } + + for (int i = 0; i < NUM_WRITER; i++) + pthread_join(t1[i], NULL); + for (int i = 0; i < NUM_READER; i++) + pthread_join(t2[i], NULL); + + free(q); + + return 0; +} diff --git a/verify/bbq/bbq_spsc.c b/verify/bbq/bbq_spsc.c new file mode 100644 index 00000000..6a684aff --- /dev/null +++ b/verify/bbq/bbq_spsc.c @@ -0,0 +1,83 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#include +#include +#include +#include +#include + +/* small enough such that producer may block with queue full */ +#define BBQ_BLOCK_NUM_LOG 1U +#define BUFFER_ENTRY_NUM 2 + +/* large enough such that blocks are reused at least once */ +#define NUM 5UL + +#include +#include +struct bbq_spsc_s *q; +uintptr_t total_enq = 0; +uintptr_t total_deq = 0; + +static void * +writer(void *arg) +{ + V_UNUSED(arg); + for (vuintptr_t i = 0; i < NUM; i++) { + if (bbq_spsc_enqueue(q, &total_enq, 1, true)) { + total_enq++; + } + } + return NULL; +} + +static void * +reader(void *arg) +{ + V_UNUSED(arg); + vuintptr_t buf; + for (vuintptr_t i = 0; i < NUM; i++) { + if (bbq_spsc_dequeue(q, &buf, 1, true)) { + ASSERT(buf == total_deq); + total_deq++; + } + } + return NULL; +} + +#define SIZE \ + ((sizeof(struct bbq_spsc_s)) + \ + (BUFFER_ENTRY_NUM * \ + (sizeof(struct bbq_spsc_block_s) + sizeof(vuintptr_t)))) + +int +main(void) +{ + ASSERT((SIZE) <= 1024); + + vsize_t sz = bbq_spsc_memsize(BUFFER_ENTRY_NUM); + q = (struct bbq_spsc_s *)malloc(sz); + ASSERT(q != NULL && "failed to create the ring buffer"); + BBQ_VERIFY_BLK_COUNT(BUFFER_ENTRY_NUM); + vbool_t success = bbq_spsc_init(q, sz); + ASSERT(success); + ASSERT(BBQ_SPSC_COUNT(q) == 0); + + pthread_t t1, t2; + + pthread_create(&t1, NULL, writer, NULL); + pthread_create(&t2, NULL, reader, NULL); + + pthread_join(t1, NULL); + pthread_join(t2, NULL); + + ASSERT(BBQ_SPSC_COUNT(q) == total_enq - total_deq); + + free(q); + + return 0; +} diff --git a/verify/cachedq/CMakeLists.txt b/verify/cachedq/CMakeLists.txt index 76a9dc0b..1e1a81cd 100644 --- a/verify/cachedq/CMakeLists.txt +++ b/verify/cachedq/CMakeLists.txt @@ -11,3 +11,15 @@ add_vsyncer_check( TIMEOUT 30 DEPENDENCIES vsync MEMORY_MODELS imm rc11) + +set(DAT3M_BOUND_test_cachedq 1) + +if(${DAT3M_BOUND_${TEST_NAME}}) + add_vsyncer_check( + TARGET check_cachedq + SOURCE ${FILE_NAME} + TIMEOUT 180 + MEMORY_MODELS vmm USE_DAT3M DARTAGNAN_OPTIONS + --bound=${DAT3M_BOUND_${TEST_NAME}} + DEPENDENCIES vsync) +endif() diff --git a/verify/mpsc/CMakeLists.txt b/verify/mpsc/CMakeLists.txt new file mode 100644 index 00000000..ac8f708e --- /dev/null +++ b/verify/mpsc/CMakeLists.txt @@ -0,0 +1,75 @@ +# detect test cases header files +file(GLOB TEST_CASES test_case*.h) + +set(TEST_DEFS -D_GNU_SOURCE -DNTHREADS=3) + +if(NOT DEFINED ALGOS) + set(ALGOS MPSC) +endif() + +# verification template file +set(VERIFY_FILE verify.c) + +# disabled for now with dat3m set(DAT3M_BOUND_LEVEL_QUEUE 4) +set(DAT3M_BOUND_MPSC 4) + +set(LEVEL_QUEUE_CONFIG -DLEVEL_QUEUE_BLOCK_NUM_ENTRIES=1 + -DLEVEL_QUEUE_BLOCK_NUM_ENTRIES=2) + +set(MPSC_CONFIG -DDEFAULT) + +set(CHECK_TIMEOUT 3600) + +foreach(test_path IN ITEMS ${TEST_CASES}) + # name without extension + get_filename_component(test_case ${test_path} NAME_WE) + + # define TEST_CASE, we need to pass it like this to be recognized + set(tc TEST_CASE="${test_path}") + # we have to escape it like this to work for check + set(tc_check -DTEST_CASE="'\"${test_path}\"'" -DVMEM_LIB_ALLOC_TRACKING_OFF) + + foreach(algo IN ITEMS ${ALGOS}) + + set(TEST_PREFIX ${test_case}_${algo}) + string(TOLOWER ${TEST_PREFIX} TEST_PREFIX) + + set(CONFIG_IDX 0) + foreach(CONFIG IN ITEMS ${${algo}_CONFIG}) + math(EXPR CONFIG_IDX "${CONFIG_IDX}+1") + set(TEST ${TEST_PREFIX}_${CONFIG_IDX}) + # add the executable + add_executable(${TEST} ${VERIFY_FILE}) + # link libs + target_link_libraries(${TEST} vsync pthread) + # activate target algo by adding the appropriate define + target_compile_definitions(${TEST} PUBLIC ${algo} ${TEST_DEFS} + ${tc} ${CONFIG}) + + # add it as a test + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + + add_vsyncer_check( + TARGET check_${TEST} + SOURCE ${VERIFY_FILE} + CFLAGS -D${algo} ${TEST_DEFS} ${tc_check} ${CONFIG} + DEPENDENCIES vsync + TIMEOUT ${CHECK_TIMEOUT} # + ) + + if(${DAT3M_BOUND_${algo}}) + add_vsyncer_check( + TARGET check_${TEST} + SOURCE ${VERIFY_FILE} + CFLAGS -D${algo} ${TEST_DEFS} ${tc_check} ${CONFIG} + USE_DAT3M + MEMORY_MODELS vmm DARTAGNAN_OPTIONS + --bound=${DAT3M_BOUND_${algo}} + DEPENDENCIES vsync + TIMEOUT ${CHECK_TIMEOUT} # + ${CHECK_OPTIONS}) + endif() + + endforeach(CONFIG) + endforeach(algo) +endforeach(test_path) diff --git a/verify/mpsc/test_case_2psc.h b/verify/mpsc/test_case_2psc.h new file mode 100644 index 00000000..09e5c8f1 --- /dev/null +++ b/verify/mpsc/test_case_2psc.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef TEST_CASE_H +#define TEST_CASE_H + +/** + * Two producers vs. single consumer. + * + * Ensure message passing is sound. + * Ensure FIFO at the end. + * + */ + +#define MAX_DEQUEUE 2U +vuintptr_t vals[MAX_DEQUEUE] = {0}; +vsize_t len = 0; + +int msg = 0; + +#define VALID_COMB_LEN 7U +vuintptr_t valid_com[VALID_COMB_LEN][MAX_DEQUEUE] = { + {3, 4}, {1, 2}, {1, 3}, {3, 1}, {0, 0}, {1, 0}, {3, 0}}; + +vbool_t +is_valid(void) +{ + for (vsize_t i = 0; i < VALID_COMB_LEN; i++) { + vbool_t match = true; + for (vsize_t j = 0; j < MAX_DEQUEUE; j++) { + if (valid_com[i][j] != vals[j]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; +} + +int +_deq(void) +{ + vuintptr_t v = deq(); + if (v) { + vals[len++] = v; + } + return v; +} + +void +pre(void) +{ +} + +void +consumer(void) +{ + vuintptr_t a = _deq(); + vuintptr_t b = _deq(); + if (b == 4) { + assert(msg == 11); + } + V_UNUSED(a); +} + +void +p1(void) +{ + vsize_t tid = 0; + enq(tid, 1); + enq(tid, 2); +} + +void +p2(void) +{ + vsize_t tid = 1; + enq(tid, 3); + msg = 11; + enq(tid, 4); +} + +void +post(void) +{ + vbool_t fifo_valid = is_valid(); + if (!fifo_valid) { + printf("{"); + for (vsize_t i = 0; i < len; i++) { + printf("%lu, ", vals[i]); + } + printf("}\n"); + ASSERT(0 && "FIFO violated!"); + } +} +#endif diff --git a/verify/mpsc/test_case_blk_edge.h b/verify/mpsc/test_case_blk_edge.h new file mode 100644 index 00000000..cd03266a --- /dev/null +++ b/verify/mpsc/test_case_blk_edge.h @@ -0,0 +1,135 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef TEST_CASE_H +#define TEST_CASE_H +/** + * This test case targets an edge case in level queue. + * It is relevant when block capacity is 2. + * + * The edge case happens when the block is not full yet, + * but the consumer has consumed all produced elements in that block so far. + * The consumer then decides to reattach the block to the end of the queue, and + * continue to consume from a newly dequeued block. Between these two steps the + * producer of that block might already fill the block and attach a new block. + * If the current block is abandoned by the consumer then that will lead to + * violating FIFO. Hence, the consumer must recheck if the state of the + * block has changed after it moved it to the end of the queue: + * - if it has not changed it means the producer will use it for future + * enqueues, and all of its future blocks will be enqueued in order. + * - if it has changed and there are new elements to consume then the consumer + * will make sure to consume them before moving to another block. + * + * Test case description:0 + * + * Producers: + * P1[tid=1]: enq 1, 2, 3 + * P2[tid=2]: enq 4 + * + * consumer: consumes 4 elements + * + * P1 enqueues 1, P2 enqueues 4: + * + * blk_1 [1, X] -> blk_2 [5, X] + * + * consumer consumes 1, found blk_1 has no data + * + * P1 enqueues 2, 3 and 4: + * blk_1 [X, 2] -> blk_2 [5, X] -> blk_3 [3, X] + * + * consumer move blk_1 to the end: + * blk_2 [4, X] -> blk_3 [3, X] -> blk_1 [X, 2] + * + * If the consumer does not recheck the state of blk_1 + * it will consume 1, 4, 3, 2 // FIFO order violated 3 is consumed before 2 + * If it does it will consume: 1, 2, 4, 3 // FIFO order maintained 2 is consumed + * before 3 + * + */ + +#define MAX_DEQUEUE 4U +#define VALID_COMB_LEN 4U + +vuintptr_t vals[MAX_DEQUEUE] = {0}; + +/* acceptable dequeue order maintaining FIFO */ +vuintptr_t valid_com[VALID_COMB_LEN][MAX_DEQUEUE] = {{4, 1, 2, 3}, + {1, 4, 2, 3}, + {1, 2, 4, 3}, + {1, 2, 3, 4}}; + +vbool_t +is_valid(void) +{ + for (vsize_t i = 0; i < VALID_COMB_LEN; i++) { + vbool_t match = true; + for (vsize_t j = 0; j < MAX_DEQUEUE; j++) { + if (valid_com[i][j] != vals[j]) { + match = false; + break; + } + } + if (match) { + return true; + } + } + return false; +} + +void +pre(void) +{ + // pre-enqueue for thread 1, to speed up the test case + enq(1, 1); +#if LEVEL_QUEUE_BLOCK_NUM_ENTRIES == 1 || defined(MPSC) + // speed up for block capacity 2 or when targeting the other queue + enq(1, 2); +#endif +} + +void +consumer(void) +{ + vuintptr_t v = 0; + for (vsize_t i = 0; i < MAX_DEQUEUE; i++) { + do { + v = deq(); + verification_assume(v != 0); + vals[i] = v; + } while (v == 0); + } +} + +void +p1(void) +{ + vsize_t tid = 1; +#if LEVEL_QUEUE_BLOCK_NUM_ENTRIES == 2 && !defined(MPSC) + enq(tid, 2); +#endif + enq(tid, 3); +} + +void +p2(void) +{ + vsize_t tid = 2; + enq(tid, 4); +} + +void +post(void) +{ + vbool_t fifo_valid = is_valid(); + if (!fifo_valid) { + printf("{"); + for (vsize_t i = 0; i < MAX_DEQUEUE; i++) { + printf("%lu, ", vals[i]); + } + printf("}\n"); + ASSERT(0 && "FIFO violated!"); + } +} +#endif diff --git a/verify/mpsc/test_case_spsc.h b/verify/mpsc/test_case_spsc.h new file mode 100644 index 00000000..125aad84 --- /dev/null +++ b/verify/mpsc/test_case_spsc.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef TEST_CASE_H +#define TEST_CASE_H + +/*** + * Single producer vs. single consumer. + * + * Enqueue 4 values. Dequeue max 4. + * Verify dequeue order is FIFO at the end. + */ + +#define MAX_DEQUEUE 3U +vuintptr_t vals[MAX_DEQUEUE] = {0}; +vsize_t len = 0; + +void pre(void){}; +void +consumer(void) +{ + for (vsize_t i = 0; i < MAX_DEQUEUE; i++) { + vuintptr_t value = deq(); + if (value) { + vals[len++] = value; + } + } +} + +void +p1(void) +{ + for (vuintptr_t v = 1; v <= 4; v++) { + enq(0, v); + } +} + +void +p2(void) +{ +} + +void +post(void) +{ + ASSERT(len <= MAX_DEQUEUE); + vuintptr_t last = 0; + vbool_t violation = false; + for (vsize_t i = 0; i < len; i++) { + if (vals[i] != last + 1U) { + violation = true; + break; + } + last = vals[i]; + } + if (violation) { + printf("Dequeue Order = {"); + for (vsize_t i = 0; i < len; i++) { + if (i == len - 1) { + printf("%lu", vals[i]); + } else { + printf("%lu, ", vals[i]); + } + } + printf("}\n"); + ASSERT(0 && "Single thread FIFO violated!"); + } +} + +#endif diff --git a/verify/mpsc/verify.c b/verify/mpsc/verify.c new file mode 100644 index 00000000..f410fdba --- /dev/null +++ b/verify/mpsc/verify.c @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2026. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include + +#ifdef TEST_CASE + #include TEST_CASE +#else + #error "no test case was defined" +#endif + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(uintptr_t)args; + switch (tid) { + case 0: + consumer(); + break; + case 1: + p1(); + break; + case 2: + p2(); + break; + default: + break; + } + return NULL; +} + +int +main(void) +{ + init(); + pre(); + launch_threads(NTHREADS, run); + post(); + fini(); + return 0; +}