From 419835a4adac0f7a8110ef680aa88db857d9bcda Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Tue, 3 Feb 2026 23:38:43 +0900 Subject: [PATCH 01/11] feat(engine): SlateDB storage backend integration (#164) --- .gitignore | 2 + engine/build.gradle.kts | 3 +- .../storage/slatedb/SlateDbException.java | 14 + .../engine/storage/slatedb/SlateDbNative.java | 432 ++++++++++++++++++ .../com/kakao/actionbase/v2/engine/Graph.kt | 7 + .../v2/engine/entity/LabelEntity.kt | 7 +- .../v2/engine/entity/StorageEntity.kt | 4 + .../kakao/actionbase/v2/engine/ffi/MathOps.kt | 30 +- .../engine/label/slatedb/SlateDbHashLabel.kt | 289 ++++++++++++ .../label/slatedb/SlateDbIndexedLabel.kt | 69 +++ .../v2/engine/metadata/StorageType.kt | 1 + .../storage/slatedb/SlateDbConnections.kt | 61 +++ .../engine/storage/slatedb/SlateDbOptions.kt | 28 ++ .../engine/storage/slatedb/SlateDbStorage.kt | 11 + .../v2/engine/storage/slatedb/SlateDbTable.kt | 72 +++ .../storage/slatedb/SlateDbNativeTest.java | 113 +++++ .../label/slatedb/SlateDbHashLabelTest.kt | 241 ++++++++++ .../label/slatedb/SlateDbIndexedLabelTest.kt | 294 ++++++++++++ .../storage/slatedb/SlateDbStorageTest.kt | 65 +++ .../storage/slatedb/SlateDbTableTest.kt | 89 ++++ .../v2/engine/test/GraphFixtures.kt | 6 - native/build-slatedb.sh | 56 +++ server/build.gradle.kts | 9 +- .../actionbase/server/ffi/FfiController.kt | 11 +- .../main/resources/application-slatedb.yaml | 18 + 25 files changed, 1904 insertions(+), 28 deletions(-) create mode 100644 engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java create mode 100644 engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabel.kt create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorage.kt create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt create mode 100644 engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt create mode 100755 native/build-slatedb.sh create mode 100644 server/src/main/resources/application-slatedb.yaml diff --git a/.gitignore b/.gitignore index ac6ace56..6ec7ee05 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ engine/target/ # Native library build artifacts native/src/c/*.dylib native/src/c/*.so +native/slatedb/ +native/lib/ diff --git a/engine/build.gradle.kts b/engine/build.gradle.kts index c69b7cba..7985c0fc 100644 --- a/engine/build.gradle.kts +++ b/engine/build.gradle.kts @@ -1,6 +1,7 @@ -import actionbase.dependencies.Dependencies import org.gradle.jvm.toolchain.JavaLanguageVersion +import actionbase.dependencies.Dependencies + plugins { id("actionbase.kotlin-conventions") id("actionbase.reactor-conventions") diff --git a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java new file mode 100644 index 00000000..97a2f558 --- /dev/null +++ b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java @@ -0,0 +1,14 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb; + +public class SlateDbException extends RuntimeException { + private final int code; + + public SlateDbException(int code, String message) { + super("SlateDB error (" + code + "): " + message); + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java new file mode 100644 index 00000000..78cc28cd --- /dev/null +++ b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java @@ -0,0 +1,432 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.nio.file.Path; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Minimal FFI wrapper for SlateDB C bindings. + *

+ * Uses Java 22+ Foreign Function & Memory API (JEP 454). + * Only exposes the essential operations: open, get, put, delete, close. + */ +public class SlateDbNative implements AutoCloseable { + + // CSdbValue: { data: *mut u8, len: usize } + private static final MemoryLayout C_SDB_VALUE_LAYOUT = MemoryLayout.structLayout( + ValueLayout.ADDRESS.withName("data"), + ValueLayout.JAVA_LONG.withName("len") + ); + + // CSdbResult: { error: i32(enum), padding: i32, message: *const c_char } + private static final MemoryLayout C_SDB_RESULT_LAYOUT = MemoryLayout.structLayout( + ValueLayout.JAVA_INT.withName("error"), + ValueLayout.JAVA_INT.withName("padding"), + ValueLayout.ADDRESS.withName("message") + ); + + // CSdbHandle: { _0: *mut SlateDbFFI } + private static final MemoryLayout C_SDB_HANDLE_LAYOUT = MemoryLayout.structLayout( + ValueLayout.ADDRESS.withName("_0") + ); + + // CSdbHandleResult: { handle: CSdbHandle, result: CSdbResult } + private static final MemoryLayout C_SDB_HANDLE_RESULT_LAYOUT = MemoryLayout.structLayout( + C_SDB_HANDLE_LAYOUT.withName("handle"), + C_SDB_RESULT_LAYOUT.withName("result") + ); + + // CSdbKeyValue: { key: CSdbValue, value: CSdbValue } + private static final MemoryLayout C_SDB_KEY_VALUE_LAYOUT = MemoryLayout.structLayout( + C_SDB_VALUE_LAYOUT.withName("key"), + C_SDB_VALUE_LAYOUT.withName("value") + ); + + // CSdbIteratorNextResult: { kv: CSdbKeyValue, has_value: bool(u8), padding, result: CSdbResult } + private static final MemoryLayout C_SDB_ITERATOR_NEXT_RESULT_LAYOUT = MemoryLayout.structLayout( + C_SDB_KEY_VALUE_LAYOUT.withName("kv"), + ValueLayout.JAVA_BYTE.withName("has_value"), + MemoryLayout.paddingLayout(7), + C_SDB_RESULT_LAYOUT.withName("result") + ); + + private final Arena arena; + private final MemorySegment dbHandle; + private final MethodHandle getHandle; + private final MethodHandle putHandle; + private final MethodHandle deleteHandle; + private final MethodHandle flushHandle; + private final MethodHandle closeHandle; + private final MethodHandle freeValueHandle; + private final MethodHandle scanPrefixHandle; + private final MethodHandle iteratorNextHandle; + private final MethodHandle iteratorCloseHandle; + + private SlateDbNative( + Arena arena, + MemorySegment dbHandle, + MethodHandle getHandle, + MethodHandle putHandle, + MethodHandle deleteHandle, + MethodHandle flushHandle, + MethodHandle closeHandle, + MethodHandle freeValueHandle, + MethodHandle scanPrefixHandle, + MethodHandle iteratorNextHandle, + MethodHandle iteratorCloseHandle + ) { + this.arena = arena; + this.dbHandle = dbHandle; + this.getHandle = getHandle; + this.putHandle = putHandle; + this.deleteHandle = deleteHandle; + this.flushHandle = flushHandle; + this.closeHandle = closeHandle; + this.freeValueHandle = freeValueHandle; + this.scanPrefixHandle = scanPrefixHandle; + this.iteratorNextHandle = iteratorNextHandle; + this.iteratorCloseHandle = iteratorCloseHandle; + } + + private static final int ERROR_NOT_FOUND = 2; + + public byte[] get(byte[] key) throws Throwable { + MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); + MemorySegment valueOut = arena.allocate(C_SDB_VALUE_LAYOUT); + + MemorySegment resultSegment = (MemorySegment) getHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle, + keySegment, + (long) key.length, + MemorySegment.NULL, + valueOut + ); + + // NotFound is not an error, just return null + int errorCode = resultSegment.get(ValueLayout.JAVA_INT, 0); + if (errorCode == ERROR_NOT_FOUND) { + return null; + } + checkResult(resultSegment); + + MemorySegment dataPtr = valueOut.get(ValueLayout.ADDRESS, 0); + if (dataPtr.equals(MemorySegment.NULL)) { + return null; + } + + int len = (int) valueOut.get(ValueLayout.JAVA_LONG, 8); + if (len == 0) { + return new byte[0]; + } + + byte[] bytes = new byte[len]; + dataPtr.reinterpret(len).asByteBuffer().get(bytes); + + freeValueHandle.invokeExact(valueOut); + + return bytes; + } + + public void put(byte[] key, byte[] value) throws Throwable { + MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); + MemorySegment valSegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, value); + + MemorySegment resultSegment = (MemorySegment) putHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle, + keySegment, + (long) key.length, + valSegment, + (long) value.length, + MemorySegment.NULL, + MemorySegment.NULL + ); + + checkResult(resultSegment); + } + + public void delete(byte[] key) throws Throwable { + MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); + + MemorySegment resultSegment = (MemorySegment) deleteHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle, + keySegment, + (long) key.length, + MemorySegment.NULL + ); + + checkResult(resultSegment); + } + + public void flush() throws Throwable { + MemorySegment resultSegment = (MemorySegment) flushHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle + ); + checkResult(resultSegment); + } + + /** + * Scan keys with given prefix and return up to limit key-value pairs. + */ + public List> scanPrefix(byte[] prefix, int limit) throws Throwable { + List> results = new ArrayList<>(); + + // Create iterator + MemorySegment prefixSegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, prefix); + MemorySegment iteratorPtrOut = arena.allocate(ValueLayout.ADDRESS); + + MemorySegment resultSegment = (MemorySegment) scanPrefixHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle, + prefixSegment, + (long) prefix.length, + MemorySegment.NULL, // scan_options + iteratorPtrOut + ); + checkResult(resultSegment); + + MemorySegment iteratorPtr = iteratorPtrOut.get(ValueLayout.ADDRESS, 0); + if (iteratorPtr.equals(MemorySegment.NULL)) { + return results; + } + + try { + // Iterate up to limit + for (int i = 0; i < limit; i++) { + MemorySegment nextResult = (MemorySegment) iteratorNextHandle.invokeExact( + (SegmentAllocator) arena, + iteratorPtr + ); + + // Check if there's a value (has_value at offset 32: 2 CSdbValue = 32 bytes) + byte hasValue = nextResult.get(ValueLayout.JAVA_BYTE, 32); + if (hasValue == 0) { + break; // No more values + } + + // Check for errors (result at offset 40: 32 + 1 + 7 padding = 40) + int errorCode = nextResult.get(ValueLayout.JAVA_INT, 40); + if (errorCode != 0) { + break; // Error or end of iteration + } + + // Extract key (CSdbValue at offset 0) + MemorySegment keyDataPtr = nextResult.get(ValueLayout.ADDRESS, 0); + int keyLen = (int) nextResult.get(ValueLayout.JAVA_LONG, 8); + + // Extract value (CSdbValue at offset 16) + MemorySegment valueDataPtr = nextResult.get(ValueLayout.ADDRESS, 16); + int valueLen = (int) nextResult.get(ValueLayout.JAVA_LONG, 24); + + if (!keyDataPtr.equals(MemorySegment.NULL) && keyLen > 0) { + byte[] keyBytes = new byte[keyLen]; + keyDataPtr.reinterpret(keyLen).asByteBuffer().get(keyBytes); + + byte[] valueBytes = new byte[valueLen]; + if (!valueDataPtr.equals(MemorySegment.NULL) && valueLen > 0) { + valueDataPtr.reinterpret(valueLen).asByteBuffer().get(valueBytes); + } + + results.add(new AbstractMap.SimpleEntry<>(keyBytes, valueBytes)); + } + } + } finally { + // Close iterator + MemorySegment closeResult = (MemorySegment) iteratorCloseHandle.invokeExact( + (SegmentAllocator) arena, + iteratorPtr + ); + // Ignore close errors + } + + return results; + } + + @Override + public void close() throws Exception { + try { + MemorySegment resultSegment = (MemorySegment) closeHandle.invokeExact( + (SegmentAllocator) arena, + dbHandle + ); + checkResult(resultSegment); + } catch (Throwable e) { + throw new SlateDbException(-1, "Failed to close database: " + e.getMessage()); + } finally { + arena.close(); + } + } + + private void checkResult(MemorySegment resultSegment) throws SlateDbException { + int errorCode = resultSegment.get(ValueLayout.JAVA_INT, 0); + if (errorCode != 0) { + MemorySegment msgPtr = resultSegment.get(ValueLayout.ADDRESS, 8); + String msg; + if (!msgPtr.equals(MemorySegment.NULL)) { + msg = msgPtr.reinterpret(1024).getString(0); + } else { + msg = "Unknown error"; + } + throw new SlateDbException(errorCode, msg); + } + } + + /** + * Open database with object store URL. + * For local filesystem, use "file:///path/to/storage" as the url. + */ + public static SlateDbNative open(String dbPath, String url, Path libraryPath) throws Throwable { + Arena arena = Arena.ofShared(); + SymbolLookup lookup = SymbolLookup.libraryLookup(libraryPath, arena); + Linker linker = Linker.nativeLinker(); + + // slatedb_open(path, url, env_file) -> CSdbHandleResult + MethodHandle openHandle = linker.downcallHandle( + lookup.find("slatedb_open").orElseThrow(() -> new IllegalStateException("Function 'slatedb_open' not found")), + FunctionDescriptor.of( + C_SDB_HANDLE_RESULT_LAYOUT, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS + ) + ); + + MemorySegment pathSegment = arena.allocateFrom(dbPath); + MemorySegment urlSegment = url != null ? arena.allocateFrom(url) : MemorySegment.NULL; + MemorySegment handleResultSegment = (MemorySegment) openHandle.invokeExact( + (SegmentAllocator) arena, + pathSegment, + urlSegment, + MemorySegment.NULL + ); + + int errorCode = handleResultSegment.get(ValueLayout.JAVA_INT, 8); + if (errorCode != 0) { + MemorySegment msgPtr = handleResultSegment.get(ValueLayout.ADDRESS, 16); + String msg; + if (!msgPtr.equals(MemorySegment.NULL)) { + msg = msgPtr.reinterpret(1024).getString(0); + } else { + msg = "Failed to open database"; + } + arena.close(); + throw new SlateDbException(errorCode, msg); + } + + MemorySegment dbHandle = handleResultSegment.asSlice(0, 8); + return createInstance(arena, dbHandle, lookup, linker); + } + + private static SlateDbNative createInstance(Arena arena, MemorySegment dbHandle, SymbolLookup lookup, Linker linker) { + // slatedb_get_with_options + MethodHandle getHandle = linker.downcallHandle( + lookup.find("slatedb_get_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_get_with_options' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS + ) + ); + + // slatedb_put_with_options + MethodHandle putHandle = linker.downcallHandle( + lookup.find("slatedb_put_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_put_with_options' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS + ) + ); + + // slatedb_delete_with_options + MethodHandle deleteHandle = linker.downcallHandle( + lookup.find("slatedb_delete_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_delete_with_options' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS + ) + ); + + // slatedb_flush + MethodHandle flushHandle = linker.downcallHandle( + lookup.find("slatedb_flush").orElseThrow(() -> new IllegalStateException("Function 'slatedb_flush' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT + ) + ); + + // slatedb_close + MethodHandle closeHandle = linker.downcallHandle( + lookup.find("slatedb_close").orElseThrow(() -> new IllegalStateException("Function 'slatedb_close' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT + ) + ); + + // slatedb_free_value + MethodHandle freeValueHandle = linker.downcallHandle( + lookup.find("slatedb_free_value").orElseThrow(() -> new IllegalStateException("Function 'slatedb_free_value' not found")), + FunctionDescriptor.ofVoid(C_SDB_VALUE_LAYOUT) + ); + + // slatedb_scan_prefix_with_options(handle, prefix, prefix_len, scan_options, iterator_ptr) -> CSdbResult + MethodHandle scanPrefixHandle = linker.downcallHandle( + lookup.find("slatedb_scan_prefix_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_scan_prefix_with_options' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + C_SDB_HANDLE_LAYOUT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS + ) + ); + + // slatedb_iterator_next(iter) -> CSdbIteratorNextResult + MethodHandle iteratorNextHandle = linker.downcallHandle( + lookup.find("slatedb_iterator_next").orElseThrow(() -> new IllegalStateException("Function 'slatedb_iterator_next' not found")), + FunctionDescriptor.of( + C_SDB_ITERATOR_NEXT_RESULT_LAYOUT, + ValueLayout.ADDRESS + ) + ); + + // slatedb_iterator_close(iter) -> CSdbResult + MethodHandle iteratorCloseHandle = linker.downcallHandle( + lookup.find("slatedb_iterator_close").orElseThrow(() -> new IllegalStateException("Function 'slatedb_iterator_close' not found")), + FunctionDescriptor.of( + C_SDB_RESULT_LAYOUT, + ValueLayout.ADDRESS + ) + ); + + return new SlateDbNative(arena, dbHandle, getHandle, putHandle, deleteHandle, flushHandle, closeHandle, freeValueHandle, scanPrefixHandle, iteratorNextHandle, iteratorCloseHandle); + } +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/Graph.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/Graph.kt index fc1dae98..dd56cf2f 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/Graph.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/Graph.kt @@ -62,6 +62,8 @@ import com.kakao.actionbase.v2.engine.sql.toRowFlux import com.kakao.actionbase.v2.engine.storage.hbase.HBaseConnections import com.kakao.actionbase.v2.engine.storage.hbase.HBaseOptions import com.kakao.actionbase.v2.engine.storage.jdbc.MetadataTable +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbConnections +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbOptions import com.kakao.actionbase.v2.engine.util.getLogger import com.kakao.actionbase.v2.engine.wal.Wal import com.kakao.actionbase.v2.engine.wal.WalFactory @@ -616,6 +618,10 @@ class Graph( val options = storage.materialize().options as HBaseOptions options.checkConnection() } + StorageType.SLATEDB -> { + val options = storage.materialize().options as SlateDbOptions + options.checkConnection() + } else -> Mono.just(false) } @@ -892,6 +898,7 @@ class Graph( intervalDisposable?.dispose() log.info("Disposed Flux.interval for reloading metastore - {}", intervalDisposable) HBaseConnections.closeConnections().block() + SlateDbConnections.closeConnections().block() DefaultHBaseCluster.INSTANCE.close() } diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/LabelEntity.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/LabelEntity.kt index 4fa10b50..028ed29a 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/LabelEntity.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/LabelEntity.kt @@ -21,12 +21,15 @@ import com.kakao.actionbase.v2.engine.label.hbase.HBaseIndexedLabel import com.kakao.actionbase.v2.engine.label.metastore.JdbcHashLabel import com.kakao.actionbase.v2.engine.label.metastore.LocalBackedJdbcHashLabel import com.kakao.actionbase.v2.engine.label.nil.NilLabel +import com.kakao.actionbase.v2.engine.label.slatedb.SlateDbHashLabel +import com.kakao.actionbase.v2.engine.label.slatedb.SlateDbIndexedLabel import com.kakao.actionbase.v2.engine.service.ddl.LabelCreateRequest import com.kakao.actionbase.v2.engine.sql.RowWithSchema import com.kakao.actionbase.v2.engine.storage.DatastoreStorage import com.kakao.actionbase.v2.engine.storage.hbase.HBaseStorage import com.kakao.actionbase.v2.engine.storage.jdbc.JdbcStorage import com.kakao.actionbase.v2.engine.storage.local.LocalStorage +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbStorage import com.kakao.actionbase.v2.engine.util.getLogger import org.slf4j.Logger @@ -79,6 +82,7 @@ data class LabelEntity( is JdbcStorage -> JdbcHashLabel.create(this, graph, storage, block) is HBaseStorage -> HBaseHashLabel.create(this, graph, storage) is DatastoreStorage -> DatastoreHashLabel.create(this, graph, block) + is SlateDbStorage -> SlateDbHashLabel.create(this, graph, storage) else -> { logger.error( "{} supports only Local, Jdbc, HBase storage types. {} is not supported. Fallback to NilLabel", @@ -99,9 +103,10 @@ data class LabelEntity( when (storage) { is HBaseStorage -> HBaseIndexedLabel.create(this, graph, storage) is DatastoreStorage -> DatastoreIndexedLabel.create(this, graph, block) + is SlateDbStorage -> SlateDbIndexedLabel.create(this, graph, storage) else -> { logger.error( - "{} supports only Jdbc, HBase storage types. {} is not supported. Fallback to NilLabel", + "{} supports only Jdbc, HBase, SlateDb storage types. {} is not supported. Fallback to NilLabel", type, storage, ) diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/StorageEntity.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/StorageEntity.kt index 5b0cdec1..2f689668 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/StorageEntity.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/entity/StorageEntity.kt @@ -11,6 +11,7 @@ import com.kakao.actionbase.v2.engine.storage.hbase.HBaseStorage import com.kakao.actionbase.v2.engine.storage.jdbc.JdbcStorage import com.kakao.actionbase.v2.engine.storage.local.LocalStorage import com.kakao.actionbase.v2.engine.storage.nil.NilStorage +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbStorage import com.kakao.actionbase.v2.engine.util.getLogger import org.slf4j.Logger @@ -41,6 +42,9 @@ data class StorageEntity( StorageType.HBASE -> { HBaseStorage(this) } + StorageType.SLATEDB -> { + SlateDbStorage(this) + } StorageType.DATASTORE -> { DatastoreStorage } diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt index 85a05d66..500a2ee2 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt @@ -17,10 +17,10 @@ class MathOps private constructor( private val arena: Arena, private val addHandle: MethodHandle, ) : AutoCloseable { - - fun add(a: Int, b: Int): Int { - return addHandle.invokeExact(a, b) as Int - } + fun add( + a: Int, + b: Int, + ): Int = addHandle.invokeExact(a, b) as Int override fun close() { arena.close() @@ -32,16 +32,18 @@ class MathOps private constructor( val lookup = SymbolLookup.libraryLookup(libraryPath, arena) val linker = Linker.nativeLinker() - val descriptor = FunctionDescriptor.of( - ValueLayout.JAVA_INT, - ValueLayout.JAVA_INT, - ValueLayout.JAVA_INT, - ) - - val addHandle = linker.downcallHandle( - lookup.find("add").orElseThrow { IllegalStateException("Function 'add' not found") }, - descriptor, - ) + val descriptor = + FunctionDescriptor.of( + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ) + + val addHandle = + linker.downcallHandle( + lookup.find("add").orElseThrow { IllegalStateException("Function 'add' not found") }, + descriptor, + ) return MathOps(arena, addHandle) } diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt new file mode 100644 index 00000000..a18a54f7 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt @@ -0,0 +1,289 @@ +package com.kakao.actionbase.v2.engine.label.slatedb + +import com.kakao.actionbase.v2.core.code.EdgeEncoder +import com.kakao.actionbase.v2.core.code.EncodedKey +import com.kakao.actionbase.v2.core.code.IdEdgeEncoder +import com.kakao.actionbase.v2.core.code.KeyFieldValue +import com.kakao.actionbase.v2.core.code.KeyValue +import com.kakao.actionbase.v2.core.edge.Edge +import com.kakao.actionbase.v2.core.edge.SchemaEdge +import com.kakao.actionbase.v2.core.metadata.Direction +import com.kakao.actionbase.v2.engine.GraphDefaults +import com.kakao.actionbase.v2.engine.edge.decodeByteArray +import com.kakao.actionbase.v2.engine.edge.toRow +import com.kakao.actionbase.v2.engine.entity.LabelEntity +import com.kakao.actionbase.v2.engine.label.AbstractLabel +import com.kakao.actionbase.v2.engine.label.LabelFactory +import com.kakao.actionbase.v2.engine.sql.DataFrame +import com.kakao.actionbase.v2.engine.sql.Row +import com.kakao.actionbase.v2.engine.sql.StatKey +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbStorage +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbTable + +import java.nio.ByteBuffer +import java.util.Arrays + +import reactor.core.publisher.Mono + +open class SlateDbHashLabel( + entity: LabelEntity, + coder: EdgeEncoder, + private val table: Mono, +) : AbstractLabel(entity, coder) { + override fun findHashEdge(keyField: EncodedKey): Mono { + require(keyField.field == null) { "field must be null" } + return table.flatMap { it.get(keyField.key) } + } + + override fun create( + keyField: EncodedKey, + value: ByteArray, + ): Mono> { + require(keyField.field == null) { "field must be null" } + return table + .flatMap { it.put(keyField.key, value) } + .thenReturn(emptyList()) + } + + override fun update( + keyField: EncodedKey, + value: ByteArray, + ): Mono> = create(keyField, value) + + override fun delete(keyField: EncodedKey): Mono> { + require(keyField.field == null) { "field must be null" } + return table + .flatMap { it.delete(keyField.key) } + .thenReturn(emptyList()) + } + + override fun setnx( + keyField: EncodedKey, + value: ByteArray, + ): Mono { + require(keyField.field == null) { "field must be null" } + return table.flatMap { tbl -> + tbl + .get(keyField.key) + .hasElement() + .flatMap { exists -> + if (exists) { + Mono.just(false) + } else { + tbl.put(keyField.key, value).thenReturn(true) + } + } + } + } + + override fun cad( + keyField: EncodedKey, + value: ByteArray, + ): Mono { + require(keyField.field == null) { "field must be null" } + return table.flatMap { tbl -> + tbl + .get(keyField.key) + .flatMap { existingValue -> + if (Arrays.equals(existingValue, value)) { + tbl.delete(keyField.key).thenReturn(1L) + } else { + Mono.just(0L) + } + }.defaultIfEmpty(0L) + } + } + + override fun findLockValue(keyField: EncodedKey): Mono { + require(keyField.field == null) { "field must be null" } + return table.flatMap { it.get(keyField.key) } + } + + override fun incrby( + key: ByteArray, + acc: Long, + ): Mono> = + table.flatMap { tbl -> + tbl + .get(key) + .map { bytes -> ByteBuffer.wrap(bytes).getLong() } + .defaultIfEmpty(0L) + .flatMap { current -> + val newValue = ByteBuffer.allocate(8).putLong(current + acc).array() + tbl.put(key, newValue).thenReturn(emptyList()) + } + } + + override fun scanStorage( + prefix: EncodedKey, + limit: Int, + start: EncodedKey?, + end: EncodedKey?, + ): Mono>> = + table + .flatMap { it.scanPrefix(prefix.key, limit + 1) } + .map { results -> + results + // Filter by start key (exclusive) + .dropWhile { (key, _) -> + start?.key?.let { startKey -> Arrays.compareUnsigned(startKey, key) >= 0 } ?: false + } + // Filter by end key (exclusive) + .dropLastWhile { (key, _) -> + end?.key?.let { endKey -> Arrays.compareUnsigned(endKey, key) < 0 } ?: false + } + .take(limit) + .map { (key, value) -> KeyFieldValue(key, value) } + } + + override fun encodedEdgeToSchemaEdge(keyFieldValue: KeyFieldValue): SchemaEdge = entity.schema.decodeByteArray(keyFieldValue) + + override fun deleteOnLock(keyField: KeyValue): Mono = cad(EncodedKey(keyField.key), keyField.value).map { it > 0 } + + override fun getSelf( + src: List, + stats: Set, + idEdgeEncoder: IdEdgeEncoder, + ): Mono { + val withAll = stats.contains(StatKey.WITH_ALL) + val withEdgeId = withAll || stats.contains(StatKey.EDGE_ID) + + val keysMono = + Mono.just( + src.map { + val edge = Edge(0L, it, it).ensureType(entity.schema) + coder.encodeHashEdgeKey(edge, entity.id) + }, + ) + + return keysMono + .flatMap { keys -> + table.flatMap { tbl -> + Mono.zip( + keys.map { key -> tbl.get(key.key).map { key to it } }, + ) { results -> + results + .filterIsInstance, ByteArray>>() + .mapNotNull { (key, value) -> + try { + encodedEdgeToSchemaEdge(KeyFieldValue(key.key, value)) + } catch (e: Exception) { + null + } + }.filter { withAll || it.isActive } + .map { + if (withEdgeId) { + it.toRow(withAll, idEdgeEncoder) + } else { + it.toRow(withAll, null) + } + } + } + } + }.map { rows -> + DataFrame( + rows, + if (withAll) { + entity.schema.allStructType + } else if (withEdgeId) { + entity.schema.edgeIdStructType + } else { + entity.schema.structType + }, + ) + }.defaultIfEmpty(DataFrame.empty(entity.schema.allStructType)) + } + + override fun get( + src: Any, + tgt: List, + dir: Direction, + stats: Set, + idEdgeEncoder: IdEdgeEncoder, + ): Mono { + val withAll = stats.contains(StatKey.WITH_ALL) + val withEdgeId = withAll || stats.contains(StatKey.EDGE_ID) + + val keys = + tgt.map { + val edge = Edge(0L, src, it).ensureType(entity.schema) + coder.encodeHashEdgeKey(edge, entity.id) + } + + return table + .flatMap { tbl -> + if (keys.isEmpty()) { + Mono.just(emptyList()) + } else { + Mono.zip( + keys.map { key -> tbl.get(key.key).map { key to it }.defaultIfEmpty(key to ByteArray(0)) }, + ) { results -> + results + .filterIsInstance, ByteArray>>() + .filter { it.second.isNotEmpty() } + .mapNotNull { (key, value) -> + try { + encodedEdgeToSchemaEdge(KeyFieldValue(key.key, value)) + } catch (e: Exception) { + null + } + }.filter { withAll || it.isActive } + .map { + if (withEdgeId) { + it.toRow(withAll, idEdgeEncoder, isMultiEdge) + } else { + it.toRow(withAll, null, isMultiEdge) + } + } + } + } + }.map { rows -> + DataFrame( + rows, + if (withAll) { + entity.schema.allStructType + } else if (withEdgeId) { + entity.schema.edgeIdStructType + } else { + entity.schema.structType + }, + ) + }.defaultIfEmpty(DataFrame.empty(entity.schema.allStructType)) + } + + override fun getCountRows( + srcAndKeys: List>, + dir: Direction, + ): Mono> = + table.flatMap { tbl -> + if (srcAndKeys.isEmpty()) { + Mono.just(emptyList()) + } else { + Mono.zip( + srcAndKeys.map { (src, key) -> + tbl + .get(key) + .map { bytes -> ByteBuffer.wrap(bytes).getLong() } + .defaultIfEmpty(0L) + .map { count -> Row(arrayOf(src, count, dir)) } + }, + ) { results -> results.filterIsInstance() } + } + } + + companion object : LabelFactory { + override fun create( + entity: LabelEntity, + graph: GraphDefaults, + storage: SlateDbStorage, + block: SlateDbHashLabel.() -> Unit, + ): SlateDbHashLabel { + val table = storage.options.getTable() + return SlateDbHashLabel( + entity = entity, + coder = graph.edgeEncoderFactory.bytesKeyValueEncoder, + table = table, + ).apply(block) + } + } +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabel.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabel.kt new file mode 100644 index 00000000..953e3431 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabel.kt @@ -0,0 +1,69 @@ +package com.kakao.actionbase.v2.engine.label.slatedb + +import com.kakao.actionbase.v2.core.code.EdgeEncoder +import com.kakao.actionbase.v2.core.code.IdEdgeEncoder +import com.kakao.actionbase.v2.core.code.Index +import com.kakao.actionbase.v2.engine.GraphDefaults +import com.kakao.actionbase.v2.engine.cdc.CdcContext +import com.kakao.actionbase.v2.engine.entity.LabelEntity +import com.kakao.actionbase.v2.engine.label.AbstractLabel +import com.kakao.actionbase.v2.engine.label.LabelFactory +import com.kakao.actionbase.v2.engine.label.mixin.IndexedLabelMixin +import com.kakao.actionbase.v2.engine.sql.DataFrame +import com.kakao.actionbase.v2.engine.sql.ScanFilter +import com.kakao.actionbase.v2.engine.sql.StatKey +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbStorage +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbTable + +import reactor.core.publisher.Mono + +/** + * Manages IndexedEdgeEncoder in SlateDB + */ +class SlateDbIndexedLabel( + entity: LabelEntity, + coder: EdgeEncoder, + override val indices: List, + override val indexNameToIndex: Map, + table: Mono, +) : SlateDbHashLabel( + entity = entity, + coder = coder, + table = table, + ), + IndexedLabelMixin { + override val self: AbstractLabel = this + + override fun finalizeEdgeMutationUnderLock(context: CdcContext): Mono> = mutateIndexedEdges(context) + + override fun scan( + scanFilter: ScanFilter, + stats: Set, + idEdgeEncoder: IdEdgeEncoder, + ): Mono = + scanIndexedEdges( + scanFilter, + stats, + idEdgeEncoder, + ) + + companion object : LabelFactory { + override fun create( + entity: LabelEntity, + graph: GraphDefaults, + storage: SlateDbStorage, + block: SlateDbIndexedLabel.() -> Unit, + ): SlateDbIndexedLabel { + val table = storage.options.getTable() + val indices: List = entity.indices + val indexNameToId = indices.associateBy { it.name } + return SlateDbIndexedLabel( + entity = entity, + coder = graph.edgeEncoderFactory.bytesKeyValueEncoder, + indices = indices, + indexNameToIndex = indexNameToId, + table = table, + ) + } + } +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metadata/StorageType.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metadata/StorageType.kt index 55c968f5..f278f38b 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metadata/StorageType.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metadata/StorageType.kt @@ -5,6 +5,7 @@ enum class StorageType { LOCAL, JDBC, HBASE, + SLATEDB, DATASTORE, ; diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt new file mode 100644 index 00000000..8bf136a5 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt @@ -0,0 +1,61 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import com.kakao.actionbase.v2.engine.util.getLogger + +import java.nio.file.Path +import java.util.concurrent.ConcurrentHashMap + +import reactor.core.publisher.Mono +import reactor.core.scheduler.Schedulers + +object SlateDbConnections { + private val logger = getLogger() + + private val connections: ConcurrentHashMap> = ConcurrentHashMap() + + fun getConnection( + dbPath: String, + url: String, + libraryPath: Path, + ): Mono { + val cacheKey = getCacheKey(dbPath, url) + + return connections.computeIfAbsent(cacheKey) { key -> + Mono + .fromCallable { + val native = SlateDbNative.open(dbPath, url, libraryPath) + SlateDbTable.create(native) + }.subscribeOn(Schedulers.boundedElastic()) + .doOnSuccess { + logger.info("Successfully opened SlateDB connection for cacheKey: {}", key) + }.doOnError { error -> + logger.error("Failed to open SlateDB connection for cacheKey: {}", key, error) + connections.remove(key) + }.cache() + } + } + + private fun getCacheKey( + dbPath: String, + url: String, + ): String = "$url/$dbPath" + + fun closeConnections(): Mono { + val closeMonos = + connections.entries.map { (key, tableMono) -> + tableMono + .flatMap { table -> + Mono + .fromRunnable { + try { + table.close() + logger.info("Closed SlateDB connection for cacheKey: {}", key) + } catch (e: Exception) { + logger.error("Error closing SlateDB connection for cacheKey: {}", key, e) + } + }.subscribeOn(Schedulers.boundedElastic()) + } + } + return Mono.`when`(closeMonos).doFinally { connections.clear() } + } +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt new file mode 100644 index 00000000..8d1f32d2 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt @@ -0,0 +1,28 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import java.nio.file.Path + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties + +import reactor.core.publisher.Mono + +@JsonIgnoreProperties(ignoreUnknown = true) +data class SlateDbOptions( + val path: String = "data", + val url: String = "", + val libraryPath: String = "", +) { + fun checkConnection(): Mono = + if (url.isBlank() || libraryPath.isBlank()) { + Mono.just(false) + } else { + Mono.just(true) + } + + fun getTable(): Mono = + SlateDbConnections.getConnection( + dbPath = path, + url = url, + libraryPath = Path.of(libraryPath), + ) +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorage.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorage.kt new file mode 100644 index 00000000..ea6546b4 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorage.kt @@ -0,0 +1,11 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import com.kakao.actionbase.v2.engine.entity.StorageEntity +import com.kakao.actionbase.v2.engine.storage.Storage +import com.kakao.actionbase.v2.engine.storage.Storage.Companion.parseOptions + +class SlateDbStorage( + override val entity: StorageEntity, +) : Storage { + override val options: SlateDbOptions = parseOptions(entity.conf) +} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt new file mode 100644 index 00000000..959e5860 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt @@ -0,0 +1,72 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import reactor.core.publisher.Mono +import reactor.core.scheduler.Schedulers + +interface SlateDbTable : AutoCloseable { + fun get(key: ByteArray): Mono + + fun put( + key: ByteArray, + value: ByteArray, + ): Mono + + fun delete(key: ByteArray): Mono + + fun flush(): Mono + + fun scanPrefix( + prefix: ByteArray, + limit: Int, + ): Mono>> + + companion object { + fun create(native: SlateDbNative): SlateDbTable = SlateDbTableImpl(native) + } +} + +internal class SlateDbTableImpl( + private val native: SlateDbNative, +) : SlateDbTable { + override fun get(key: ByteArray): Mono = + Mono + .fromCallable { native.get(key) } + .flatMap { Mono.justOrEmpty(it) } + .subscribeOn(Schedulers.boundedElastic()) + + override fun put( + key: ByteArray, + value: ByteArray, + ): Mono = + Mono + .fromCallable { native.put(key, value) } + .subscribeOn(Schedulers.boundedElastic()) + .then() + + override fun delete(key: ByteArray): Mono = + Mono + .fromCallable { native.delete(key) } + .subscribeOn(Schedulers.boundedElastic()) + .then() + + override fun flush(): Mono = + Mono + .fromCallable { native.flush() } + .subscribeOn(Schedulers.boundedElastic()) + .then() + + override fun scanPrefix( + prefix: ByteArray, + limit: Int, + ): Mono>> = + Mono + .fromCallable { + native.scanPrefix(prefix, limit).map { entry -> + entry.key to entry.value + } + }.subscribeOn(Schedulers.boundedElastic()) + + override fun close() { + native.close() + } +} diff --git a/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java b/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java new file mode 100644 index 00000000..d4faa054 --- /dev/null +++ b/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java @@ -0,0 +1,113 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +class SlateDbNativeTest { + + @TempDir + Path tempDir; + + private SlateDbNative db; + + private static Path findLibraryPath() { + // Find project root by looking for settings.gradle.kts + Path dir = Path.of(System.getProperty("user.dir")); + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.getParent() != null) { + dir = dir.getParent(); + } + return dir.resolve("native/lib/libslatedb_c.dylib"); + } + + @BeforeEach + void setUp() throws Throwable { + // SlateDB uses object_store: + // - url: object store root (file:// for local filesystem) + // - path: prefix/directory within the store + String fileUrl = "file://" + tempDir.toAbsolutePath(); + String dbPath = "data"; // prefix within the object store + db = SlateDbNative.open(dbPath, fileUrl, findLibraryPath()); + } + + @AfterEach + void tearDown() throws Exception { + if (db != null) { + db.close(); + } + } + + @Test + void putAndGet() throws Throwable { + // Given + byte[] key = "hello".getBytes(StandardCharsets.UTF_8); + byte[] value = "world".getBytes(StandardCharsets.UTF_8); + + // When + db.put(key, value); + byte[] result = db.get(key); + + // Then + assertNotNull(result); + assertEquals("world", new String(result, StandardCharsets.UTF_8)); + } + + @Test + void getNonExistentKeyReturnsNull() throws Throwable { + // When + byte[] result = db.get("nonexistent".getBytes(StandardCharsets.UTF_8)); + + // Then + assertNull(result); + } + + @Test + void deleteRemovesKey() throws Throwable { + // Given + byte[] key = "to-delete".getBytes(StandardCharsets.UTF_8); + db.put(key, "value".getBytes(StandardCharsets.UTF_8)); + + // When + db.delete(key); + byte[] result = db.get(key); + + // Then + assertNull(result); + } + + @Test + void overwriteExistingKey() throws Throwable { + // Given + byte[] key = "key".getBytes(StandardCharsets.UTF_8); + db.put(key, "value1".getBytes(StandardCharsets.UTF_8)); + + // When + db.put(key, "value2".getBytes(StandardCharsets.UTF_8)); + byte[] result = db.get(key); + + // Then + assertNotNull(result); + assertEquals("value2", new String(result, StandardCharsets.UTF_8)); + } + + @Test + void binaryData() throws Throwable { + // Given + byte[] key = new byte[]{0x00, 0x01, 0x02, (byte) 0xFF}; + byte[] value = new byte[]{(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}; + + // When + db.put(key, value); + byte[] result = db.get(key); + + // Then + assertNotNull(result); + assertArrayEquals(value, result); + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt new file mode 100644 index 00000000..6a8ee022 --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt @@ -0,0 +1,241 @@ +package com.kakao.actionbase.v2.engine.label.slatedb + +import com.kakao.actionbase.v2.core.metadata.DirectionType +import com.kakao.actionbase.v2.core.metadata.EdgeOperation +import com.kakao.actionbase.v2.core.metadata.LabelType +import com.kakao.actionbase.v2.core.types.DataType +import com.kakao.actionbase.v2.core.types.EdgeSchema +import com.kakao.actionbase.v2.core.types.Field +import com.kakao.actionbase.v2.core.types.VertexField +import com.kakao.actionbase.v2.core.types.VertexType +import com.kakao.actionbase.v2.engine.Graph +import com.kakao.actionbase.v2.engine.GraphConfig +import com.kakao.actionbase.v2.engine.client.kafka.impl.DefaultKafkaClientFactory +import com.kakao.actionbase.v2.engine.client.web.impl.DefaultWebClientFactory +import com.kakao.actionbase.v2.engine.entity.EntityName +import com.kakao.actionbase.v2.engine.entity.LabelEntity +import com.kakao.actionbase.v2.engine.label.EdgeOperationStatus +import com.kakao.actionbase.v2.engine.metadata.StorageType +import com.kakao.actionbase.v2.engine.service.ddl.DdlStatus +import com.kakao.actionbase.v2.engine.service.ddl.ServiceCreateRequest +import com.kakao.actionbase.v2.engine.service.ddl.StorageCreateRequest +import com.kakao.actionbase.v2.engine.test.cdc.InMemoryCdcFactory +import com.kakao.actionbase.v2.engine.test.wal.InMemoryWalFactory + +import java.nio.file.Path +import java.util.UUID + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper + +import io.kotest.matchers.shouldBe +import reactor.kotlin.test.test + +class SlateDbHashLabelTest { + @TempDir + lateinit var tempDir: Path + + private lateinit var graph: Graph + private val serviceName = "slatedb_test_service" + private val storageName = "slatedb_storage" + private val labelName = "slatedb_label" + + private fun findLibraryPath(): Path { + var dir = Path.of(System.getProperty("user.dir")) + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { + dir = dir.parent + } + return dir.resolve("native/lib/libslatedb_c.dylib") + } + + private fun createGraph(): Graph { + val config = + GraphConfig + .Builder() + .withMetastoreUrl("jdbc:h2:mem:${UUID.randomUUID()};DB_CLOSE_DELAY=-1;MODE=MYSQL") + .build() + return Graph.create(config, InMemoryWalFactory, InMemoryCdcFactory, DefaultKafkaClientFactory, DefaultWebClientFactory) + } + + @BeforeEach + fun setUp() { + // Skip test if native library not found + assumeTrue(findLibraryPath().toFile().exists(), "SlateDB native library not found") + + graph = createGraph() + graph.updateAllMetadata().block() + + // Create service + graph.serviceDdl + .create(EntityName.fromOrigin(serviceName), ServiceCreateRequest(desc = "test service")) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + // Create SlateDB storage + val conf = + jacksonObjectMapper().createObjectNode().apply { + put("path", "test-data") + put("url", "file://${tempDir.toAbsolutePath()}") + put("libraryPath", findLibraryPath().toString()) + } + + graph.storageDdl + .create(EntityName.fromOrigin(storageName), StorageCreateRequest(desc = "slatedb storage", type = StorageType.SLATEDB, conf = conf)) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + // Create label with SlateDB storage + val labelEntity = + LabelEntity( + active = true, + name = EntityName(serviceName, labelName), + desc = "test slatedb label", + type = LabelType.HASH, + schema = + EdgeSchema( + VertexField(VertexType.LONG), + VertexField(VertexType.LONG), + listOf( + Field("score", DataType.LONG, false), + Field("memo", DataType.STRING, true), + ), + ), + dirType = DirectionType.OUT, + storage = storageName, + ) + + graph.labelDdl + .create(labelEntity.name, labelEntity.toCreateRequest()) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + graph.updateAllMetadata().block() + } + + @AfterEach + fun tearDown() { + graph.close() + } + + @Test + fun `insert and get edge`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val edge = + com.kakao.actionbase.v2.core.edge.Edge( + System.currentTimeMillis(), + 100L, + 200L, + mapOf("score" to 42L, "memo" to "hello"), + ) + + // Insert + label + .mutate(edge.toTraceEdge(), EdgeOperation.INSERT) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.CREATED + }.verifyComplete() + + // Get + graph + .queryGet( + EntityName(serviceName, labelName), + 100L, + 200L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + // tgt is at index 1 in the schema (src, tgt, ts, ...) + val row = df.toRowWithSchema().first() + row.getLong("tgt") shouldBe 200L + }.verifyComplete() + } + + @Test + fun `delete edge`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val edge = + com.kakao.actionbase.v2.core.edge.Edge( + System.currentTimeMillis(), + 101L, + 201L, + mapOf("score" to 100L), + ) + + // Insert + label.mutate(edge.toTraceEdge(), EdgeOperation.INSERT).block() + + // Delete + label + .mutate(edge.toTraceEdge(), EdgeOperation.DELETE) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.DELETED + }.verifyComplete() + + // Verify deleted (should return empty) + graph + .queryGet( + EntityName(serviceName, labelName), + 101L, + 201L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 0 + }.verifyComplete() + } + + @Test + fun `update edge`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val ts = System.currentTimeMillis() + + val edge1 = + com.kakao.actionbase.v2.core.edge.Edge( + ts, + 102L, + 202L, + mapOf("score" to 50L, "memo" to "original"), + ) + + val edge2 = + com.kakao.actionbase.v2.core.edge.Edge( + ts + 1, + 102L, + 202L, + mapOf("score" to 100L, "memo" to "updated"), + ) + + // Insert + label.mutate(edge1.toTraceEdge(), EdgeOperation.INSERT).block() + + // Update + label + .mutate(edge2.toTraceEdge(), EdgeOperation.INSERT) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.UPDATED + }.verifyComplete() + + // Verify updated + graph + .queryGet( + EntityName(serviceName, labelName), + 102L, + 202L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + val row = df.toRowWithSchema().first() + row.getString("memo") shouldBe "updated" + }.verifyComplete() + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt new file mode 100644 index 00000000..2df71560 --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt @@ -0,0 +1,294 @@ +package com.kakao.actionbase.v2.engine.label.slatedb + +import com.kakao.actionbase.v2.core.code.Index +import com.kakao.actionbase.v2.core.code.hbase.Order +import com.kakao.actionbase.v2.core.metadata.DirectionType +import com.kakao.actionbase.v2.core.metadata.EdgeOperation +import com.kakao.actionbase.v2.core.metadata.LabelType +import com.kakao.actionbase.v2.core.types.DataType +import com.kakao.actionbase.v2.core.types.EdgeSchema +import com.kakao.actionbase.v2.core.types.Field +import com.kakao.actionbase.v2.core.types.VertexField +import com.kakao.actionbase.v2.core.types.VertexType +import com.kakao.actionbase.v2.engine.Graph +import com.kakao.actionbase.v2.engine.GraphConfig +import com.kakao.actionbase.v2.engine.client.kafka.impl.DefaultKafkaClientFactory +import com.kakao.actionbase.v2.engine.client.web.impl.DefaultWebClientFactory +import com.kakao.actionbase.v2.engine.entity.EntityName +import com.kakao.actionbase.v2.engine.entity.LabelEntity +import com.kakao.actionbase.v2.engine.label.EdgeOperationStatus +import com.kakao.actionbase.v2.engine.metadata.StorageType +import com.kakao.actionbase.v2.engine.service.ddl.DdlStatus +import com.kakao.actionbase.v2.engine.service.ddl.ServiceCreateRequest +import com.kakao.actionbase.v2.engine.service.ddl.StorageCreateRequest +import com.kakao.actionbase.v2.engine.test.cdc.InMemoryCdcFactory +import com.kakao.actionbase.v2.engine.test.wal.InMemoryWalFactory + +import java.nio.file.Path +import java.util.UUID + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper + +import io.kotest.matchers.shouldBe +import reactor.kotlin.test.test + +class SlateDbIndexedLabelTest { + @TempDir + lateinit var tempDir: Path + + private lateinit var graph: Graph + private val serviceName = "slatedb_indexed_test_service" + private val storageName = "slatedb_indexed_storage" + private val labelName = "slatedb_indexed_label" + + private fun findLibraryPath(): Path { + var dir = Path.of(System.getProperty("user.dir")) + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { + dir = dir.parent + } + return dir.resolve("native/lib/libslatedb_c.dylib") + } + + private fun createGraph(): Graph { + val config = + GraphConfig + .Builder() + .withMetastoreUrl("jdbc:h2:mem:${UUID.randomUUID()};DB_CLOSE_DELAY=-1;MODE=MYSQL") + .build() + return Graph.create(config, InMemoryWalFactory, InMemoryCdcFactory, DefaultKafkaClientFactory, DefaultWebClientFactory) + } + + @BeforeEach + fun setUp() { + // Skip test if native library not found + assumeTrue(findLibraryPath().toFile().exists(), "SlateDB native library not found") + + graph = createGraph() + graph.updateAllMetadata().block() + + // Create service + graph.serviceDdl + .create(EntityName.fromOrigin(serviceName), ServiceCreateRequest(desc = "test service")) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + // Create SlateDB storage + val conf = + jacksonObjectMapper().createObjectNode().apply { + put("path", "test-data") + put("url", "file://${tempDir.toAbsolutePath()}") + put("libraryPath", findLibraryPath().toString()) + } + + graph.storageDdl + .create(EntityName.fromOrigin(storageName), StorageCreateRequest(desc = "slatedb indexed storage", type = StorageType.SLATEDB, conf = conf)) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + // Create INDEXED label with SlateDB storage + val labelEntity = + LabelEntity( + active = true, + name = EntityName(serviceName, labelName), + desc = "test slatedb indexed label", + type = LabelType.INDEXED, + schema = + EdgeSchema( + VertexField(VertexType.LONG), + VertexField(VertexType.LONG), + listOf( + Field("score", DataType.LONG, false), + Field("memo", DataType.STRING, true), + ), + ), + dirType = DirectionType.OUT, + storage = storageName, + indices = + listOf( + Index( + "score_desc", + listOf(Index.Field("score", Order.DESC)), + "Score descending index", + ), + ), + ) + + graph.labelDdl + .create(labelEntity.name, labelEntity.toCreateRequest()) + .test() + .assertNext { it.status shouldBe DdlStatus.Status.CREATED } + .verifyComplete() + + graph.updateAllMetadata().block() + } + + @AfterEach + fun tearDown() { + graph.close() + } + + @Test + fun `insert and get edge with index`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val edge = + com.kakao.actionbase.v2.core.edge.Edge( + System.currentTimeMillis(), + 100L, + 200L, + mapOf("score" to 42L, "memo" to "hello"), + ) + + // Insert + label + .mutate(edge.toTraceEdge(), EdgeOperation.INSERT) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.CREATED + }.verifyComplete() + + // Get + graph + .queryGet( + EntityName(serviceName, labelName), + 100L, + 200L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + val row = df.toRowWithSchema().first() + row.getLong("tgt") shouldBe 200L + row.getLong("score") shouldBe 42L + }.verifyComplete() + } + + @Test + fun `insert multiple edges and verify index order`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val ts = System.currentTimeMillis() + + // Insert edges with different scores + val edges = + listOf( + com.kakao.actionbase.v2.core.edge.Edge(ts, 100L, 201L, mapOf("score" to 10L, "memo" to "low")), + com.kakao.actionbase.v2.core.edge.Edge(ts + 1, 100L, 202L, mapOf("score" to 50L, "memo" to "medium")), + com.kakao.actionbase.v2.core.edge.Edge(ts + 2, 100L, 203L, mapOf("score" to 100L, "memo" to "high")), + ) + + edges.forEach { edge -> + label.mutate(edge.toTraceEdge(), EdgeOperation.INSERT).block() + } + + // Verify all edges exist + edges.forEach { edge -> + graph + .queryGet( + EntityName(serviceName, labelName), + edge.src, + edge.tgt, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + }.verifyComplete() + } + } + + @Test + fun `delete edge removes from index`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val edge = + com.kakao.actionbase.v2.core.edge.Edge( + System.currentTimeMillis(), + 101L, + 201L, + mapOf("score" to 100L), + ) + + // Insert + label.mutate(edge.toTraceEdge(), EdgeOperation.INSERT).block() + + // Verify exists + graph + .queryGet( + EntityName(serviceName, labelName), + 101L, + 201L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + }.verifyComplete() + + // Delete + label + .mutate(edge.toTraceEdge(), EdgeOperation.DELETE) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.DELETED + }.verifyComplete() + + // Verify deleted + graph + .queryGet( + EntityName(serviceName, labelName), + 101L, + 201L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 0 + }.verifyComplete() + } + + @Test + fun `update edge updates index`() { + val label = graph.getLabel(EntityName(serviceName, labelName)) + val ts = System.currentTimeMillis() + + val edge1 = + com.kakao.actionbase.v2.core.edge.Edge( + ts, + 102L, + 202L, + mapOf("score" to 50L, "memo" to "original"), + ) + + val edge2 = + com.kakao.actionbase.v2.core.edge.Edge( + ts + 1, + 102L, + 202L, + mapOf("score" to 100L, "memo" to "updated"), + ) + + // Insert + label.mutate(edge1.toTraceEdge(), EdgeOperation.INSERT).block() + + // Update + label + .mutate(edge2.toTraceEdge(), EdgeOperation.INSERT) + .test() + .assertNext { context -> + context.status shouldBe EdgeOperationStatus.UPDATED + }.verifyComplete() + + // Verify updated + graph + .queryGet( + EntityName(serviceName, labelName), + 102L, + 202L, + ).test() + .assertNext { df -> + df.rows.size shouldBe 1 + val row = df.toRowWithSchema().first() + row.getLong("score") shouldBe 100L + row.getString("memo") shouldBe "updated" + }.verifyComplete() + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt new file mode 100644 index 00000000..d4d14e6b --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt @@ -0,0 +1,65 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import com.kakao.actionbase.v2.engine.Graph +import com.kakao.actionbase.v2.engine.GraphConfig +import com.kakao.actionbase.v2.engine.client.kafka.impl.DefaultKafkaClientFactory +import com.kakao.actionbase.v2.engine.client.web.impl.DefaultWebClientFactory +import com.kakao.actionbase.v2.engine.entity.EntityName +import com.kakao.actionbase.v2.engine.metadata.StorageType +import com.kakao.actionbase.v2.engine.test.GraphFixtures +import com.kakao.actionbase.v2.engine.test.cdc.InMemoryCdcFactory +import com.kakao.actionbase.v2.engine.test.wal.InMemoryWalFactory + +import java.util.UUID + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper + +import reactor.kotlin.test.test + +class SlateDbStorageTest { + private lateinit var graph: Graph + + private fun createGraph(): Graph { + val config = + GraphConfig + .Builder() + .withMetastoreUrl("jdbc:h2:mem:${UUID.randomUUID()};DB_CLOSE_DELAY=-1;MODE=MYSQL") + .build() + return Graph.create(config, InMemoryWalFactory, InMemoryCdcFactory, DefaultKafkaClientFactory, DefaultWebClientFactory) + } + + @BeforeEach + fun setUp() { + graph = createGraph() + graph.updateAllMetadata().block() + } + + @AfterEach + fun tearDown() { + graph.close() + } + + @Test + fun `create SlateDB storage entity`() { + val conf = + jacksonObjectMapper().createObjectNode().apply { + put("path", "test-data") + put("url", "file:///tmp/slatedb-test") + put("libraryPath", "native/lib/libslatedb_c.dylib") + } + + GraphFixtures.createStorage(graph, "slatedb_test", StorageType.SLATEDB, conf) + + graph.storageDdl + .getSingle(EntityName.fromOrigin("slatedb_test")) + .test() + .assertNext { storage -> + assert(storage.type == StorageType.SLATEDB) + assert(storage.active) + }.verifyComplete() + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt new file mode 100644 index 00000000..4e27ca35 --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt @@ -0,0 +1,89 @@ +package com.kakao.actionbase.v2.engine.storage.slatedb + +import java.nio.charset.StandardCharsets +import java.nio.file.Path + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import reactor.test.StepVerifier + +class SlateDbTableTest { + @TempDir + lateinit var tempDir: Path + + private lateinit var table: SlateDbTable + + private fun findLibraryPath(): Path { + var dir = Path.of(System.getProperty("user.dir")) + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { + dir = dir.parent + } + return dir.resolve("native/lib/libslatedb_c.dylib") + } + + @BeforeEach + fun setUp() { + val fileUrl = "file://${tempDir.toAbsolutePath()}" + val dbPath = "data" + val native = SlateDbNative.open(dbPath, fileUrl, findLibraryPath()) + table = SlateDbTable.create(native) + } + + @AfterEach + fun tearDown() { + table.close() + } + + @Test + fun `put and get with reactive API`() { + val key = "hello".toByteArray(StandardCharsets.UTF_8) + val value = "world".toByteArray(StandardCharsets.UTF_8) + + StepVerifier + .create( + table.put(key, value).then(table.get(key)), + ).expectNextMatches { it != null && String(it, StandardCharsets.UTF_8) == "world" } + .verifyComplete() + } + + @Test + fun `get non-existent key returns empty`() { + val key = "nonexistent".toByteArray(StandardCharsets.UTF_8) + + StepVerifier + .create(table.get(key)) + .verifyComplete() + } + + @Test + fun `delete removes key`() { + val key = "to-delete".toByteArray(StandardCharsets.UTF_8) + val value = "value".toByteArray(StandardCharsets.UTF_8) + + StepVerifier + .create( + table + .put(key, value) + .then(table.delete(key)) + .then(table.get(key)), + ).verifyComplete() + } + + @Test + fun `flush persists data`() { + val key = "flush-key".toByteArray(StandardCharsets.UTF_8) + val value = "flush-value".toByteArray(StandardCharsets.UTF_8) + + StepVerifier + .create( + table + .put(key, value) + .then(table.flush()) + .then(table.get(key)), + ).expectNextMatches { it != null && String(it, StandardCharsets.UTF_8) == "flush-value" } + .verifyComplete() + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/test/GraphFixtures.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/test/GraphFixtures.kt index 0bd7c839..ee963bea 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/test/GraphFixtures.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/test/GraphFixtures.kt @@ -48,7 +48,6 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotest.matchers.shouldBe -import reactor.blockhound.BlockHound import reactor.core.publisher.Flux import reactor.kotlin.core.publisher.toFlux import reactor.kotlin.test.test @@ -221,11 +220,6 @@ object GraphFixtures { } fun create(withTestData: Boolean = true): Graph { - BlockHound - .builder() - .allowBlockingCallsInside("org.apache.hadoop.hbase.client.mock.MockHTable", "mutateRow") - .install() - val config = GraphConfig .Builder() diff --git a/native/build-slatedb.sh b/native/build-slatedb.sh new file mode 100755 index 00000000..1df82b1e --- /dev/null +++ b/native/build-slatedb.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SLATEDB_DIR="$SCRIPT_DIR/slatedb" +LIB_DIR="$SCRIPT_DIR/lib" + +# Check for rustup (needed for nightly) +if ! command -v rustup &> /dev/null; then + echo "Error: rustup is required (slatedb uses nightly Rust features)" + echo "" + echo "Install rustup:" + echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" + echo "" + echo "Then install nightly:" + echo " rustup install nightly" + exit 1 +fi + +# Ensure nightly is installed +if ! rustup run nightly rustc --version &> /dev/null; then + echo "Installing Rust nightly..." + rustup install nightly +fi + +# Clone if not exists +if [ ! -d "$SLATEDB_DIR" ]; then + echo "Cloning slatedb..." + git clone --depth 1 https://github.com/slatedb/slatedb.git "$SLATEDB_DIR" +fi + +# Build slatedb-c with nightly +echo "Building slatedb-c (nightly)..." +cd "$SLATEDB_DIR" +cargo +nightly build --release -p slatedb-c + +# Copy library +mkdir -p "$LIB_DIR" + +OS="$(uname -s)" +case "$OS" in + Darwin) + cp target/release/libslatedb_c.dylib "$LIB_DIR/" + echo "Built: $LIB_DIR/libslatedb_c.dylib" + ;; + Linux) + cp target/release/libslatedb_c.so "$LIB_DIR/" + echo "Built: $LIB_DIR/libslatedb_c.so" + ;; + *) + echo "Unsupported OS: $OS" + exit 1 + ;; +esac + +echo "Done." diff --git a/server/build.gradle.kts b/server/build.gradle.kts index b2a7a26d..62905bd0 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -1,6 +1,5 @@ import actionbase.BuildParameter import actionbase.dependencies.Dependencies -import org.gradle.jvm.toolchain.JavaLanguageVersion plugins { id("actionbase.kotlin-conventions") @@ -84,6 +83,14 @@ springBoot { tasks.named("bootRun") { jvmArgs("--enable-native-access=ALL-UNNAMED") + // Use Java 25 for SlateDB FFI support + javaLauncher.set( + javaToolchains.launcherFor { + languageVersion.set(JavaLanguageVersion.of(25)) + } + ) + // Set working directory to project root for SlateDB native library path resolution + workingDir = rootProject.projectDir } jib { diff --git a/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt b/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt index 1532cc5f..01d7482d 100644 --- a/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt +++ b/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt @@ -1,13 +1,16 @@ package com.kakao.actionbase.server.ffi import com.kakao.actionbase.v2.engine.ffi.MathOps -import jakarta.annotation.PreDestroy + +import java.nio.file.Path + import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController -import java.nio.file.Path + +import jakarta.annotation.PreDestroy @RestController @ConditionalOnProperty(name = ["ffi.enabled"], havingValue = "true", matchIfMissing = false) @@ -20,9 +23,7 @@ class FfiController( fun add( @RequestParam a: Int, @RequestParam b: Int, - ): Int { - return mathOps.add(a, b) - } + ): Int = mathOps.add(a, b) @PreDestroy fun cleanup() { diff --git a/server/src/main/resources/application-slatedb.yaml b/server/src/main/resources/application-slatedb.yaml new file mode 100644 index 00000000..223ddda4 --- /dev/null +++ b/server/src/main/resources/application-slatedb.yaml @@ -0,0 +1,18 @@ +# SlateDB storage profile +# Usage: --spring.profiles.active=slatedb +# +# Prerequisites: +# 1. Build slatedb-c: ./native/build-slatedb.sh +# 2. Set library path via environment variable or update libraryPath below + +actionbase: + tenant: ab-slatedb + +kc: + graph: + defaultStorage: + type: SLATEDB + conf: + path: data + url: file://${SLATEDB_STORAGE_PATH:/tmp/actionbase-slatedb} + libraryPath: ${SLATEDB_LIBRARY_PATH:native/lib/libslatedb_c.dylib} From d289baa9a653c4a698cd522493b28420cc07f165 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 4 Feb 2026 15:51:24 +0900 Subject: [PATCH 02/11] feat(engine): migrate to upstream slatedb-java bindings (#167) --- .gitignore | 5 +- engine/build.gradle.kts | 3 + .../storage/slatedb/SlateDbException.java | 14 - .../engine/storage/slatedb/SlateDbNative.java | 432 ------------------ .../kakao/actionbase/v2/engine/ffi/MathOps.kt | 51 --- .../engine/label/slatedb/SlateDbHashLabel.kt | 3 +- .../storage/slatedb/SlateDbConnections.kt | 19 +- .../engine/storage/slatedb/SlateDbOptions.kt | 4 +- .../v2/engine/storage/slatedb/SlateDbTable.kt | 71 ++- .../storage/slatedb/SlateDbNativeTest.java | 113 ----- .../label/slatedb/SlateDbIndexedLabelTest.kt | 9 +- .../storage/slatedb/SlateDbTableTest.kt | 105 ++++- native/build-slatedb.sh | 125 +++-- native/src/c/math_ops.c | 13 - 14 files changed, 279 insertions(+), 688 deletions(-) delete mode 100644 engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java delete mode 100644 engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java delete mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt delete mode 100644 engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java delete mode 100644 native/src/c/math_ops.c diff --git a/.gitignore b/.gitignore index 6ec7ee05..f3b8cb5b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,8 +81,7 @@ CLAUDE.md !**/.cursor/rules/** engine/target/ -# Native library build artifacts -native/src/c/*.dylib -native/src/c/*.so +# Native library build artifacts (slatedb) native/slatedb/ native/lib/ +!native/lib/.gitkeep diff --git a/engine/build.gradle.kts b/engine/build.gradle.kts index 7985c0fc..c38cccce 100644 --- a/engine/build.gradle.kts +++ b/engine/build.gradle.kts @@ -31,6 +31,9 @@ dependencies { implementation(Dependencies.Cache.CAFFEINE) implementation(Dependencies.Validation.JAKARTA_VALIDATION_API) + // SlateDB Java bindings (built from native/build-slatedb.sh) + implementation(files("${rootProject.projectDir}/native/lib/slatedb.jar")) + // reactor implementation(Dependencies.Reactor.CORE) implementation(Dependencies.Reactor.KOTLIN_EXTENSIONS) diff --git a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java deleted file mode 100644 index 97a2f558..00000000 --- a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbException.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.kakao.actionbase.v2.engine.storage.slatedb; - -public class SlateDbException extends RuntimeException { - private final int code; - - public SlateDbException(int code, String message) { - super("SlateDB error (" + code + "): " + message); - this.code = code; - } - - public int getCode() { - return code; - } -} diff --git a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java b/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java deleted file mode 100644 index 78cc28cd..00000000 --- a/engine/src/main/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNative.java +++ /dev/null @@ -1,432 +0,0 @@ -package com.kakao.actionbase.v2.engine.storage.slatedb; - -import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; -import java.lang.foreign.MemoryLayout; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.SegmentAllocator; -import java.lang.foreign.SymbolLookup; -import java.lang.foreign.ValueLayout; -import java.lang.invoke.MethodHandle; -import java.nio.file.Path; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Minimal FFI wrapper for SlateDB C bindings. - *

- * Uses Java 22+ Foreign Function & Memory API (JEP 454). - * Only exposes the essential operations: open, get, put, delete, close. - */ -public class SlateDbNative implements AutoCloseable { - - // CSdbValue: { data: *mut u8, len: usize } - private static final MemoryLayout C_SDB_VALUE_LAYOUT = MemoryLayout.structLayout( - ValueLayout.ADDRESS.withName("data"), - ValueLayout.JAVA_LONG.withName("len") - ); - - // CSdbResult: { error: i32(enum), padding: i32, message: *const c_char } - private static final MemoryLayout C_SDB_RESULT_LAYOUT = MemoryLayout.structLayout( - ValueLayout.JAVA_INT.withName("error"), - ValueLayout.JAVA_INT.withName("padding"), - ValueLayout.ADDRESS.withName("message") - ); - - // CSdbHandle: { _0: *mut SlateDbFFI } - private static final MemoryLayout C_SDB_HANDLE_LAYOUT = MemoryLayout.structLayout( - ValueLayout.ADDRESS.withName("_0") - ); - - // CSdbHandleResult: { handle: CSdbHandle, result: CSdbResult } - private static final MemoryLayout C_SDB_HANDLE_RESULT_LAYOUT = MemoryLayout.structLayout( - C_SDB_HANDLE_LAYOUT.withName("handle"), - C_SDB_RESULT_LAYOUT.withName("result") - ); - - // CSdbKeyValue: { key: CSdbValue, value: CSdbValue } - private static final MemoryLayout C_SDB_KEY_VALUE_LAYOUT = MemoryLayout.structLayout( - C_SDB_VALUE_LAYOUT.withName("key"), - C_SDB_VALUE_LAYOUT.withName("value") - ); - - // CSdbIteratorNextResult: { kv: CSdbKeyValue, has_value: bool(u8), padding, result: CSdbResult } - private static final MemoryLayout C_SDB_ITERATOR_NEXT_RESULT_LAYOUT = MemoryLayout.structLayout( - C_SDB_KEY_VALUE_LAYOUT.withName("kv"), - ValueLayout.JAVA_BYTE.withName("has_value"), - MemoryLayout.paddingLayout(7), - C_SDB_RESULT_LAYOUT.withName("result") - ); - - private final Arena arena; - private final MemorySegment dbHandle; - private final MethodHandle getHandle; - private final MethodHandle putHandle; - private final MethodHandle deleteHandle; - private final MethodHandle flushHandle; - private final MethodHandle closeHandle; - private final MethodHandle freeValueHandle; - private final MethodHandle scanPrefixHandle; - private final MethodHandle iteratorNextHandle; - private final MethodHandle iteratorCloseHandle; - - private SlateDbNative( - Arena arena, - MemorySegment dbHandle, - MethodHandle getHandle, - MethodHandle putHandle, - MethodHandle deleteHandle, - MethodHandle flushHandle, - MethodHandle closeHandle, - MethodHandle freeValueHandle, - MethodHandle scanPrefixHandle, - MethodHandle iteratorNextHandle, - MethodHandle iteratorCloseHandle - ) { - this.arena = arena; - this.dbHandle = dbHandle; - this.getHandle = getHandle; - this.putHandle = putHandle; - this.deleteHandle = deleteHandle; - this.flushHandle = flushHandle; - this.closeHandle = closeHandle; - this.freeValueHandle = freeValueHandle; - this.scanPrefixHandle = scanPrefixHandle; - this.iteratorNextHandle = iteratorNextHandle; - this.iteratorCloseHandle = iteratorCloseHandle; - } - - private static final int ERROR_NOT_FOUND = 2; - - public byte[] get(byte[] key) throws Throwable { - MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); - MemorySegment valueOut = arena.allocate(C_SDB_VALUE_LAYOUT); - - MemorySegment resultSegment = (MemorySegment) getHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle, - keySegment, - (long) key.length, - MemorySegment.NULL, - valueOut - ); - - // NotFound is not an error, just return null - int errorCode = resultSegment.get(ValueLayout.JAVA_INT, 0); - if (errorCode == ERROR_NOT_FOUND) { - return null; - } - checkResult(resultSegment); - - MemorySegment dataPtr = valueOut.get(ValueLayout.ADDRESS, 0); - if (dataPtr.equals(MemorySegment.NULL)) { - return null; - } - - int len = (int) valueOut.get(ValueLayout.JAVA_LONG, 8); - if (len == 0) { - return new byte[0]; - } - - byte[] bytes = new byte[len]; - dataPtr.reinterpret(len).asByteBuffer().get(bytes); - - freeValueHandle.invokeExact(valueOut); - - return bytes; - } - - public void put(byte[] key, byte[] value) throws Throwable { - MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); - MemorySegment valSegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, value); - - MemorySegment resultSegment = (MemorySegment) putHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle, - keySegment, - (long) key.length, - valSegment, - (long) value.length, - MemorySegment.NULL, - MemorySegment.NULL - ); - - checkResult(resultSegment); - } - - public void delete(byte[] key) throws Throwable { - MemorySegment keySegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, key); - - MemorySegment resultSegment = (MemorySegment) deleteHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle, - keySegment, - (long) key.length, - MemorySegment.NULL - ); - - checkResult(resultSegment); - } - - public void flush() throws Throwable { - MemorySegment resultSegment = (MemorySegment) flushHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle - ); - checkResult(resultSegment); - } - - /** - * Scan keys with given prefix and return up to limit key-value pairs. - */ - public List> scanPrefix(byte[] prefix, int limit) throws Throwable { - List> results = new ArrayList<>(); - - // Create iterator - MemorySegment prefixSegment = arena.allocateFrom(ValueLayout.JAVA_BYTE, prefix); - MemorySegment iteratorPtrOut = arena.allocate(ValueLayout.ADDRESS); - - MemorySegment resultSegment = (MemorySegment) scanPrefixHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle, - prefixSegment, - (long) prefix.length, - MemorySegment.NULL, // scan_options - iteratorPtrOut - ); - checkResult(resultSegment); - - MemorySegment iteratorPtr = iteratorPtrOut.get(ValueLayout.ADDRESS, 0); - if (iteratorPtr.equals(MemorySegment.NULL)) { - return results; - } - - try { - // Iterate up to limit - for (int i = 0; i < limit; i++) { - MemorySegment nextResult = (MemorySegment) iteratorNextHandle.invokeExact( - (SegmentAllocator) arena, - iteratorPtr - ); - - // Check if there's a value (has_value at offset 32: 2 CSdbValue = 32 bytes) - byte hasValue = nextResult.get(ValueLayout.JAVA_BYTE, 32); - if (hasValue == 0) { - break; // No more values - } - - // Check for errors (result at offset 40: 32 + 1 + 7 padding = 40) - int errorCode = nextResult.get(ValueLayout.JAVA_INT, 40); - if (errorCode != 0) { - break; // Error or end of iteration - } - - // Extract key (CSdbValue at offset 0) - MemorySegment keyDataPtr = nextResult.get(ValueLayout.ADDRESS, 0); - int keyLen = (int) nextResult.get(ValueLayout.JAVA_LONG, 8); - - // Extract value (CSdbValue at offset 16) - MemorySegment valueDataPtr = nextResult.get(ValueLayout.ADDRESS, 16); - int valueLen = (int) nextResult.get(ValueLayout.JAVA_LONG, 24); - - if (!keyDataPtr.equals(MemorySegment.NULL) && keyLen > 0) { - byte[] keyBytes = new byte[keyLen]; - keyDataPtr.reinterpret(keyLen).asByteBuffer().get(keyBytes); - - byte[] valueBytes = new byte[valueLen]; - if (!valueDataPtr.equals(MemorySegment.NULL) && valueLen > 0) { - valueDataPtr.reinterpret(valueLen).asByteBuffer().get(valueBytes); - } - - results.add(new AbstractMap.SimpleEntry<>(keyBytes, valueBytes)); - } - } - } finally { - // Close iterator - MemorySegment closeResult = (MemorySegment) iteratorCloseHandle.invokeExact( - (SegmentAllocator) arena, - iteratorPtr - ); - // Ignore close errors - } - - return results; - } - - @Override - public void close() throws Exception { - try { - MemorySegment resultSegment = (MemorySegment) closeHandle.invokeExact( - (SegmentAllocator) arena, - dbHandle - ); - checkResult(resultSegment); - } catch (Throwable e) { - throw new SlateDbException(-1, "Failed to close database: " + e.getMessage()); - } finally { - arena.close(); - } - } - - private void checkResult(MemorySegment resultSegment) throws SlateDbException { - int errorCode = resultSegment.get(ValueLayout.JAVA_INT, 0); - if (errorCode != 0) { - MemorySegment msgPtr = resultSegment.get(ValueLayout.ADDRESS, 8); - String msg; - if (!msgPtr.equals(MemorySegment.NULL)) { - msg = msgPtr.reinterpret(1024).getString(0); - } else { - msg = "Unknown error"; - } - throw new SlateDbException(errorCode, msg); - } - } - - /** - * Open database with object store URL. - * For local filesystem, use "file:///path/to/storage" as the url. - */ - public static SlateDbNative open(String dbPath, String url, Path libraryPath) throws Throwable { - Arena arena = Arena.ofShared(); - SymbolLookup lookup = SymbolLookup.libraryLookup(libraryPath, arena); - Linker linker = Linker.nativeLinker(); - - // slatedb_open(path, url, env_file) -> CSdbHandleResult - MethodHandle openHandle = linker.downcallHandle( - lookup.find("slatedb_open").orElseThrow(() -> new IllegalStateException("Function 'slatedb_open' not found")), - FunctionDescriptor.of( - C_SDB_HANDLE_RESULT_LAYOUT, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS - ) - ); - - MemorySegment pathSegment = arena.allocateFrom(dbPath); - MemorySegment urlSegment = url != null ? arena.allocateFrom(url) : MemorySegment.NULL; - MemorySegment handleResultSegment = (MemorySegment) openHandle.invokeExact( - (SegmentAllocator) arena, - pathSegment, - urlSegment, - MemorySegment.NULL - ); - - int errorCode = handleResultSegment.get(ValueLayout.JAVA_INT, 8); - if (errorCode != 0) { - MemorySegment msgPtr = handleResultSegment.get(ValueLayout.ADDRESS, 16); - String msg; - if (!msgPtr.equals(MemorySegment.NULL)) { - msg = msgPtr.reinterpret(1024).getString(0); - } else { - msg = "Failed to open database"; - } - arena.close(); - throw new SlateDbException(errorCode, msg); - } - - MemorySegment dbHandle = handleResultSegment.asSlice(0, 8); - return createInstance(arena, dbHandle, lookup, linker); - } - - private static SlateDbNative createInstance(Arena arena, MemorySegment dbHandle, SymbolLookup lookup, Linker linker) { - // slatedb_get_with_options - MethodHandle getHandle = linker.downcallHandle( - lookup.find("slatedb_get_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_get_with_options' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS - ) - ); - - // slatedb_put_with_options - MethodHandle putHandle = linker.downcallHandle( - lookup.find("slatedb_put_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_put_with_options' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS - ) - ); - - // slatedb_delete_with_options - MethodHandle deleteHandle = linker.downcallHandle( - lookup.find("slatedb_delete_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_delete_with_options' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS - ) - ); - - // slatedb_flush - MethodHandle flushHandle = linker.downcallHandle( - lookup.find("slatedb_flush").orElseThrow(() -> new IllegalStateException("Function 'slatedb_flush' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT - ) - ); - - // slatedb_close - MethodHandle closeHandle = linker.downcallHandle( - lookup.find("slatedb_close").orElseThrow(() -> new IllegalStateException("Function 'slatedb_close' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT - ) - ); - - // slatedb_free_value - MethodHandle freeValueHandle = linker.downcallHandle( - lookup.find("slatedb_free_value").orElseThrow(() -> new IllegalStateException("Function 'slatedb_free_value' not found")), - FunctionDescriptor.ofVoid(C_SDB_VALUE_LAYOUT) - ); - - // slatedb_scan_prefix_with_options(handle, prefix, prefix_len, scan_options, iterator_ptr) -> CSdbResult - MethodHandle scanPrefixHandle = linker.downcallHandle( - lookup.find("slatedb_scan_prefix_with_options").orElseThrow(() -> new IllegalStateException("Function 'slatedb_scan_prefix_with_options' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - C_SDB_HANDLE_LAYOUT, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS - ) - ); - - // slatedb_iterator_next(iter) -> CSdbIteratorNextResult - MethodHandle iteratorNextHandle = linker.downcallHandle( - lookup.find("slatedb_iterator_next").orElseThrow(() -> new IllegalStateException("Function 'slatedb_iterator_next' not found")), - FunctionDescriptor.of( - C_SDB_ITERATOR_NEXT_RESULT_LAYOUT, - ValueLayout.ADDRESS - ) - ); - - // slatedb_iterator_close(iter) -> CSdbResult - MethodHandle iteratorCloseHandle = linker.downcallHandle( - lookup.find("slatedb_iterator_close").orElseThrow(() -> new IllegalStateException("Function 'slatedb_iterator_close' not found")), - FunctionDescriptor.of( - C_SDB_RESULT_LAYOUT, - ValueLayout.ADDRESS - ) - ); - - return new SlateDbNative(arena, dbHandle, getHandle, putHandle, deleteHandle, flushHandle, closeHandle, freeValueHandle, scanPrefixHandle, iteratorNextHandle, iteratorCloseHandle); - } -} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt deleted file mode 100644 index 500a2ee2..00000000 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt +++ /dev/null @@ -1,51 +0,0 @@ -package com.kakao.actionbase.v2.engine.ffi - -import java.lang.foreign.Arena -import java.lang.foreign.FunctionDescriptor -import java.lang.foreign.Linker -import java.lang.foreign.SymbolLookup -import java.lang.foreign.ValueLayout -import java.lang.invoke.MethodHandle -import java.nio.file.Path - -/** - * Java FFI wrapper for native math operations. - * - * Demonstrates Java 22+ Foreign Function & Memory API (JEP 454). - */ -class MathOps private constructor( - private val arena: Arena, - private val addHandle: MethodHandle, -) : AutoCloseable { - fun add( - a: Int, - b: Int, - ): Int = addHandle.invokeExact(a, b) as Int - - override fun close() { - arena.close() - } - - companion object { - fun load(libraryPath: Path): MathOps { - val arena = Arena.ofShared() - val lookup = SymbolLookup.libraryLookup(libraryPath, arena) - val linker = Linker.nativeLinker() - - val descriptor = - FunctionDescriptor.of( - ValueLayout.JAVA_INT, - ValueLayout.JAVA_INT, - ValueLayout.JAVA_INT, - ) - - val addHandle = - linker.downcallHandle( - lookup.find("add").orElseThrow { IllegalStateException("Function 'add' not found") }, - descriptor, - ) - - return MathOps(arena, addHandle) - } - } -} diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt index a18a54f7..b958cd82 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabel.kt @@ -131,8 +131,7 @@ open class SlateDbHashLabel( // Filter by end key (exclusive) .dropLastWhile { (key, _) -> end?.key?.let { endKey -> Arrays.compareUnsigned(endKey, key) < 0 } ?: false - } - .take(limit) + }.take(limit) .map { (key, value) -> KeyFieldValue(key, value) } } diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt index 8bf136a5..8bcec6c9 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt @@ -2,29 +2,40 @@ package com.kakao.actionbase.v2.engine.storage.slatedb import com.kakao.actionbase.v2.engine.util.getLogger -import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import io.slatedb.SlateDb import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers object SlateDbConnections { private val logger = getLogger() + private val libraryLoaded = AtomicBoolean(false) private val connections: ConcurrentHashMap> = ConcurrentHashMap() + fun loadLibrary(libraryPath: String) { + if (libraryLoaded.compareAndSet(false, true)) { + logger.info("Loading SlateDB native library from: {}", libraryPath) + SlateDb.loadLibrary(libraryPath) + SlateDb.initLogging("info") + } + } + fun getConnection( dbPath: String, url: String, - libraryPath: Path, + libraryPath: String, ): Mono { val cacheKey = getCacheKey(dbPath, url) return connections.computeIfAbsent(cacheKey) { key -> Mono .fromCallable { - val native = SlateDbNative.open(dbPath, url, libraryPath) - SlateDbTable.create(native) + loadLibrary(libraryPath) + val db = SlateDb.open(dbPath, url, null) + SlateDbTable.create(db) }.subscribeOn(Schedulers.boundedElastic()) .doOnSuccess { logger.info("Successfully opened SlateDB connection for cacheKey: {}", key) diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt index 8d1f32d2..7f1bddf8 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt @@ -1,7 +1,5 @@ package com.kakao.actionbase.v2.engine.storage.slatedb -import java.nio.file.Path - import com.fasterxml.jackson.annotation.JsonIgnoreProperties import reactor.core.publisher.Mono @@ -23,6 +21,6 @@ data class SlateDbOptions( SlateDbConnections.getConnection( dbPath = path, url = url, - libraryPath = Path.of(libraryPath), + libraryPath = libraryPath, ) } diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt index 959e5860..38d26d03 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTable.kt @@ -1,8 +1,29 @@ package com.kakao.actionbase.v2.engine.storage.slatedb +import java.nio.ByteBuffer + +import io.slatedb.SlateDb +import io.slatedb.SlateDbKeyValue import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers +sealed class BatchOperation { + data class Put( + val key: ByteArray, + val value: ByteArray, + ) : BatchOperation() + + data class Delete( + val key: ByteArray, + ) : BatchOperation() + + /** Not yet supported - waiting for merge operator in slatedb-c (#1250) */ + data class Increment( + val key: ByteArray, + val delta: Long, + ) : BatchOperation() +} + interface SlateDbTable : AutoCloseable { fun get(key: ByteArray): Mono @@ -20,17 +41,19 @@ interface SlateDbTable : AutoCloseable { limit: Int, ): Mono>> + fun batch(operations: List): Mono + companion object { - fun create(native: SlateDbNative): SlateDbTable = SlateDbTableImpl(native) + fun create(db: SlateDb): SlateDbTable = SlateDbTableImpl(db) } } internal class SlateDbTableImpl( - private val native: SlateDbNative, + private val db: SlateDb, ) : SlateDbTable { override fun get(key: ByteArray): Mono = Mono - .fromCallable { native.get(key) } + .fromCallable { db.get(key) } .flatMap { Mono.justOrEmpty(it) } .subscribeOn(Schedulers.boundedElastic()) @@ -39,19 +62,19 @@ internal class SlateDbTableImpl( value: ByteArray, ): Mono = Mono - .fromCallable { native.put(key, value) } + .fromCallable { db.put(key, value) } .subscribeOn(Schedulers.boundedElastic()) .then() override fun delete(key: ByteArray): Mono = Mono - .fromCallable { native.delete(key) } + .fromCallable { db.delete(key) } .subscribeOn(Schedulers.boundedElastic()) .then() override fun flush(): Mono = Mono - .fromCallable { native.flush() } + .fromCallable { db.flush() } .subscribeOn(Schedulers.boundedElastic()) .then() @@ -61,12 +84,42 @@ internal class SlateDbTableImpl( ): Mono>> = Mono .fromCallable { - native.scanPrefix(prefix, limit).map { entry -> - entry.key to entry.value + val results = mutableListOf>() + db.scanPrefix(prefix).use { iterator -> + var kv: SlateDbKeyValue? = iterator.next() + var count = 0 + while (kv != null && count < limit) { + results.add(kv.key() to kv.value()) + count++ + kv = iterator.next() + } } + results.toList() }.subscribeOn(Schedulers.boundedElastic()) + override fun batch(operations: List): Mono = + Mono + .fromCallable { + SlateDb.newWriteBatch().use { batch -> + operations.forEach { op -> + when (op) { + is BatchOperation.Put -> batch.put(op.key, op.value) + is BatchOperation.Delete -> batch.delete(op.key) + is BatchOperation.Increment -> { + // TODO: Replace with batch.merge() once slatedb#1250 is resolved + // WARNING: This is NOT atomic - race condition possible + val currentValue = db.get(op.key)?.let { ByteBuffer.wrap(it).long } ?: 0L + val newBytes = ByteBuffer.allocate(Long.SIZE_BYTES).putLong(currentValue + op.delta).array() + batch.put(op.key, newBytes) + } + } + } + db.write(batch) + } + }.subscribeOn(Schedulers.boundedElastic()) + .then() + override fun close() { - native.close() + db.close() } } diff --git a/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java b/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java deleted file mode 100644 index d4faa054..00000000 --- a/engine/src/test/java/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbNativeTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.kakao.actionbase.v2.engine.storage.slatedb; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.*; - -class SlateDbNativeTest { - - @TempDir - Path tempDir; - - private SlateDbNative db; - - private static Path findLibraryPath() { - // Find project root by looking for settings.gradle.kts - Path dir = Path.of(System.getProperty("user.dir")); - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.getParent() != null) { - dir = dir.getParent(); - } - return dir.resolve("native/lib/libslatedb_c.dylib"); - } - - @BeforeEach - void setUp() throws Throwable { - // SlateDB uses object_store: - // - url: object store root (file:// for local filesystem) - // - path: prefix/directory within the store - String fileUrl = "file://" + tempDir.toAbsolutePath(); - String dbPath = "data"; // prefix within the object store - db = SlateDbNative.open(dbPath, fileUrl, findLibraryPath()); - } - - @AfterEach - void tearDown() throws Exception { - if (db != null) { - db.close(); - } - } - - @Test - void putAndGet() throws Throwable { - // Given - byte[] key = "hello".getBytes(StandardCharsets.UTF_8); - byte[] value = "world".getBytes(StandardCharsets.UTF_8); - - // When - db.put(key, value); - byte[] result = db.get(key); - - // Then - assertNotNull(result); - assertEquals("world", new String(result, StandardCharsets.UTF_8)); - } - - @Test - void getNonExistentKeyReturnsNull() throws Throwable { - // When - byte[] result = db.get("nonexistent".getBytes(StandardCharsets.UTF_8)); - - // Then - assertNull(result); - } - - @Test - void deleteRemovesKey() throws Throwable { - // Given - byte[] key = "to-delete".getBytes(StandardCharsets.UTF_8); - db.put(key, "value".getBytes(StandardCharsets.UTF_8)); - - // When - db.delete(key); - byte[] result = db.get(key); - - // Then - assertNull(result); - } - - @Test - void overwriteExistingKey() throws Throwable { - // Given - byte[] key = "key".getBytes(StandardCharsets.UTF_8); - db.put(key, "value1".getBytes(StandardCharsets.UTF_8)); - - // When - db.put(key, "value2".getBytes(StandardCharsets.UTF_8)); - byte[] result = db.get(key); - - // Then - assertNotNull(result); - assertEquals("value2", new String(result, StandardCharsets.UTF_8)); - } - - @Test - void binaryData() throws Throwable { - // Given - byte[] key = new byte[]{0x00, 0x01, 0x02, (byte) 0xFF}; - byte[] value = new byte[]{(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}; - - // When - db.put(key, value); - byte[] result = db.get(key); - - // Then - assertNotNull(result); - assertArrayEquals(value, result); - } -} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt index 2df71560..c6a5f8d5 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt @@ -177,9 +177,12 @@ class SlateDbIndexedLabelTest { // Insert edges with different scores val edges = listOf( - com.kakao.actionbase.v2.core.edge.Edge(ts, 100L, 201L, mapOf("score" to 10L, "memo" to "low")), - com.kakao.actionbase.v2.core.edge.Edge(ts + 1, 100L, 202L, mapOf("score" to 50L, "memo" to "medium")), - com.kakao.actionbase.v2.core.edge.Edge(ts + 2, 100L, 203L, mapOf("score" to 100L, "memo" to "high")), + com.kakao.actionbase.v2.core.edge + .Edge(ts, 100L, 201L, mapOf("score" to 10L, "memo" to "low")), + com.kakao.actionbase.v2.core.edge + .Edge(ts + 1, 100L, 202L, mapOf("score" to 50L, "memo" to "medium")), + com.kakao.actionbase.v2.core.edge + .Edge(ts + 2, 100L, 203L, mapOf("score" to 100L, "memo" to "high")), ) edges.forEach { edge -> diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt index 4e27ca35..5332be47 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt @@ -1,5 +1,6 @@ package com.kakao.actionbase.v2.engine.storage.slatedb +import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.nio.file.Path @@ -8,6 +9,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir +import io.slatedb.SlateDb import reactor.test.StepVerifier class SlateDbTableTest { @@ -16,20 +18,29 @@ class SlateDbTableTest { private lateinit var table: SlateDbTable - private fun findLibraryPath(): Path { + private fun findLibraryPath(): String { var dir = Path.of(System.getProperty("user.dir")) while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { dir = dir.parent } - return dir.resolve("native/lib/libslatedb_c.dylib") + val libName = + if (System.getProperty("os.name").lowercase().contains("linux")) { + "libslatedb_c.so" + } else { + "libslatedb_c.dylib" + } + return dir.resolve("native/lib/$libName").toAbsolutePath().toString() } @BeforeEach fun setUp() { + val libraryPath = findLibraryPath() + SlateDb.loadLibrary(libraryPath) + val fileUrl = "file://${tempDir.toAbsolutePath()}" val dbPath = "data" - val native = SlateDbNative.open(dbPath, fileUrl, findLibraryPath()) - table = SlateDbTable.create(native) + val db = SlateDb.open(dbPath, fileUrl, null) + table = SlateDbTable.create(db) } @AfterEach @@ -86,4 +97,90 @@ class SlateDbTableTest { ).expectNextMatches { it != null && String(it, StandardCharsets.UTF_8) == "flush-value" } .verifyComplete() } + + @Test + fun `batch writes multiple keys atomically`() { + val key1 = "batch-key-1".toByteArray(StandardCharsets.UTF_8) + val value1 = "batch-value-1".toByteArray(StandardCharsets.UTF_8) + val key2 = "batch-key-2".toByteArray(StandardCharsets.UTF_8) + val value2 = "batch-value-2".toByteArray(StandardCharsets.UTF_8) + val key3 = "batch-key-3".toByteArray(StandardCharsets.UTF_8) + val value3 = "batch-value-3".toByteArray(StandardCharsets.UTF_8) + + val operations = + listOf( + BatchOperation.Put(key1, value1), + BatchOperation.Put(key2, value2), + BatchOperation.Put(key3, value3), + ) + + StepVerifier + .create( + table + .batch(operations) + .then(table.get(key1)), + ).expectNextMatches { String(it, StandardCharsets.UTF_8) == "batch-value-1" } + .verifyComplete() + + StepVerifier + .create(table.get(key2)) + .expectNextMatches { String(it, StandardCharsets.UTF_8) == "batch-value-2" } + .verifyComplete() + } + + @Test + fun `batch with put and delete`() { + val key1 = "batch-put".toByteArray(StandardCharsets.UTF_8) + val value1 = "value".toByteArray(StandardCharsets.UTF_8) + val key2 = "batch-delete".toByteArray(StandardCharsets.UTF_8) + val value2 = "to-be-deleted".toByteArray(StandardCharsets.UTF_8) + + // First, put key2 + StepVerifier + .create(table.put(key2, value2)) + .verifyComplete() + + // Then batch: put key1, delete key2 + val operations = + listOf( + BatchOperation.Put(key1, value1), + BatchOperation.Delete(key2), + ) + + StepVerifier + .create( + table + .batch(operations) + .then(table.get(key1)), + ).expectNextMatches { String(it, StandardCharsets.UTF_8) == "value" } + .verifyComplete() + + // key2 should be deleted + StepVerifier + .create(table.get(key2)) + .verifyComplete() + } + + @Test + fun `batch increment adds delta to value`() { + val key = "counter".toByteArray(StandardCharsets.UTF_8) + + // Increment non-existent key (starts at 0) + StepVerifier + .create( + table + .batch(listOf(BatchOperation.Increment(key, 5))) + .then(table.get(key)), + ).expectNextMatches { ByteBuffer.wrap(it).long == 5L } + .verifyComplete() + + // Increment existing key + StepVerifier + .create( + table + .batch(listOf(BatchOperation.Increment(key, 3))) + .then(table.get(key)), + ).expectNextMatches { ByteBuffer.wrap(it).long == 8L } + .verifyComplete() + } } diff --git a/native/build-slatedb.sh b/native/build-slatedb.sh index 1df82b1e..ead4b107 100755 --- a/native/build-slatedb.sh +++ b/native/build-slatedb.sh @@ -5,52 +5,103 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SLATEDB_DIR="$SCRIPT_DIR/slatedb" LIB_DIR="$SCRIPT_DIR/lib" -# Check for rustup (needed for nightly) -if ! command -v rustup &> /dev/null; then - echo "Error: rustup is required (slatedb uses nightly Rust features)" - echo "" - echo "Install rustup:" - echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" - echo "" - echo "Then install nightly:" - echo " rustup install nightly" - exit 1 +# Branch containing slatedb-java (PR #1253) +# TODO: Change to 'main' after PR is merged +SLATEDB_BRANCH="java-bindings" +SLATEDB_REPO="https://github.com/criccomini/slatedb.git" + +# Parse arguments +BUILD_RUST=true +BUILD_JAVA=true +RUN_TESTS=false + +for arg in "$@"; do + case $arg in + --java-only) + BUILD_RUST=false + ;; + --rust-only) + BUILD_JAVA=false + ;; + --test) + RUN_TESTS=true + ;; + esac +done + +# Determine native library name +OS="$(uname -s)" +case "$OS" in + Darwin) NATIVE_LIB="libslatedb_c.dylib" ;; + Linux) NATIVE_LIB="libslatedb_c.so" ;; + *) NATIVE_LIB="" ;; +esac + +# Skip Rust build if native library already exists +if [ "$BUILD_RUST" = true ] && [ -f "$LIB_DIR/$NATIVE_LIB" ]; then + echo "Native library already exists: $LIB_DIR/$NATIVE_LIB" + echo "Skipping Rust build (use 'rm $LIB_DIR/$NATIVE_LIB' to force rebuild)" + BUILD_RUST=false fi -# Ensure nightly is installed -if ! rustup run nightly rustc --version &> /dev/null; then - echo "Installing Rust nightly..." - rustup install nightly +# Check for cargo only if we need to build Rust +if [ "$BUILD_RUST" = true ]; then + if ! command -v cargo &> /dev/null; then + echo "Error: cargo is required for building slatedb-c" + echo "" + echo "Install Rust:" + echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" + echo "" + echo "Or run with --java-only to skip Rust build (requires native lib to exist)" + exit 1 + fi fi # Clone if not exists if [ ! -d "$SLATEDB_DIR" ]; then - echo "Cloning slatedb..." - git clone --depth 1 https://github.com/slatedb/slatedb.git "$SLATEDB_DIR" + echo "Cloning slatedb from $SLATEDB_REPO (branch: $SLATEDB_BRANCH)..." + git clone --depth 1 --branch "$SLATEDB_BRANCH" "$SLATEDB_REPO" "$SLATEDB_DIR" fi -# Build slatedb-c with nightly -echo "Building slatedb-c (nightly)..." -cd "$SLATEDB_DIR" -cargo +nightly build --release -p slatedb-c - -# Copy library mkdir -p "$LIB_DIR" -OS="$(uname -s)" -case "$OS" in - Darwin) - cp target/release/libslatedb_c.dylib "$LIB_DIR/" - echo "Built: $LIB_DIR/libslatedb_c.dylib" - ;; - Linux) - cp target/release/libslatedb_c.so "$LIB_DIR/" - echo "Built: $LIB_DIR/libslatedb_c.so" - ;; - *) - echo "Unsupported OS: $OS" - exit 1 - ;; -esac +# Build slatedb-c +if [ "$BUILD_RUST" = true ]; then + echo "Building slatedb-c..." + cd "$SLATEDB_DIR" + cargo build --release -p slatedb-c + + cp "target/release/$NATIVE_LIB" "$LIB_DIR/" + echo "Built: $LIB_DIR/$NATIVE_LIB" +fi + +# Build slatedb-java +if [ "$BUILD_JAVA" = true ]; then + if [ -d "$SLATEDB_DIR/slatedb-java" ]; then + echo "Building slatedb-java..." + cd "$SLATEDB_DIR/slatedb-java" + + # Set SLATEDB_C_LIB for tests (as per upstream CI) + export SLATEDB_C_LIB="$LIB_DIR/$NATIVE_LIB" + + if [ "$RUN_TESTS" = true ]; then + echo "Running slatedb-java tests..." + ./gradlew check + else + ./gradlew jar --quiet + fi + + # Copy JAR to lib directory + JAR_FILE=$(find build/libs -name "slatedb-*.jar" -not -name "*-sources*" -not -name "*-javadoc*" | head -1) + if [ -n "$JAR_FILE" ]; then + cp "$JAR_FILE" "$LIB_DIR/slatedb.jar" + echo "Built: $LIB_DIR/slatedb.jar" + else + echo "Warning: slatedb-java JAR not found" + fi + else + echo "Warning: slatedb-java directory not found, skipping Java bindings" + fi +fi echo "Done." diff --git a/native/src/c/math_ops.c b/native/src/c/math_ops.c deleted file mode 100644 index b89d28bc..00000000 --- a/native/src/c/math_ops.c +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Simple C library for FFI demonstration. - * - * Compile on macOS: - * clang -shared -o libmathops.dylib math_ops.c - * - * Compile on Linux: - * gcc -shared -fPIC -o libmathops.so math_ops.c - */ - -int add(int a, int b) { - return a + b; -} From 80d736dc22579520cbbbe25dee794eba11b3e050 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 4 Feb 2026 17:06:07 +0900 Subject: [PATCH 03/11] test(engine): add storage backend compatibility tests (#168) --- .../datastore/DatastoreCompatibilityTest.kt | 285 ++++++++++++++++++ .../HBaseDatastoreCompatibilityTest.kt | 140 +++++++++ .../MemoryDatastoreCompatibilityTest.kt | 61 ++++ .../engine/datastore/StorageOperations.kt | 57 ++++ 4 files changed, 543 insertions(+) create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/DatastoreCompatibilityTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/HBaseDatastoreCompatibilityTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/MemoryDatastoreCompatibilityTest.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/StorageOperations.kt diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/DatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/DatastoreCompatibilityTest.kt new file mode 100644 index 00000000..379160da --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/DatastoreCompatibilityTest.kt @@ -0,0 +1,285 @@ +package com.kakao.actionbase.engine.datastore + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test + +/** + * Abstract compatibility test for storage backends. + * + * Required operations: get, scan, put, delete, increment, batch, checkAndMutate. + * + * @see Storage Backends + */ +abstract class DatastoreCompatibilityTest { + protected abstract fun createStore(): StorageOperations + + protected open fun cleanup() {} + + protected open fun supportsCheckAndMutate(): Boolean = true + + protected open fun supportsScanLimit(): Boolean = true + + private lateinit var store: StorageOperations + + @BeforeEach fun setUp() { + store = createStore() + } + + @AfterEach fun tearDown() { + cleanup() + } + + @Nested + @DisplayName("get") + inner class GetTest { + @Test fun `returns value when key exists`() { + store.put(b("key"), b("value")) + assert(store.get(b("key"))?.contentEquals(b("value")) == true) + } + + @Test fun `returns null when key not exists`() { + assert(store.get(b("missing")) == null) + } + + @Test fun `getAll returns matching records`() { + store.put(b("k1"), b("v1")) + store.put(b("k2"), b("v2")) + assert(store.getAll(listOf(b("k1"), b("k2"))).size == 2) + } + + @Test fun `getAll skips missing keys`() { + store.put(b("exists"), b("v")) + assert(store.getAll(listOf(b("exists"), b("missing"))).size == 1) + } + } + + @Nested + @DisplayName("scan") + inner class ScanTest { + @BeforeEach fun setup() { + listOf("user:001:a", "user:001:b", "user:002:a", "post:001").forEach { + store.put(b(it), b("v")) + } + } + + @Test fun `returns matching prefix`() { + val results = store.scan(b("user:001"), 100) + assert(results.size == 2) + assert(results.all { String(it.first).startsWith("user:001") }) + } + + @Test fun `returns empty for non-matching prefix`() { + assert(store.scan(b("nonexistent"), 100).isEmpty()) + } + + @Test fun `returns sorted keys`() { + val keys = store.scan(b("user:"), 100).map { String(it.first) } + assert(keys == keys.sorted()) + } + + @Test fun `respects limit`() { + assumeTrue(supportsScanLimit()) + assert(store.scan(b("user:"), 2).size == 2) + } + } + + @Nested + @DisplayName("put") + inner class PutTest { + @Test fun `stores value`() { + store.put(b("k"), b("v")) + assert(store.get(b("k"))?.contentEquals(b("v")) == true) + } + + @Test fun `overwrites existing`() { + store.put(b("k"), b("old")) + store.put(b("k"), b("new")) + assert(String(store.get(b("k"))!!) == "new") + } + } + + @Nested + @DisplayName("delete") + inner class DeleteTest { + @Test fun `removes key`() { + store.put(b("k"), b("v")) + store.delete(b("k")) + assert(store.get(b("k")) == null) + } + + @Test fun `silently succeeds for missing key`() { + store.delete(b("nonexistent")) + } + } + + @Nested + @DisplayName("increment") + inner class IncrementTest { + @Test fun `creates counter if not exists`() { + assert(store.increment(b("cnt"), 10) == 10L) + } + + @Test fun `updates existing counter`() { + store.put(b("cnt"), longToBytes(100)) + assert(store.increment(b("cnt"), 50) == 150L) + } + + @Test fun `decrements with negative delta`() { + store.put(b("cnt"), longToBytes(100)) + assert(store.increment(b("cnt"), -30) == 70L) + } + } + + @Nested + @DisplayName("batch") + inner class BatchTest { + @Test fun `executes puts`() { + store.batch(listOf(Mutation.Put(b("b1"), b("v1")), Mutation.Put(b("b2"), b("v2")))) + assert(store.getAll(listOf(b("b1"), b("b2"))).size == 2) + } + + @Test fun `executes deletes`() { + store.put(b("d1"), b("v")) + store.put(b("d2"), b("v")) + store.batch(listOf(Mutation.Delete(b("d1")), Mutation.Delete(b("d2")))) + assert(store.getAll(listOf(b("d1"), b("d2"))).isEmpty()) + } + + @Test fun `executes increments`() { + store.batch(listOf(Mutation.Increment(b("c1"), 10), Mutation.Increment(b("c2"), 20))) + assert(bytesToLong(store.get(b("c1"))!!) == 10L) + assert(bytesToLong(store.get(b("c2"))!!) == 20L) + } + + @Test fun `executes mixed mutations`() { + store.put(b("to-delete"), b("v")) + store.batch( + listOf( + Mutation.Put(b("new"), b("v")), + Mutation.Delete(b("to-delete")), + Mutation.Increment(b("cnt"), 100), + ), + ) + assert(store.get(b("new")) != null) + assert(store.get(b("to-delete")) == null) + assert(bytesToLong(store.get(b("cnt"))!!) == 100L) + } + } + + @Nested + @DisplayName("checkAndMutate") + inner class CheckAndMutateTest { + @BeforeEach fun checkSupport() { + assumeTrue(supportsCheckAndMutate()) + } + + @Nested + @DisplayName("setIfNotExists") + inner class SetIfNotExistsTest { + @Test fun `succeeds when key not exists`() { + assert(store.setIfNotExists(b("lock"), b("owner"))) + assert(store.get(b("lock"))?.contentEquals(b("owner")) == true) + } + + @Test fun `fails when key exists`() { + store.put(b("lock"), b("existing")) + assert(!store.setIfNotExists(b("lock"), b("new"))) + assert(String(store.get(b("lock"))!!) == "existing") + } + } + + @Nested + @DisplayName("deleteIfEquals") + inner class DeleteIfEqualsTest { + @Test fun `succeeds when value matches`() { + store.put(b("lock"), b("owner")) + assert(store.deleteIfEquals(b("lock"), b("owner"))) + assert(store.get(b("lock")) == null) + } + + @Test fun `fails when value differs`() { + store.put(b("lock"), b("owner")) + assert(!store.deleteIfEquals(b("lock"), b("different"))) + assert(store.get(b("lock")) != null) + } + + @Test fun `fails when key not exists`() { + assert(!store.deleteIfEquals(b("missing"), b("v"))) + } + } + + @Nested + @DisplayName("concurrent") + inner class ConcurrentTest { + @Test fun `only one thread acquires lock`() { + val threads = 10 + val acquired = AtomicInteger(0) + val latch = CountDownLatch(threads) + val executor = Executors.newFixedThreadPool(threads) + + repeat(threads) { i -> + executor.submit { + try { + if (store.setIfNotExists(b("lock"), b("owner-$i"))) { + acquired.incrementAndGet() + } + } finally { + latch.countDown() + } + } + } + + latch.await() + executor.shutdown() + assert(acquired.get() == 1) { "Expected 1 but got ${acquired.get()}" } + } + + @Test fun `only owner releases lock`() { + store.put(b("lock"), b("owner-0")) + val threads = 10 + val released = AtomicInteger(0) + val latch = CountDownLatch(threads) + val executor = Executors.newFixedThreadPool(threads) + + repeat(threads) { i -> + executor.submit { + try { + if (store.deleteIfEquals(b("lock"), b("owner-$i"))) { + released.incrementAndGet() + } + } finally { + latch.countDown() + } + } + } + + latch.await() + executor.shutdown() + assert(released.get() == 1) { "Expected 1 but got ${released.get()}" } + } + } + } + + companion object { + fun b(s: String): ByteArray = s.toByteArray() + + fun longToBytes(v: Long): ByteArray = + ByteBuffer + .allocate(8) + .order(ByteOrder.BIG_ENDIAN) + .putLong(v) + .array() + + fun bytesToLong(b: ByteArray): Long = ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).long + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/HBaseDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/HBaseDatastoreCompatibilityTest.kt new file mode 100644 index 00000000..b7d768bb --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/HBaseDatastoreCompatibilityTest.kt @@ -0,0 +1,140 @@ +package com.kakao.actionbase.engine.datastore + +import com.kakao.actionbase.test.hbase.HBaseTestingCluster +import com.kakao.actionbase.v2.engine.storage.hbase.HBaseConnections + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +import org.apache.hadoop.hbase.NamespaceDescriptor +import org.apache.hadoop.hbase.TableName +import org.apache.hadoop.hbase.client.Admin +import org.apache.hadoop.hbase.client.CheckAndMutate +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder +import org.apache.hadoop.hbase.client.Delete +import org.apache.hadoop.hbase.client.Get +import org.apache.hadoop.hbase.client.Increment +import org.apache.hadoop.hbase.client.Put +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.client.Table +import org.apache.hadoop.hbase.client.TableDescriptorBuilder +import org.apache.hadoop.hbase.filter.PrefixFilter +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance + +/** + * HBase compatibility test. Default: MockConnection. Set HBASE_MINI_CLUSTER=true for mini cluster. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class HBaseDatastoreCompatibilityTest : DatastoreCompatibilityTest() { + private lateinit var table: Table + private val tableName = TableName.valueOf("test", "compatibility_test") + private val cf = "f".toByteArray() + private val useMiniCluster = System.getenv("HBASE_MINI_CLUSTER") == "true" + + @BeforeAll + fun setUpHBase() { + val connection = + if (useMiniCluster) { + HBaseTestingCluster.startIfNeeded() + HBaseTestingCluster.connection.also { createTableIfNeeded(it.admin) } + } else { + HBaseConnections.getMockConnection("test") + } + table = connection.getTable(tableName) + } + + @AfterAll + fun tearDownHBase() { + table.close() + if (useMiniCluster) HBaseTestingCluster.stopIfNeeded() + } + + override fun createStore(): StorageOperations = HBaseOps(table, cf) + + override fun supportsCheckAndMutate() = useMiniCluster + + override fun supportsScanLimit() = useMiniCluster + + override fun cleanup() { + table.getScanner(Scan()).use { s -> + s.map { Delete(it.row) }.takeIf { it.isNotEmpty() }?.let { table.delete(it) } + } + } + + private fun createTableIfNeeded(admin: Admin) { + val ns = tableName.namespaceAsString + if (admin.listNamespaceDescriptors().none { it.name == ns }) { + admin.createNamespace(NamespaceDescriptor.create(ns).build()) + } + if (!admin.tableExists(tableName)) { + admin.createTable( + TableDescriptorBuilder + .newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)) + .build(), + ) + } + } + + private class HBaseOps( + private val t: Table, + private val cf: ByteArray, + ) : StorageOperations { + private val q = "v".toByteArray() + + override fun get(key: ByteArray) = t.get(Get(key).addColumn(cf, q)).getValue(cf, q) + + override fun getAll(keys: List) = t.get(keys.map { Get(it).addColumn(cf, q) }).mapNotNull { r -> r.getValue(cf, q)?.let { r.row to it } } + + override fun scan( + prefix: ByteArray, + limit: Int, + ) = t + .getScanner(Scan().setFilter(PrefixFilter(prefix)).addColumn(cf, q).setLimit(limit)) + .use { s -> s.mapNotNull { r -> r.getValue(cf, q)?.let { r.row to it } } } + + override fun put( + key: ByteArray, + value: ByteArray, + ) { + t.put(Put(key).addColumn(cf, q, value)) + } + + override fun delete(key: ByteArray) { + t.delete(Delete(key)) + } + + override fun increment( + key: ByteArray, + delta: Long, + ) = ByteBuffer + .wrap(t.increment(Increment(key).addColumn(cf, q, delta)).getValue(cf, q)) + .order(ByteOrder.BIG_ENDIAN) + .long + + override fun batch(mutations: List) { + if (mutations.isEmpty()) return + val actions = + mutations.map { m -> + when (m) { + is Mutation.Put -> Put(m.key).addColumn(cf, q, m.value) + is Mutation.Delete -> Delete(m.key) + is Mutation.Increment -> Increment(m.key).addColumn(cf, q, m.delta) + } + } + t.batch(actions, arrayOfNulls(actions.size)) + } + + override fun setIfNotExists( + key: ByteArray, + value: ByteArray, + ) = t.checkAndMutate(CheckAndMutate.newBuilder(key).ifNotExists(cf, q).build(Put(key).addColumn(cf, q, value))).isSuccess + + override fun deleteIfEquals( + key: ByteArray, + expectedValue: ByteArray, + ) = t.checkAndMutate(CheckAndMutate.newBuilder(key).ifEquals(cf, q, expectedValue).build(Delete(key))).isSuccess + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/MemoryDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/MemoryDatastoreCompatibilityTest.kt new file mode 100644 index 00000000..fa47beeb --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/MemoryDatastoreCompatibilityTest.kt @@ -0,0 +1,61 @@ +package com.kakao.actionbase.engine.datastore + +import com.kakao.actionbase.engine.datastore.impl.ByteArrayStore + +/** Memory (ByteArrayStore) compatibility test. */ +class MemoryDatastoreCompatibilityTest : DatastoreCompatibilityTest() { + private lateinit var store: ByteArrayStore + + override fun createStore(): StorageOperations { + store = ByteArrayStore() + return MemoryOps(store) + } + + private class MemoryOps( + private val s: ByteArrayStore, + ) : StorageOperations { + override fun get(key: ByteArray) = s[key] + + override fun getAll(keys: List) = keys.mapNotNull { k -> s[k]?.let { k to it } } + + override fun scan( + prefix: ByteArray, + limit: Int, + ) = s.prefixScan(prefix).take(limit).map { it.key to it.value } + + override fun put( + key: ByteArray, + value: ByteArray, + ) { + s[key] = value + } + + override fun delete(key: ByteArray) { + s.remove(key) + } + + override fun increment( + key: ByteArray, + delta: Long, + ) = s.increment(key, delta) + + override fun batch(mutations: List) = + mutations.forEach { m -> + when (m) { + is Mutation.Put -> s[m.key] = m.value + is Mutation.Delete -> s.remove(m.key) + is Mutation.Increment -> s.increment(m.key, m.delta) + } + } + + override fun setIfNotExists( + key: ByteArray, + value: ByteArray, + ) = s.checkAndSet(key, null, value) + + override fun deleteIfEquals( + key: ByteArray, + expectedValue: ByteArray, + ) = s.checkAndSet(key, expectedValue, null) + } +} diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/StorageOperations.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/StorageOperations.kt new file mode 100644 index 00000000..142c417c --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/StorageOperations.kt @@ -0,0 +1,57 @@ +package com.kakao.actionbase.engine.datastore + +/** + * Storage operations interface for compatibility testing. + * + * Defines the minimal operations required for Actionbase storage backends. + */ +interface StorageOperations { + fun get(key: ByteArray): ByteArray? + + fun getAll(keys: List): List> + + fun scan( + prefix: ByteArray, + limit: Int, + ): List> + + fun put( + key: ByteArray, + value: ByteArray, + ) + + fun delete(key: ByteArray) + + fun increment( + key: ByteArray, + delta: Long, + ): Long + + fun batch(mutations: List) + + fun setIfNotExists( + key: ByteArray, + value: ByteArray, + ): Boolean + + fun deleteIfEquals( + key: ByteArray, + expectedValue: ByteArray, + ): Boolean +} + +sealed class Mutation { + class Put( + val key: ByteArray, + val value: ByteArray, + ) : Mutation() + + class Delete( + val key: ByteArray, + ) : Mutation() + + class Increment( + val key: ByteArray, + val delta: Long, + ) : Mutation() +} From ec35907ed339303fcc8e0d84e094b2f5e7f37790 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 4 Feb 2026 17:38:26 +0900 Subject: [PATCH 04/11] test(engine): add SlateDB compatibility test Implements StorageOperations adapter for SlateDB backend. - 19 tests pass (get, scan, put, delete, increment, batch) - 7 tests skipped (checkAndMutate not supported by SlateDB) Co-Authored-By: Claude Opus 4.5 --- .../SlateDBDatastoreCompatibilityTest.kt | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt new file mode 100644 index 00000000..4e4bacce --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt @@ -0,0 +1,131 @@ +package com.kakao.actionbase.engine.datastore + +import com.kakao.actionbase.v2.engine.storage.slatedb.BatchOperation +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbTable + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.file.Path + +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.io.TempDir + +import io.slatedb.SlateDb + +/** + * SlateDB compatibility test. + * + * Requires native library: native/lib/libslatedb_c.dylib (macOS) or libslatedb_c.so (Linux) + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SlateDBDatastoreCompatibilityTest : DatastoreCompatibilityTest() { + private lateinit var table: SlateDbTable + private lateinit var tempDir: Path + + @BeforeAll + fun setUpSlateDB( + @TempDir dir: Path, + ) { + tempDir = dir + SlateDb.loadLibrary(findLibraryPath()) + val db = SlateDb.open("data", "file://${tempDir.toAbsolutePath()}", null) + table = SlateDbTable.create(db) + } + + @AfterAll + fun tearDownSlateDB() { + table.close() + } + + override fun createStore(): StorageOperations = SlateDBOps(table) + + override fun supportsCheckAndMutate() = false + + override fun cleanup() { + // Scan with empty prefix to get all keys, then delete them + table.scanPrefix(ByteArray(0), Int.MAX_VALUE).block()?.forEach { (key, _) -> + table.delete(key).block() + } + } + + private fun findLibraryPath(): String { + var dir = Path.of(System.getProperty("user.dir")) + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { + dir = dir.parent + } + val libName = + if (System.getProperty("os.name").lowercase().contains("linux")) { + "libslatedb_c.so" + } else { + "libslatedb_c.dylib" + } + return dir.resolve("native/lib/$libName").toAbsolutePath().toString() + } + + private class SlateDBOps( + private val t: SlateDbTable, + ) : StorageOperations { + override fun get(key: ByteArray): ByteArray? = t.get(key).block() + + override fun getAll(keys: List) = keys.mapNotNull { k -> t.get(k).block()?.let { k to it } } + + override fun scan( + prefix: ByteArray, + limit: Int, + ) = t.scanPrefix(prefix, limit).block() ?: emptyList() + + override fun put( + key: ByteArray, + value: ByteArray, + ) { + t.put(key, value).block() + } + + override fun delete(key: ByteArray) { + t.delete(key).block() + } + + override fun increment( + key: ByteArray, + delta: Long, + ): Long { + val current = + t.get(key).block()?.let { + ByteBuffer.wrap(it).order(ByteOrder.BIG_ENDIAN).long + } ?: 0L + val newValue = current + delta + val bytes = + ByteBuffer + .allocate(8) + .order(ByteOrder.BIG_ENDIAN) + .putLong(newValue) + .array() + t.put(key, bytes).block() + return newValue + } + + override fun batch(mutations: List) { + val ops = + mutations.map { m -> + when (m) { + is Mutation.Put -> BatchOperation.Put(m.key, m.value) + is Mutation.Delete -> BatchOperation.Delete(m.key) + is Mutation.Increment -> BatchOperation.Increment(m.key, m.delta) + } + } + t.batch(ops).block() + } + + override fun setIfNotExists( + key: ByteArray, + value: ByteArray, + ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") + + override fun deleteIfEquals( + key: ByteArray, + expectedValue: ByteArray, + ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") + } +} From f447ae4f76ce70da686b0fff4553526b8367b337 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 4 Feb 2026 17:52:09 +0900 Subject: [PATCH 05/11] Revert "test(engine): add SlateDB compatibility test" This reverts commit ec35907ed339303fcc8e0d84e094b2f5e7f37790. --- .../SlateDBDatastoreCompatibilityTest.kt | 131 ------------------ 1 file changed, 131 deletions(-) delete mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt deleted file mode 100644 index 4e4bacce..00000000 --- a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt +++ /dev/null @@ -1,131 +0,0 @@ -package com.kakao.actionbase.engine.datastore - -import com.kakao.actionbase.v2.engine.storage.slatedb.BatchOperation -import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbTable - -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.nio.file.Path - -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.TestInstance -import org.junit.jupiter.api.io.TempDir - -import io.slatedb.SlateDb - -/** - * SlateDB compatibility test. - * - * Requires native library: native/lib/libslatedb_c.dylib (macOS) or libslatedb_c.so (Linux) - */ -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -class SlateDBDatastoreCompatibilityTest : DatastoreCompatibilityTest() { - private lateinit var table: SlateDbTable - private lateinit var tempDir: Path - - @BeforeAll - fun setUpSlateDB( - @TempDir dir: Path, - ) { - tempDir = dir - SlateDb.loadLibrary(findLibraryPath()) - val db = SlateDb.open("data", "file://${tempDir.toAbsolutePath()}", null) - table = SlateDbTable.create(db) - } - - @AfterAll - fun tearDownSlateDB() { - table.close() - } - - override fun createStore(): StorageOperations = SlateDBOps(table) - - override fun supportsCheckAndMutate() = false - - override fun cleanup() { - // Scan with empty prefix to get all keys, then delete them - table.scanPrefix(ByteArray(0), Int.MAX_VALUE).block()?.forEach { (key, _) -> - table.delete(key).block() - } - } - - private fun findLibraryPath(): String { - var dir = Path.of(System.getProperty("user.dir")) - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { - dir = dir.parent - } - val libName = - if (System.getProperty("os.name").lowercase().contains("linux")) { - "libslatedb_c.so" - } else { - "libslatedb_c.dylib" - } - return dir.resolve("native/lib/$libName").toAbsolutePath().toString() - } - - private class SlateDBOps( - private val t: SlateDbTable, - ) : StorageOperations { - override fun get(key: ByteArray): ByteArray? = t.get(key).block() - - override fun getAll(keys: List) = keys.mapNotNull { k -> t.get(k).block()?.let { k to it } } - - override fun scan( - prefix: ByteArray, - limit: Int, - ) = t.scanPrefix(prefix, limit).block() ?: emptyList() - - override fun put( - key: ByteArray, - value: ByteArray, - ) { - t.put(key, value).block() - } - - override fun delete(key: ByteArray) { - t.delete(key).block() - } - - override fun increment( - key: ByteArray, - delta: Long, - ): Long { - val current = - t.get(key).block()?.let { - ByteBuffer.wrap(it).order(ByteOrder.BIG_ENDIAN).long - } ?: 0L - val newValue = current + delta - val bytes = - ByteBuffer - .allocate(8) - .order(ByteOrder.BIG_ENDIAN) - .putLong(newValue) - .array() - t.put(key, bytes).block() - return newValue - } - - override fun batch(mutations: List) { - val ops = - mutations.map { m -> - when (m) { - is Mutation.Put -> BatchOperation.Put(m.key, m.value) - is Mutation.Delete -> BatchOperation.Delete(m.key) - is Mutation.Increment -> BatchOperation.Increment(m.key, m.delta) - } - } - t.batch(ops).block() - } - - override fun setIfNotExists( - key: ByteArray, - value: ByteArray, - ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") - - override fun deleteIfEquals( - key: ByteArray, - expectedValue: ByteArray, - ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") - } -} From 50535b33bf8cf9b8845089aeb8bf40fcc865f764 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 4 Feb 2026 18:31:47 +0900 Subject: [PATCH 06/11] test(engine): add SlateDB compatibility test (#170) --- .../SlateDBDatastoreCompatibilityTest.kt | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt new file mode 100644 index 00000000..bd4cd7e6 --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt @@ -0,0 +1,140 @@ +package com.kakao.actionbase.engine.datastore + +import com.kakao.actionbase.v2.engine.storage.slatedb.BatchOperation +import com.kakao.actionbase.v2.engine.storage.slatedb.SlateDbTable + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.file.Path + +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.io.TempDir + +import io.slatedb.SlateDb + +/** + * SlateDB compatibility test. + * + * Disabled by default. Set SLATEDB_TEST=true to run. + * Requires native library: native/lib/libslatedb_c.dylib (macOS) or libslatedb_c.so (Linux) + * + * To run: + * SLATEDB_TEST=true ./gradlew :engine:test --tests "*SlateDBDatastoreCompatibilityTest*" + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SlateDBDatastoreCompatibilityTest : DatastoreCompatibilityTest() { + private var table: SlateDbTable? = null + private lateinit var tempDir: Path + + private val enabled = System.getenv("SLATEDB_TEST") == "true" + + @BeforeAll + fun setUpSlateDB( + @TempDir dir: Path, + ) { + assumeTrue(enabled, "SLATEDB_TEST=true not set") + tempDir = dir + SlateDb.loadLibrary(findLibraryPath()) + val db = SlateDb.open("data", "file://${tempDir.toAbsolutePath()}", null) + table = SlateDbTable.create(db) + } + + @AfterAll + fun tearDownSlateDB() { + table?.close() + } + + override fun createStore(): StorageOperations = SlateDBOperations(table!!) + + override fun supportsCheckAndMutate() = false + + override fun cleanup() { + table?.let { t -> + t.scanPrefix(ByteArray(0), Int.MAX_VALUE).block()?.forEach { (key, _) -> + t.delete(key).block() + } + } + } + + private fun findLibraryPath(): String { + var dir = Path.of(System.getProperty("user.dir")) + while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { + dir = dir.parent + } + val libName = + if (System.getProperty("os.name").lowercase().contains("linux")) { + "libslatedb_c.so" + } else { + "libslatedb_c.dylib" + } + return dir.resolve("native/lib/$libName").toAbsolutePath().toString() + } + + private class SlateDBOperations( + private val table: SlateDbTable, + ) : StorageOperations { + override fun get(key: ByteArray): ByteArray? = table.get(key).block() + + override fun getAll(keys: List) = keys.mapNotNull { k -> table.get(k).block()?.let { k to it } } + + override fun scan( + prefix: ByteArray, + limit: Int, + ) = table.scanPrefix(prefix, limit).block() ?: emptyList() + + override fun put( + key: ByteArray, + value: ByteArray, + ) { + table.put(key, value).block() + } + + override fun delete(key: ByteArray) { + table.delete(key).block() + } + + override fun increment( + key: ByteArray, + delta: Long, + ): Long { + val current = + table.get(key).block()?.let { + ByteBuffer.wrap(it).order(ByteOrder.BIG_ENDIAN).long + } ?: 0L + val newValue = current + delta + val bytes = + ByteBuffer + .allocate(8) + .order(ByteOrder.BIG_ENDIAN) + .putLong(newValue) + .array() + table.put(key, bytes).block() + return newValue + } + + override fun batch(mutations: List) { + val ops = + mutations.map { m -> + when (m) { + is Mutation.Put -> BatchOperation.Put(m.key, m.value) + is Mutation.Delete -> BatchOperation.Delete(m.key) + is Mutation.Increment -> BatchOperation.Increment(m.key, m.delta) + } + } + table.batch(ops).block() + } + + override fun setIfNotExists( + key: ByteArray, + value: ByteArray, + ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") + + override fun deleteIfEquals( + key: ByteArray, + expectedValue: ByteArray, + ): Boolean = throw UnsupportedOperationException("SlateDB does not support checkAndMutate") + } +} From 313f7b3b20498cf16733f8486e0386b3bc33ee00 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 25 Feb 2026 10:21:34 +0900 Subject: [PATCH 07/11] feat(engine): switch to JAR-bundled native library loading Replace manual native library path management with JAR-bundled native library from slatedb/slatedb#1329. NativeLibraryLoader automatically extracts and loads the platform-specific library from classpath resources. - Remove SlateDb.loadLibrary(path) calls (API removed in upstream PR) - Use SlateDb.initLogging(SlateDbConfig.LogLevel.INFO) for initialization - Remove libraryPath from SlateDbOptions, SlateDbConnections, tests, and config - Remove working directory hack in server bootRun task Tested: all 4 SlateDB test suites pass (SlateDbTableTest, SlateDbStorageTest, SlateDbHashLabelTest, SlateDbIndexedLabelTest) --- .../storage/slatedb/SlateDbConnections.kt | 15 +++++++-------- .../engine/storage/slatedb/SlateDbOptions.kt | 4 +--- .../SlateDBDatastoreCompatibilityTest.kt | 18 ++---------------- .../label/slatedb/SlateDbHashLabelTest.kt | 13 ------------- .../label/slatedb/SlateDbIndexedLabelTest.kt | 13 ------------- .../storage/slatedb/SlateDbStorageTest.kt | 1 - .../engine/storage/slatedb/SlateDbTableTest.kt | 18 ++---------------- server/build.gradle.kts | 2 -- .../main/resources/application-slatedb.yaml | 5 +---- 9 files changed, 13 insertions(+), 76 deletions(-) diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt index 8bcec6c9..c3cccd4a 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbConnections.kt @@ -6,34 +6,33 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean import io.slatedb.SlateDb +import io.slatedb.SlateDbConfig import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers object SlateDbConnections { private val logger = getLogger() - private val libraryLoaded = AtomicBoolean(false) + private val initialized = AtomicBoolean(false) private val connections: ConcurrentHashMap> = ConcurrentHashMap() - fun loadLibrary(libraryPath: String) { - if (libraryLoaded.compareAndSet(false, true)) { - logger.info("Loading SlateDB native library from: {}", libraryPath) - SlateDb.loadLibrary(libraryPath) - SlateDb.initLogging("info") + fun ensureInitialized() { + if (initialized.compareAndSet(false, true)) { + logger.info("Initializing SlateDB (native library loaded from JAR classpath)") + SlateDb.initLogging(SlateDbConfig.LogLevel.INFO) } } fun getConnection( dbPath: String, url: String, - libraryPath: String, ): Mono { val cacheKey = getCacheKey(dbPath, url) return connections.computeIfAbsent(cacheKey) { key -> Mono .fromCallable { - loadLibrary(libraryPath) + ensureInitialized() val db = SlateDb.open(dbPath, url, null) SlateDbTable.create(db) }.subscribeOn(Schedulers.boundedElastic()) diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt index 7f1bddf8..0864a311 100644 --- a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbOptions.kt @@ -8,10 +8,9 @@ import reactor.core.publisher.Mono data class SlateDbOptions( val path: String = "data", val url: String = "", - val libraryPath: String = "", ) { fun checkConnection(): Mono = - if (url.isBlank() || libraryPath.isBlank()) { + if (url.isBlank()) { Mono.just(false) } else { Mono.just(true) @@ -21,6 +20,5 @@ data class SlateDbOptions( SlateDbConnections.getConnection( dbPath = path, url = url, - libraryPath = libraryPath, ) } diff --git a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt index bd4cd7e6..cb6a7f52 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/engine/datastore/SlateDBDatastoreCompatibilityTest.kt @@ -14,12 +14,12 @@ import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.io.TempDir import io.slatedb.SlateDb +import io.slatedb.SlateDbConfig /** * SlateDB compatibility test. * * Disabled by default. Set SLATEDB_TEST=true to run. - * Requires native library: native/lib/libslatedb_c.dylib (macOS) or libslatedb_c.so (Linux) * * To run: * SLATEDB_TEST=true ./gradlew :engine:test --tests "*SlateDBDatastoreCompatibilityTest*" @@ -37,7 +37,7 @@ class SlateDBDatastoreCompatibilityTest : DatastoreCompatibilityTest() { ) { assumeTrue(enabled, "SLATEDB_TEST=true not set") tempDir = dir - SlateDb.loadLibrary(findLibraryPath()) + SlateDb.initLogging(SlateDbConfig.LogLevel.INFO) val db = SlateDb.open("data", "file://${tempDir.toAbsolutePath()}", null) table = SlateDbTable.create(db) } @@ -59,20 +59,6 @@ class SlateDBDatastoreCompatibilityTest : DatastoreCompatibilityTest() { } } - private fun findLibraryPath(): String { - var dir = Path.of(System.getProperty("user.dir")) - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { - dir = dir.parent - } - val libName = - if (System.getProperty("os.name").lowercase().contains("linux")) { - "libslatedb_c.so" - } else { - "libslatedb_c.dylib" - } - return dir.resolve("native/lib/$libName").toAbsolutePath().toString() - } - private class SlateDBOperations( private val table: SlateDbTable, ) : StorageOperations { diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt index 6a8ee022..fa6e4d56 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbHashLabelTest.kt @@ -26,7 +26,6 @@ import java.nio.file.Path import java.util.UUID import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir @@ -45,14 +44,6 @@ class SlateDbHashLabelTest { private val storageName = "slatedb_storage" private val labelName = "slatedb_label" - private fun findLibraryPath(): Path { - var dir = Path.of(System.getProperty("user.dir")) - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { - dir = dir.parent - } - return dir.resolve("native/lib/libslatedb_c.dylib") - } - private fun createGraph(): Graph { val config = GraphConfig @@ -64,9 +55,6 @@ class SlateDbHashLabelTest { @BeforeEach fun setUp() { - // Skip test if native library not found - assumeTrue(findLibraryPath().toFile().exists(), "SlateDB native library not found") - graph = createGraph() graph.updateAllMetadata().block() @@ -82,7 +70,6 @@ class SlateDbHashLabelTest { jacksonObjectMapper().createObjectNode().apply { put("path", "test-data") put("url", "file://${tempDir.toAbsolutePath()}") - put("libraryPath", findLibraryPath().toString()) } graph.storageDdl diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt index c6a5f8d5..470ae134 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/label/slatedb/SlateDbIndexedLabelTest.kt @@ -28,7 +28,6 @@ import java.nio.file.Path import java.util.UUID import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir @@ -47,14 +46,6 @@ class SlateDbIndexedLabelTest { private val storageName = "slatedb_indexed_storage" private val labelName = "slatedb_indexed_label" - private fun findLibraryPath(): Path { - var dir = Path.of(System.getProperty("user.dir")) - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { - dir = dir.parent - } - return dir.resolve("native/lib/libslatedb_c.dylib") - } - private fun createGraph(): Graph { val config = GraphConfig @@ -66,9 +57,6 @@ class SlateDbIndexedLabelTest { @BeforeEach fun setUp() { - // Skip test if native library not found - assumeTrue(findLibraryPath().toFile().exists(), "SlateDB native library not found") - graph = createGraph() graph.updateAllMetadata().block() @@ -84,7 +72,6 @@ class SlateDbIndexedLabelTest { jacksonObjectMapper().createObjectNode().apply { put("path", "test-data") put("url", "file://${tempDir.toAbsolutePath()}") - put("libraryPath", findLibraryPath().toString()) } graph.storageDdl diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt index d4d14e6b..54799982 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbStorageTest.kt @@ -49,7 +49,6 @@ class SlateDbStorageTest { jacksonObjectMapper().createObjectNode().apply { put("path", "test-data") put("url", "file:///tmp/slatedb-test") - put("libraryPath", "native/lib/libslatedb_c.dylib") } GraphFixtures.createStorage(graph, "slatedb_test", StorageType.SLATEDB, conf) diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt index 5332be47..9cd26215 100644 --- a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/storage/slatedb/SlateDbTableTest.kt @@ -10,6 +10,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import io.slatedb.SlateDb +import io.slatedb.SlateDbConfig import reactor.test.StepVerifier class SlateDbTableTest { @@ -18,24 +19,9 @@ class SlateDbTableTest { private lateinit var table: SlateDbTable - private fun findLibraryPath(): String { - var dir = Path.of(System.getProperty("user.dir")) - while (!dir.resolve("settings.gradle.kts").toFile().exists() && dir.parent != null) { - dir = dir.parent - } - val libName = - if (System.getProperty("os.name").lowercase().contains("linux")) { - "libslatedb_c.so" - } else { - "libslatedb_c.dylib" - } - return dir.resolve("native/lib/$libName").toAbsolutePath().toString() - } - @BeforeEach fun setUp() { - val libraryPath = findLibraryPath() - SlateDb.loadLibrary(libraryPath) + SlateDb.initLogging(SlateDbConfig.LogLevel.INFO) val fileUrl = "file://${tempDir.toAbsolutePath()}" val dbPath = "data" diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 62905bd0..b512ed04 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -89,8 +89,6 @@ tasks.named("bootRun") { languageVersion.set(JavaLanguageVersion.of(25)) } ) - // Set working directory to project root for SlateDB native library path resolution - workingDir = rootProject.projectDir } jib { diff --git a/server/src/main/resources/application-slatedb.yaml b/server/src/main/resources/application-slatedb.yaml index 223ddda4..9c2537c6 100644 --- a/server/src/main/resources/application-slatedb.yaml +++ b/server/src/main/resources/application-slatedb.yaml @@ -1,9 +1,7 @@ # SlateDB storage profile # Usage: --spring.profiles.active=slatedb # -# Prerequisites: -# 1. Build slatedb-c: ./native/build-slatedb.sh -# 2. Set library path via environment variable or update libraryPath below +# Native library is bundled in the slatedb JAR (loaded automatically from classpath). actionbase: tenant: ab-slatedb @@ -15,4 +13,3 @@ kc: conf: path: data url: file://${SLATEDB_STORAGE_PATH:/tmp/actionbase-slatedb} - libraryPath: ${SLATEDB_LIBRARY_PATH:native/lib/libslatedb_c.dylib} From b120d85735234142f9ec51ccde98673867651462 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 25 Feb 2026 10:28:00 +0900 Subject: [PATCH 08/11] revert: restore FfiController to base branch state Out-of-scope change (import reordering, expression body) that was unrelated to SlateDB integration. --- .../com/kakao/actionbase/server/ffi/FfiController.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt b/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt index 01d7482d..1532cc5f 100644 --- a/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt +++ b/server/src/main/kotlin/com/kakao/actionbase/server/ffi/FfiController.kt @@ -1,16 +1,13 @@ package com.kakao.actionbase.server.ffi import com.kakao.actionbase.v2.engine.ffi.MathOps - -import java.nio.file.Path - +import jakarta.annotation.PreDestroy import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController - -import jakarta.annotation.PreDestroy +import java.nio.file.Path @RestController @ConditionalOnProperty(name = ["ffi.enabled"], havingValue = "true", matchIfMissing = false) @@ -23,7 +20,9 @@ class FfiController( fun add( @RequestParam a: Int, @RequestParam b: Int, - ): Int = mathOps.add(a, b) + ): Int { + return mathOps.add(a, b) + } @PreDestroy fun cleanup() { From c3cc88ac4b492e2aa00c86b2173788d921a33e66 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 25 Feb 2026 10:30:15 +0900 Subject: [PATCH 09/11] chore: remove out-of-scope changes - Restore MathOps.kt and math_ops.c (FFI examples belong to java25 branch) - Remove build-slatedb.sh (obsolete with JAR-bundled native library) --- .../kakao/actionbase/v2/engine/ffi/MathOps.kt | 49 ++++++++ native/build-slatedb.sh | 107 ------------------ native/src/c/math_ops.c | 13 +++ 3 files changed, 62 insertions(+), 107 deletions(-) create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt delete mode 100755 native/build-slatedb.sh create mode 100644 native/src/c/math_ops.c diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt new file mode 100644 index 00000000..85a05d66 --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/ffi/MathOps.kt @@ -0,0 +1,49 @@ +package com.kakao.actionbase.v2.engine.ffi + +import java.lang.foreign.Arena +import java.lang.foreign.FunctionDescriptor +import java.lang.foreign.Linker +import java.lang.foreign.SymbolLookup +import java.lang.foreign.ValueLayout +import java.lang.invoke.MethodHandle +import java.nio.file.Path + +/** + * Java FFI wrapper for native math operations. + * + * Demonstrates Java 22+ Foreign Function & Memory API (JEP 454). + */ +class MathOps private constructor( + private val arena: Arena, + private val addHandle: MethodHandle, +) : AutoCloseable { + + fun add(a: Int, b: Int): Int { + return addHandle.invokeExact(a, b) as Int + } + + override fun close() { + arena.close() + } + + companion object { + fun load(libraryPath: Path): MathOps { + val arena = Arena.ofShared() + val lookup = SymbolLookup.libraryLookup(libraryPath, arena) + val linker = Linker.nativeLinker() + + val descriptor = FunctionDescriptor.of( + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ) + + val addHandle = linker.downcallHandle( + lookup.find("add").orElseThrow { IllegalStateException("Function 'add' not found") }, + descriptor, + ) + + return MathOps(arena, addHandle) + } + } +} diff --git a/native/build-slatedb.sh b/native/build-slatedb.sh deleted file mode 100755 index ead4b107..00000000 --- a/native/build-slatedb.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -SLATEDB_DIR="$SCRIPT_DIR/slatedb" -LIB_DIR="$SCRIPT_DIR/lib" - -# Branch containing slatedb-java (PR #1253) -# TODO: Change to 'main' after PR is merged -SLATEDB_BRANCH="java-bindings" -SLATEDB_REPO="https://github.com/criccomini/slatedb.git" - -# Parse arguments -BUILD_RUST=true -BUILD_JAVA=true -RUN_TESTS=false - -for arg in "$@"; do - case $arg in - --java-only) - BUILD_RUST=false - ;; - --rust-only) - BUILD_JAVA=false - ;; - --test) - RUN_TESTS=true - ;; - esac -done - -# Determine native library name -OS="$(uname -s)" -case "$OS" in - Darwin) NATIVE_LIB="libslatedb_c.dylib" ;; - Linux) NATIVE_LIB="libslatedb_c.so" ;; - *) NATIVE_LIB="" ;; -esac - -# Skip Rust build if native library already exists -if [ "$BUILD_RUST" = true ] && [ -f "$LIB_DIR/$NATIVE_LIB" ]; then - echo "Native library already exists: $LIB_DIR/$NATIVE_LIB" - echo "Skipping Rust build (use 'rm $LIB_DIR/$NATIVE_LIB' to force rebuild)" - BUILD_RUST=false -fi - -# Check for cargo only if we need to build Rust -if [ "$BUILD_RUST" = true ]; then - if ! command -v cargo &> /dev/null; then - echo "Error: cargo is required for building slatedb-c" - echo "" - echo "Install Rust:" - echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" - echo "" - echo "Or run with --java-only to skip Rust build (requires native lib to exist)" - exit 1 - fi -fi - -# Clone if not exists -if [ ! -d "$SLATEDB_DIR" ]; then - echo "Cloning slatedb from $SLATEDB_REPO (branch: $SLATEDB_BRANCH)..." - git clone --depth 1 --branch "$SLATEDB_BRANCH" "$SLATEDB_REPO" "$SLATEDB_DIR" -fi - -mkdir -p "$LIB_DIR" - -# Build slatedb-c -if [ "$BUILD_RUST" = true ]; then - echo "Building slatedb-c..." - cd "$SLATEDB_DIR" - cargo build --release -p slatedb-c - - cp "target/release/$NATIVE_LIB" "$LIB_DIR/" - echo "Built: $LIB_DIR/$NATIVE_LIB" -fi - -# Build slatedb-java -if [ "$BUILD_JAVA" = true ]; then - if [ -d "$SLATEDB_DIR/slatedb-java" ]; then - echo "Building slatedb-java..." - cd "$SLATEDB_DIR/slatedb-java" - - # Set SLATEDB_C_LIB for tests (as per upstream CI) - export SLATEDB_C_LIB="$LIB_DIR/$NATIVE_LIB" - - if [ "$RUN_TESTS" = true ]; then - echo "Running slatedb-java tests..." - ./gradlew check - else - ./gradlew jar --quiet - fi - - # Copy JAR to lib directory - JAR_FILE=$(find build/libs -name "slatedb-*.jar" -not -name "*-sources*" -not -name "*-javadoc*" | head -1) - if [ -n "$JAR_FILE" ]; then - cp "$JAR_FILE" "$LIB_DIR/slatedb.jar" - echo "Built: $LIB_DIR/slatedb.jar" - else - echo "Warning: slatedb-java JAR not found" - fi - else - echo "Warning: slatedb-java directory not found, skipping Java bindings" - fi -fi - -echo "Done." diff --git a/native/src/c/math_ops.c b/native/src/c/math_ops.c new file mode 100644 index 00000000..b89d28bc --- /dev/null +++ b/native/src/c/math_ops.c @@ -0,0 +1,13 @@ +/** + * Simple C library for FFI demonstration. + * + * Compile on macOS: + * clang -shared -o libmathops.dylib math_ops.c + * + * Compile on Linux: + * gcc -shared -fPIC -o libmathops.so math_ops.c + */ + +int add(int a, int b) { + return a + b; +} From 4ba839adb2688dc5dc681791c568dd243d092857 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 25 Feb 2026 10:39:22 +0900 Subject: [PATCH 10/11] chore: restore out-of-scope changes to base branch state - Restore .gitignore (slatedb build artifacts no longer relevant) - Restore engine/build.gradle.kts (remove import reorder and broken native/lib/slatedb.jar file dependency) - Restore server/build.gradle.kts (Java 25 javaLauncher belongs in next/java25 branch) --- .gitignore | 7 +++---- engine/build.gradle.kts | 6 +----- server/build.gradle.kts | 7 +------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index f3b8cb5b..ac6ace56 100644 --- a/.gitignore +++ b/.gitignore @@ -81,7 +81,6 @@ CLAUDE.md !**/.cursor/rules/** engine/target/ -# Native library build artifacts (slatedb) -native/slatedb/ -native/lib/ -!native/lib/.gitkeep +# Native library build artifacts +native/src/c/*.dylib +native/src/c/*.so diff --git a/engine/build.gradle.kts b/engine/build.gradle.kts index c38cccce..c69b7cba 100644 --- a/engine/build.gradle.kts +++ b/engine/build.gradle.kts @@ -1,6 +1,5 @@ -import org.gradle.jvm.toolchain.JavaLanguageVersion - import actionbase.dependencies.Dependencies +import org.gradle.jvm.toolchain.JavaLanguageVersion plugins { id("actionbase.kotlin-conventions") @@ -31,9 +30,6 @@ dependencies { implementation(Dependencies.Cache.CAFFEINE) implementation(Dependencies.Validation.JAKARTA_VALIDATION_API) - // SlateDB Java bindings (built from native/build-slatedb.sh) - implementation(files("${rootProject.projectDir}/native/lib/slatedb.jar")) - // reactor implementation(Dependencies.Reactor.CORE) implementation(Dependencies.Reactor.KOTLIN_EXTENSIONS) diff --git a/server/build.gradle.kts b/server/build.gradle.kts index b512ed04..b2a7a26d 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -1,5 +1,6 @@ import actionbase.BuildParameter import actionbase.dependencies.Dependencies +import org.gradle.jvm.toolchain.JavaLanguageVersion plugins { id("actionbase.kotlin-conventions") @@ -83,12 +84,6 @@ springBoot { tasks.named("bootRun") { jvmArgs("--enable-native-access=ALL-UNNAMED") - // Use Java 25 for SlateDB FFI support - javaLauncher.set( - javaToolchains.launcherFor { - languageVersion.set(JavaLanguageVersion.of(25)) - } - ) } jib { From 8a0cd7788a8f1dcfccc83f5f419918ab51b29b7a Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Wed, 25 Feb 2026 15:45:12 +0900 Subject: [PATCH 11/11] chore(engine): add SlateDB local build script and dependency Add native/build-slatedb-java.sh to clone, build, and publish slatedb-java to Maven local since it is not on Maven Central. Declare the dependency in engine and add mavenLocal() to repo resolution. --- .gitignore | 1 + .../actionbase/BaseConventionsPlugin.kt | 1 + engine/build.gradle.kts | 3 + native/build-slatedb-java.sh | 123 ++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100755 native/build-slatedb-java.sh diff --git a/.gitignore b/.gitignore index ac6ace56..b6747baa 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ engine/target/ # Native library build artifacts native/src/c/*.dylib native/src/c/*.so +native/slatedb/ diff --git a/conventions/src/main/kotlin/actionbase/BaseConventionsPlugin.kt b/conventions/src/main/kotlin/actionbase/BaseConventionsPlugin.kt index 3821a912..5875dabc 100644 --- a/conventions/src/main/kotlin/actionbase/BaseConventionsPlugin.kt +++ b/conventions/src/main/kotlin/actionbase/BaseConventionsPlugin.kt @@ -24,6 +24,7 @@ class BaseConventionsPlugin : Plugin { project.version = project.rootProject.version // Configure repositories + project.repositories.mavenLocal() project.repositories.mavenCentral() // Configure dependencies diff --git a/engine/build.gradle.kts b/engine/build.gradle.kts index c69b7cba..390cdb46 100644 --- a/engine/build.gradle.kts +++ b/engine/build.gradle.kts @@ -41,6 +41,9 @@ dependencies { implementation(Dependencies.Logging.SLF4J_API) implementation(Dependencies.Logging.LOGBACK_CLASSIC) + // SlateDB + implementation("io.slatedb:slatedb:0.1.0-SNAPSHOT") + // HBase implementation(Dependencies.HBase.CLIENT) implementation(Dependencies.HBase.MAPREDUCE) diff --git a/native/build-slatedb-java.sh b/native/build-slatedb-java.sh new file mode 100755 index 00000000..4033fb4b --- /dev/null +++ b/native/build-slatedb-java.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# +# Build slatedb-java from source and publish to Maven local. +# +# This script clones the slatedb repository (java-include-libs branch), +# builds the Java JAR with bundled native libraries, and publishes to +# ~/.m2/repository so Gradle can resolve it as a local dependency. +# +# Requirements: +# - Rust toolchain (cargo) +# - Java 24+ (for jextract / FFI support) +# - Git +# +# Usage: +# ./native/build-slatedb-java.sh # clone + build + publish +# ./native/build-slatedb-java.sh --clean # remove clone and rebuild +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SLATEDB_DIR="${SCRIPT_DIR}/slatedb" +SLATEDB_JAVA_DIR="${SLATEDB_DIR}/slatedb-java" +REPO_URL="https://github.com/slatedb/slatedb.git" +BRANCH="java-include-libs" + +# --- Parse arguments --- + +if [[ "${1:-}" == "--clean" ]]; then + echo "Cleaning previous clone..." + rm -rf "${SLATEDB_DIR}" +fi + +# --- Prerequisites --- + +check_command() { + if ! command -v "$1" &>/dev/null; then + echo "Error: '$1' is required but not found in PATH." + exit 1 + fi +} + +check_command git +check_command cargo +check_command java + +JAVA_VERSION=$(java -version 2>&1 | head -1 | sed -E 's/.*"([0-9]+).*/\1/') +if [[ "${JAVA_VERSION}" -lt 24 ]]; then + echo "Error: Java 24+ is required for slatedb-java (found Java ${JAVA_VERSION})." + echo "Set JAVA_HOME to a Java 24+ installation." + exit 1 +fi + +echo "Prerequisites OK: git, cargo, java ${JAVA_VERSION}" + +# --- Clone or update --- + +if [[ -d "${SLATEDB_DIR}" ]]; then + echo "Using existing clone at ${SLATEDB_DIR}" + cd "${SLATEDB_DIR}" + git fetch origin "${BRANCH}" + git checkout "${BRANCH}" + git reset --hard "origin/${BRANCH}" +else + echo "Cloning ${REPO_URL} (branch: ${BRANCH})..." + git clone --branch "${BRANCH}" --depth 1 "${REPO_URL}" "${SLATEDB_DIR}" +fi + +# --- Build --- + +echo "Building slatedb-java..." +cd "${SLATEDB_JAVA_DIR}" +./gradlew build -x test + +# --- Publish to Maven local --- +# +# The upstream build.gradle does not include the maven-publish plugin, +# so we inject it via a Gradle init script. + +INIT_SCRIPT=$(mktemp) +trap 'rm -f "${INIT_SCRIPT}"' EXIT + +cat > "${INIT_SCRIPT}" << 'INIT' +allprojects { + apply plugin: 'maven-publish' + + afterEvaluate { + publishing { + publications { + maven(MavenPublication) { + from components.java + } + } + } + } + + // Suppress Gradle module metadata so consumers don't see the + // org.gradle.jvm.version=24 attribute, which would block resolution + // from projects using a lower toolchain version. + tasks.withType(GenerateModuleMetadata) { + enabled = false + } +} +INIT + +echo "Publishing to Maven local..." +./gradlew publishToMavenLocal --init-script "${INIT_SCRIPT}" + +# --- Verify --- + +GAV="io.slatedb:slatedb:0.1.0-SNAPSHOT" +LOCAL_REPO="${HOME}/.m2/repository" +ARTIFACT_DIR="${LOCAL_REPO}/io/slatedb/slatedb/0.1.0-SNAPSHOT" + +if [[ -d "${ARTIFACT_DIR}" ]]; then + echo "" + echo "Published ${GAV} to Maven local:" + ls -la "${ARTIFACT_DIR}"/*.jar 2>/dev/null || true + echo "" + echo "Done. You can now run: ./gradlew :engine:build" +else + echo "Error: Expected artifact not found at ${ARTIFACT_DIR}" + exit 1 +fi