diff --git a/doscript/README.md b/doscript/README.md new file mode 100644 index 00000000..092226e3 --- /dev/null +++ b/doscript/README.md @@ -0,0 +1,66 @@ +# zvec DoScript Workflows + +This folder contains [DoScript](https://github.com/TheServer-lab/DoScript) automation scripts for common zvec developer tasks. + +## ⚠️ Important: Run from the project root + +All scripts must be run from the **zvec project root directory**, not from inside the `doscript/` folder. This ensures relative paths like `build/`, `data/`, and `bench_results/` all resolve correctly. + +``` +cd C:\path\to\zvec-0.2.0 +do doscript\build.do +``` + +Each script captures the working directory at startup (`cwd = capture "cd"`) so all file operations use consistent absolute paths. + +## Scripts + +| Script | Description | +|---|---| +| `build.do` | Configure and build the project | +| `test.do` | Run C++ unit tests, Python tests, or both | +| `format.do` | Check or auto-fix code formatting (clang-format + ruff) | +| `benchmark.do` | Download datasets and run index benchmarks | +| `coverage.do` | Build with coverage flags and generate HTML report | +| `release.do` | Package a release zip with binaries, headers, and optional Python wheel | +| `clean.do` | Remove build artifacts | + +## Usage + +### Interactive (prompts for input) +``` +do doscript\build.do +do doscript\test.do +do doscript\benchmark.do +``` + +### With CLI args (no prompts) +``` +do doscript\build.do Release 8 +do doscript\test.do all +do doscript\clean.do all +do doscript\release.do 0.2.0 n +do doscript\benchmark.do hnsw sift 10 +``` + +### Dry-run (preview without executing) +``` +do doscript\release.do 0.2.0 n --dry-run +``` + +## DoScript syntax reference + +| Syntax | Meaning | +|---|---| +| `global_variable = a, b` | Declare variables | +| `var = capture "cmd"` | Run command and store output in var | +| `run "cmd"` | Execute shell command | +| `run 'cmd {var}'` | Execute with variable interpolation (single quotes) | +| `copy "src" to "dst"` | Copy file | +| `zip "folder" to "archive.zip"` | Create zip archive | +| `download "url" to "path"` | Download file | +| `if x == "y" / else / end_if` | Conditionals (`==` not `=`) | +| `arg1`..`arg32` | CLI arguments passed after script name | +| `say 'Hello {name}!'` | Print with interpolation | +| `ask varname "prompt"` | Prompt user for input | +| `# comment` or `// comment` | Line comments | diff --git a/doscript/benchmark.do b/doscript/benchmark.do new file mode 100644 index 00000000..241b8f9e --- /dev/null +++ b/doscript/benchmark.do @@ -0,0 +1,63 @@ +# zvec benchmark runner +# Usage: do doscript\benchmark.do [hnsw|ivf|flat] [sift|random] [topk] + +global_variable = index_type, dataset, topk, root, cmake_check + +// Auto-detect project root: check for CMakeLists.txt in CWD +root = capture "cd" +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root (e.g. C:\Users\User\zvec-0.2.0):" +end_if + +say 'Using project root: {root}' + +if arg1 == "" + ask index_type "Index type? (hnsw/ivf/flat):" +else + index_type = arg1 +end_if + +if arg2 == "" + ask dataset "Dataset? (sift/random):" +else + dataset = arg2 +end_if + +if arg3 == "" + ask topk "Top-K results? (e.g. 10):" +else + topk = arg3 +end_if + +if index_type == "" + index_type = "hnsw" +end_if + +if dataset == "" + dataset = "sift" +end_if + +if topk == "" + topk = "10" +end_if + +run 'mkdir "{root}\data" 2>nul & mkdir "{root}\bench_results" 2>nul' + +if dataset == "sift" + say "Downloading SIFT-128 dataset..." + download "ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz" to '{root}\data\sift.tar.gz' + run 'tar -xzf "{root}\data\sift.tar.gz" -C "{root}\data"' +end_if + +if dataset == "random" + say "Generating random test vectors..." + run 'py -c "import numpy as np; np.random.rand(10000,128).astype(\"float32\").tofile(r\"{root}\data\\random_base.fvecs\")"' +end_if + +say 'Running {index_type} benchmark (top-{topk})...' +run '"{root}\build\tools\bench" --index {index_type} --data "{root}\data" --topk {topk} --output "{root}\bench_results\result_{index_type}.json"' + +say 'Benchmark complete! Results in {root}\bench_results\result_{index_type}.json' diff --git a/doscript/build.do b/doscript/build.do new file mode 100644 index 00000000..891fb1fb --- /dev/null +++ b/doscript/build.do @@ -0,0 +1,91 @@ +# zvec build script - FIXED with single quotes +global_variable = build_type, jobs, root, cmake_check, submodule_check + +say "═══════════════════════════════════" +say " zvec Build Script" +say "═══════════════════════════════════" + +# Auto-detect project root +root = capture "cd" +cmake_check = capture 'if exist "CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter zvec project root:" +end_if + +say 'Using project root: {root}' # ✅ Single quotes! +say "" + +# Get build type +if arg1 == "" + ask build_type "Build type? (Release/Debug/RelWithDebInfo):" +else + build_type = arg1 +end_if + +if build_type == "" + build_type = "Release" +end_if + +# Get parallel jobs +if arg2 == "" + ask jobs "Parallel jobs? (e.g. 8):" +else + jobs = arg2 +end_if + +if jobs == "" + jobs = "8" +end_if + +say "" +say "Configuration:" +say ' Build Type: {build_type}' # ✅ Single quotes! +say ' Jobs: {jobs}' # ✅ Single quotes! +say "" + +# Check for git submodules +say "Checking dependencies..." +submodule_check = capture 'if exist "{root}\thirdparty\googletest\googletest-1.10.0\CMakeLists.txt" (echo yes) else (echo no)' + +if submodule_check == "no" + say "→ Git submodules not found" + say "Please run manually:" + say ' cd {root}' # ✅ Single quotes! + say " git submodule update --init --recursive" + say "" + say "Or download zvec with dependencies included" + exit 1 +else + say "✓ Dependencies present" +end_if + +# Create build directory +say "→ Creating build directory..." +run 'mkdir "{root}\build" 2>nul' + +# Configure CMake +say '→ Configuring CMake ({build_type})...' +run 'cmake -S "{root}" -B "{root}\build" -DCMAKE_BUILD_TYPE={build_type} -DBUILD_TOOLS=ON' + +# Check if CMake succeeded +cmake_cache_check = capture 'if exist "{root}\build\CMakeCache.txt" (echo yes) else (echo no)' +if cmake_cache_check == "no" + say "" + say "✗✗✗ CMake Configuration FAILED ✗✗✗" + say "Check error messages above for details." + exit 1 +end_if + +say "✓ CMake configured successfully" + +# Build +say '→ Building with {jobs} parallel jobs...' +run 'cmake --build "{root}\build" --config {build_type} --parallel {jobs}' + +say "" +say "═══════════════════════════════════" +say "✓ BUILD COMPLETE!" +say ' Binaries: {root}\build\{build_type}' +say "═══════════════════════════════════" diff --git a/doscript/clean.do b/doscript/clean.do new file mode 100644 index 00000000..f13474df --- /dev/null +++ b/doscript/clean.do @@ -0,0 +1,50 @@ +# zvec clean +# Usage: do doscript\clean.do [build|all] + +global_variable = level, root, cmake_check + +// Auto-detect project root: check for CMakeLists.txt in CWD +root = capture "cd" +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root:" +end_if + +say 'Using project root: {root}' + +if arg1 == "" + ask level "Clean level? (build/all):" +else + level = arg1 +end_if + +if level == "" + level = "build" +end_if + +if level == "build" + say "Removing build directory..." + delete folder "build" + say "Build directory cleaned" +end_if + +if level == "all" + say "Removing build directory..." + delete folder "build" + say "Removing dist directory..." + delete folder "dist" + say "Removing data directory..." + delete folder "data" + say "Removing bench_results..." + delete folder "bench_results" + say "Removing Python cache..." + // Windows: Remove __pycache__ directories + run 'for /d /r . %%d in (__pycache__) do @if exist "%%d" rmdir /s /q "%%d" 2>nul' + // Remove .pyc files + run 'del /s /q *.pyc 2>nul' + // Remove .egg-info directories + run 'for /d /r . %%d in (*.egg-info) do @if exist "%%d" rmdir /s /q "%%d" 2>nul' + say "Full clean complete" +end_if diff --git a/doscript/coverage.do b/doscript/coverage.do new file mode 100644 index 00000000..3ed75a35 --- /dev/null +++ b/doscript/coverage.do @@ -0,0 +1,46 @@ +# zvec coverage report +# Usage: do doscript\coverage.do [open|noopen] +# Note: Requires gcov/lcov (typically available in Git Bash or WSL on Windows) + +global_variable = open_report, root, cmake_check + +// Auto-detect project root: check for CMakeLists.txt in CWD +root = capture "cd" +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root:" +end_if + +say 'Using project root: {root}' + +if arg1 == "" + ask open_report "Open HTML report when done? (y/n):" +else + open_report = arg1 +end_if + +say "Cleaning previous build..." +delete folder "build" +run 'mkdir "{root}\build" 2>nul' + +say "Configuring with coverage flags..." +run 'cmake -S "{root}" -B "{root}\build" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_C_FLAGS=--coverage -DBUILD_TOOLS=ON' + +say "Building..." +run 'cmake --build "{root}\build" --parallel 8' + +say "Running unit tests..." +run 'cmake --build "{root}\build" --target unittest --parallel 8' + +say "Generating coverage data..." +say "NOTE: Coverage generation requires bash/gcov (Git Bash or WSL)" +// Try Git Bash first, then WSL +run 'bash "{root}\scripts\gcov.sh" -t gcov -o "{root}\build\coverage_html" || wsl bash "{root}/scripts/gcov.sh" -t gcov -o "{root}/build/coverage_html"' + +say 'Coverage report ready at {root}\build\coverage_html\index.html' + +if open_report == "y" + open_link 'file:///{root}/build/coverage_html/index.html' +end_if diff --git a/doscript/format.do b/doscript/format.do new file mode 100644 index 00000000..27871e17 --- /dev/null +++ b/doscript/format.do @@ -0,0 +1,46 @@ +# zvec code formatter +# Usage: do doscript\format.do [check|fix] + +global_variable = mode, root, cmake_check + +// Auto-detect project root: use CWD if CMakeLists.txt is there, else ask +root = capture "cd" + +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root:" +end_if + +say 'Using project root: {root}' + +if arg1 == "" + ask mode "Mode? (check/fix):" +else + mode = arg1 +end_if + +if mode == "" + mode = "check" +end_if + +if mode == "check" + say "Checking Python files with ruff..." + run "ruff check ." + run "ruff format --check ." + say "Checking C++ files with clang-format..." + // Windows: Use for loop to check C++ files + run 'for /r src %%f in (*.cc *.h *.cpp *.hpp) do @clang-format --dry-run --Werror "%%f"' + say "Format check passed - no changes needed" +end_if + +if mode == "fix" + say "Fixing Python files with ruff..." + run "ruff check --fix ." + run "ruff format ." + say "Fixing C++ files with clang-format..." + // Windows: Use for loop to format C++ files + run 'for /r src %%f in (*.cc *.h *.cpp *.hpp) do @clang-format -i "%%f"' + say "All formatting applied" +end_if diff --git a/doscript/release.do b/doscript/release.do new file mode 100644 index 00000000..2072b47e --- /dev/null +++ b/doscript/release.do @@ -0,0 +1,70 @@ +# zvec release packager +# Usage: do doscript\release.do [version] [y|n for python wheel] + +global_variable = version, include_python, root, cmake_check + +// Auto-detect project root: check for CMakeLists.txt in CWD +root = capture "cd" +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root:" +end_if + +say 'Using project root: {root}' + +if arg1 == "" + ask version "Release version tag? (e.g. 0.2.0):" +else + version = arg1 +end_if + +if arg2 == "" + ask include_python "Include Python wheel? (y/n):" +else + include_python = arg2 +end_if + +if version == "" + version = "dev" +end_if + +say "Building release binaries..." +run 'mkdir "{root}\build" 2>nul' +run 'cmake -S "{root}" -B "{root}\build" -DCMAKE_BUILD_TYPE=Release -DBUILD_TOOLS=ON -DBUILD_PYTHON_BINDINGS=OFF' +run 'cmake --build "{root}\build" --config Release --parallel 8' + +say 'Preparing dist\zvec-{version}...' +run 'mkdir "{root}\dist\zvec-{version}" 2>nul' +run 'mkdir "{root}\dist\zvec-{version}\lib" 2>nul' +run 'mkdir "{root}\dist\zvec-{version}\include" 2>nul' +run 'mkdir "{root}\dist\zvec-{version}\tools" 2>nul' + +say "Copying library files..." +// Windows: Copy lib files from Release or Debug directory +run 'xcopy /Y /Q "{root}\build\src\Release\*.lib" "{root}\dist\zvec-{version}\lib\" 2>nul || xcopy /Y /Q "{root}\build\src\*.lib" "{root}\dist\zvec-{version}\lib\" 2>nul' +run 'xcopy /Y /Q "{root}\build\src\Release\*.dll" "{root}\dist\zvec-{version}\lib\" 2>nul || xcopy /Y /Q "{root}\build\src\*.dll" "{root}\dist\zvec-{version}\lib\" 2>nul' + +say "Copying headers..." +run 'xcopy /E /I /Y /Q "{root}\src\include\zvec" "{root}\dist\zvec-{version}\include\zvec\"' + +say "Copying tools..." +run 'copy /Y "{root}\build\tools\Release\bench.exe" "{root}\dist\zvec-{version}\tools\" 2>nul || copy /Y "{root}\build\tools\bench.exe" "{root}\dist\zvec-{version}\tools\" 2>nul' +run 'copy /Y "{root}\build\tools\Release\txt2vecs.exe" "{root}\dist\zvec-{version}\tools\" 2>nul || copy /Y "{root}\build\tools\txt2vecs.exe" "{root}\dist\zvec-{version}\tools\" 2>nul' + +say "Copying docs..." +copy "README.md" to 'dist\zvec-{version}\README.md' +copy "LICENSE" to 'dist\zvec-{version}\LICENSE' + +if include_python == "y" + say "Building Python wheel..." + run "pip install build" + run "py -m build --wheel" + run 'copy /Y "{root}\dist\*.whl" "{root}\dist\zvec-{version}\" 2>nul' +end_if + +say "Creating archive..." +zip 'dist\zvec-{version}' to 'dist\zvec-{version}.zip' + +say 'Release package ready: dist\zvec-{version}.zip' diff --git a/doscript/test.do b/doscript/test.do new file mode 100644 index 00000000..febe55f7 --- /dev/null +++ b/doscript/test.do @@ -0,0 +1,51 @@ +# zvec test runner +# Usage: python doscript.py test.do [cpp|python|all] + +global_variable = suite, root, cmake_check + +// Auto-detect project root: check for CMakeLists.txt in CWD +root = capture "cd" +cmake_check = capture 'if exist "{root}\CMakeLists.txt" (echo yes) else (echo no)' + +if cmake_check == "no" + say "CMakeLists.txt not found in current directory." + ask root "Enter full path to zvec project root (e.g. C:\Users\User\zvec-0.2.0):" +end_if + +say 'Using project root: {root}' + + + +if arg1 == "" + ask suite "Which tests? (cpp/python/all):" +else + suite = arg1 +end_if + +if suite == "" + suite = "all" +end_if + +if suite == "cpp" + say "Running C++ unit tests..." + run 'cmake --build "{root}\build" --target unittest --parallel 8' + say "C++ tests complete" +end_if + +if suite == "python" + say "Installing Python package in dev mode..." + run "pip install -e python/" + say "Running Python tests..." + run "py -m pytest python/tests/ -v" + say "Python tests complete" +end_if + +if suite == "all" + say "Running C++ unit tests..." + run 'cmake --build "{root}\build" --target unittest --parallel 8' + say "Installing Python package in dev mode..." + run "pip install -e python/" + say "Running Python tests with coverage..." + run "py -m pytest python/tests/ --cov=zvec --cov-report=term-missing -v" + say "All tests complete" +end_if