From e56e6a2e489eb6881ccce455d9e49bd1cb861e85 Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:04:32 -0600 Subject: [PATCH 1/8] chore: adding hermetic build containers --- .github/workflows/go.yaml | 27 ++- .github/workflows/release.yaml | 43 +++++ Makefile | 68 +++++++- README.md | 291 +++++++++++++++++++++++++++++---- haystack_test.go | 1 - scripts/docker-build.sh | 208 +++++++++++++++++++++++ scripts/docker-tags.sh | 195 ++++++++++++++++++++++ scripts/tree-hash.sh | 69 ++++++++ 8 files changed, 868 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/release.yaml delete mode 100644 haystack_test.go create mode 100755 scripts/docker-build.sh create mode 100755 scripts/docker-tags.sh create mode 100755 scripts/tree-hash.sh diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index a671469..0bf082a 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.4" + go-version: "1.24.6" - name: Check formatting run: | @@ -55,3 +55,28 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: bash <(curl -s https://codecov.io/bash) + + docker-build: + runs-on: ubuntu-latest + needs: [quality-checks, test] + # Only build and push Docker images on main branch after tests pass + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build info + run: make docker-info + + - name: Build and push Docker image + run: make docker-push + env: + DOCKER_PUSH: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..3415d9a --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,43 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + promote-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Build info + run: | + echo "Promoting release: ${{ steps.version.outputs.VERSION }}" + make docker-info + + - name: Promote Docker image + run: | + # The image should already exist from the main branch build + # We're just adding the version tag to it (immutable tags - no latest) + make docker-promote TAGS="${{ steps.version.outputs.VERSION }}" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + generate_release_notes: true + draft: false + prerelease: false \ No newline at end of file diff --git a/Makefile b/Makefile index 32fff9f..fee02f9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,14 @@ -.PHONEY: coverage test inspect install sec-scan lint fmt check +.PHONY: coverage test inspect install sec-scan lint fmt check +.PHONY: tree-hash docker-build docker-push docker-exists docker-promote docker-release docker-list +# Docker configuration with defaults (override with environment variables) +export DOCKER_REGISTRY ?= docker.io +export DOCKER_REPO ?= nomasters/haystack +export DOCKER_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 +export DOCKER_PUSH ?= false +export SKIP_GIT_CHECK ?= false + +# Go targets test: go test -v ./... @@ -25,4 +34,59 @@ update-deps: go get -u && go mod tidy install: - go install github.com/nomasters/haystack/cmd/haystack \ No newline at end of file + go install github.com/nomasters/haystack/cmd/haystack + +# Docker targets - Local-first hermetic builds +# These commands work identically on your machine and in CI + +# Calculate the tree hash of source files +tree-hash: + @./scripts/tree-hash.sh + +# Build Docker image (idempotent - only builds if tree hash changed) +# Requires clean git working directory (override with SKIP_GIT_CHECK=true) +docker-build: + @./scripts/docker-build.sh + +# Build and push Docker image to registry +docker-push: + @DOCKER_PUSH=true ./scripts/docker-build.sh + +# Check if image exists for current tree hash +docker-exists: + @TREE_HASH=$$(./scripts/tree-hash.sh) && \ + ./scripts/docker-tags.sh exists "tree-$$TREE_HASH" + +# Promote current commit's image with additional tags +# Usage: make docker-promote TAGS="v1.0.0" +docker-promote: + @if [ -z "$(TAGS)" ]; then \ + echo "Error: TAGS variable required. Usage: make docker-promote TAGS=\"v1.0.0\""; \ + exit 1; \ + fi + @./scripts/docker-tags.sh release $(TAGS) + +# Release workflow - build if needed, then tag +# Usage: make docker-release VERSION=v1.0.0 +docker-release: docker-push + @if [ -z "$(VERSION)" ]; then \ + echo "Error: VERSION variable required. Usage: make docker-release VERSION=v1.0.0"; \ + exit 1; \ + fi + @./scripts/docker-tags.sh release $(VERSION) + +# List all Docker tags in the registry +docker-list: + @./scripts/docker-tags.sh list + +# Clean up local Docker buildx builders +docker-clean: + @docker buildx rm haystack-builder 2>/dev/null || true + +# Show current build information +docker-info: + @echo "Tree hash: $$(./scripts/tree-hash.sh)" + @echo "Commit SHA: $$(git rev-parse --short HEAD)" + @echo "Branch: $$(git rev-parse --abbrev-ref HEAD)" + @echo "Registry: $${DOCKER_REGISTRY:-docker.io}" + @echo "Repository: $${DOCKER_REPO:-nomasters/haystack}" \ No newline at end of file diff --git a/README.md b/README.md index 59a8b2f..a6b4c7e 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,284 @@ # haystack -an ephemeral, quiet and tiny content addressed key/value store. +A tiny, ephemeral, quiet, content-addressed key/value store. -Design goals: -- no auth. the submitter declares the hash of the content and submits the content. if the hash of the content on the server matches the submitted hash, the content is accepted and any requestor with the hash can request the content -- ephemeral. key/value is short lived, no keys will live longer than a window (TBD) and this is defined by the server. Content can be resubmitted to reset the counter -- consistent payload and response to make implementation details indistinguishable to an adversary -- quiet. No network traffic happens that isn't absolutely needed. +## Overview +Haystack is a minimalist key/value store built for simplicity and efficiency. It operates over UDP with fixed-size messages, uses content-addressing via SHA256 hashes, and automatically expires data after a configurable time window. +## Key Features -goals: -- easy to use client and server -- fast -- simple +- **Content-Addressed**: Keys are SHA256 hashes of the content itself - no separate key management needed +- **Fixed-Size Messages**: All network messages are exactly 192 bytes (writes) or 32 bytes (reads) +- **Ephemeral Storage**: Data automatically expires after a configurable TTL (default 24 hours) +- **UDP-Only**: Lightweight, stateless protocol with minimal overhead +- **No Authentication**: Content validity is proven by SHA256 hash matching +- **Zero Server Confirmation**: Write operations receive no response by design -### Needle: the Haystack message +## Installation -A message in haystack is called a Needle. A needle includes a 32 bytes sha256 hash of a 160 byte payload. -All payloads are a fixed length. Both the client and server can verify the hash of the payload by verifying a 192 byte needle. +### From Source +```bash +# Clone the repository +git clone https://github.com/nomasters/haystack.git +cd haystack + +# Build and install +make install +``` + +### Using Go + +```bash +go install github.com/nomasters/haystack@latest +``` + +## Quick Start + +### Start a Server + +```bash +# Default configuration (localhost:1337) +haystack serve + +# Custom host and port +haystack serve -H 0.0.0.0 -p 9000 +``` + +### Client Operations + +```bash +# Set a value (returns the SHA256 hash) +haystack client set "hello world" + +# Get a value using its hash +haystack client get + +# Use pipes for content +echo "my message" | haystack client set - +``` + +### Go Client Library + +```go +package main + +import ( + "context" + "fmt" + "github.com/nomasters/haystack/client" +) + +func main() { + // Create a client + c, err := client.New("localhost:1337") + if err != nil { + panic(err) + } + defer c.Close() + + // Store data + hash, err := c.Set(context.Background(), []byte("hello world")) + if err != nil { + panic(err) + } + + // Retrieve data + data, err := c.Get(context.Background(), hash) + if err != nil { + panic(err) + } + + fmt.Printf("Retrieved: %s\n", data) +} +``` + +## The Needle Protocol + +A message in Haystack is called a "Needle". Each needle consists of a 32-byte SHA256 hash followed by a 160-byte payload, totaling exactly 192 bytes. + +```text +| hash | payload | +|----------------|--------------------------------------------------------------------------------| +| 32 bytes | 160 bytes | +``` + +This fixed size enables: + +- Single UDP packet transmission +- Consistent network behavior +- Message chaining for larger payloads + +### Message Chaining Example + +The 160-byte payload is large enough to contain encrypted data and a reference to the next chunk: + +```text +| needle | +|--------------------------------------------------------------------------------------------------| +| hash | payload | +|----------------|---------------------------------------------------------------------------------| +| | nonce | encrypted payload | +| |------------|--------------------------------------------------------------------| +| | | next key | padded message | +| | |----------------|---------------------------------------------------| +| 32 bytes | 24 bytes | 32 bytes | 104 bytes | +``` + +## Protocol Operations + +### Read Request (32 bytes) + +Send a 32-byte SHA256 hash to retrieve the associated needle. If found, the server responds with the complete 192-byte needle. + +### Write Request (192 bytes) + +Send a complete 192-byte needle. The server validates that the hash matches the payload and stores it if valid. No response is sent (by design). + +## Architecture + +Haystack follows a modular design with clear separation of concerns: + +- **Needle Package**: Core message structure and validation +- **Storage Package**: Interface-based storage abstraction with TTL support +- **Server Package**: High-performance UDP server with zero-copy buffer pools +- **Client Package**: Production-ready Go client with connection pooling and retry logic + +## Deployment + +### Docker Images + +Haystack uses a hermetic build system with content-based tagging. Images are built once and promoted through their lifecycle. + +#### Pull and Run + +```bash +# Run the latest build from main branch +docker run -p 1337:1337/udp nomasters/haystack:main serve + +# Run a specific version (immutable tags) +docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve + +# Run with custom configuration +docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve -H 0.0.0.0 -p 1337 +``` + +#### Available Tags + +- `sha-`: Specific commit SHA +- `tree-`: Content-based hash (same source = same tag) +- `main`: Latest build from main branch +- `v0.1.0`: Specific release version (immutable) + +### Deploy to Fly.io + +Create a `fly.toml` configuration: + +```toml +app = "haystack" + +[build] + image = "nomasters/haystack:v0.1.0" + +[[services]] + internal_port = 1337 + protocol = "udp" + + [[services.ports]] + port = 1337 ``` -hash | payload ----------|---------- -32 bytes | 160 bytes + +Then deploy: + +```bash +fly deploy ``` -The message size is small, but designed to allow _single UDP packet_ message transmission. This is meant to be light weight and efficient. +## Development + +### Building from Source + +```bash +# Install locally +make install +``` + +### Docker Development + +The build system uses content-based hashing to ensure hermetic, reproducible builds. By default, builds require a clean git working directory to ensure only committed code is deployed. + +```bash +# Show current build info +make docker-info + +# Build locally (requires clean git working directory) +make docker-build + +# Build with uncommitted changes (local testing only) +SKIP_GIT_CHECK=true make docker-build + +# Build and push to registry (always requires clean git) +make docker-push + +# Check if image exists for current source +make docker-exists + +# Promote image with version tag +make docker-promote TAGS="v0.1.0" +``` + +### Testing + +```bash +# Run tests with race detection +make test + +# Generate coverage report +make coverage + +# Run specific tests +go test -v -run TestName ./path/to/package +``` + +### Project Structure + +```text +haystack/ +├── needle/ # Core message format +├── storage/ # Storage interface and implementations +├── server/ # UDP server implementation +├── client/ # Go client library +└── cmd/ # CLI implementation +``` -While it is small, it is large enough to "chain" messages together. Such patterns must be configured client-side, but a hypothetical payload with an encrypted message +## Design Philosophy -This is large enough for the value to contain something like: +Haystack embraces several core principles: -| nonce | encrypted payload | -| |------------------------------| -| | next key | padded message | -|----------|-------------|----------------| -| 24 bytes | 32 bytes | bytes 104 | +1. **Simplicity**: Minimal protocol, clear semantics, predictable behavior +2. **Performance**: Zero-copy operations, buffer pooling, efficient packet handling +3. **Reliability**: Content-addressed storage ensures data integrity +4. **Efficiency**: Fixed-size messages and stateless UDP protocol +## Use Cases +- Temporary data storage with automatic expiration +- Distributed caching +- Message passing between services +- Content-addressed storage needs +- Lightweight key/value operations -### Reads and Writes +## Contributing -A Haystack server accepts two and only two byte length requests 32 bytes and 192 bytes. +Contributions are welcome! Please ensure: -#### Read Requests +- All tests pass with race detection enabled +- Code follows standard Go conventions +- New features include appropriate tests +- Changes maintain backward compatibility -A 32 byte request is a read request. It implies that the requestor would like the stored value of the Needle. If the Haystack server has a Needle that matches the request hash, it will respond with the full Needle in the response. +## License -#### Write Requests +The Unlicense - This is free and unencumbered software released into the public domain. See [LICENSE](LICENSE) for details. -A write request my be 192 bytes. The server will verify that these bytes are a valid Needles (that the final 160 bytes sha256 hash match the first 32 bytes hash included in the payload). If this Needle is valid, it is stored. The server provides no response for this operation. If a client wants to confirm that a write was completed successfully, it should submit a read request to confirm. \ No newline at end of file diff --git a/haystack_test.go b/haystack_test.go deleted file mode 100644 index 691492e..0000000 --- a/haystack_test.go +++ /dev/null @@ -1 +0,0 @@ -package haystack diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh new file mode 100755 index 0000000..426ebfa --- /dev/null +++ b/scripts/docker-build.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# +# docker-build.sh - Hermetic Docker build with content-based tagging +# +# This script implements idempotent Docker builds that: +# 1. Check if an image with the current tree hash already exists +# 2. Build only if necessary +# 3. Tag with both tree hash and commit SHA + +set -euo pipefail + +# Load configuration from environment or use defaults +DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" +DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" +DOCKER_PLATFORMS="${DOCKER_PLATFORMS:-linux/amd64,linux/arm64,linux/arm/v7}" +DOCKER_PUSH="${DOCKER_PUSH:-false}" +SKIP_GIT_CHECK="${SKIP_GIT_CHECK:-false}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Get the current tree hash +get_tree_hash() { + local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + "${script_dir}/tree-hash.sh" +} + +# Get the current git commit SHA (short) +get_commit_sha() { + git rev-parse --short HEAD +} + +# Get the current branch name +get_branch_name() { + git rev-parse --abbrev-ref HEAD +} + +# Check if a Docker image exists in the registry +image_exists() { + local image="$1" + + # Try to inspect the manifest + if docker manifest inspect "$image" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# Build multi-platform Docker image +build_image() { + local tree_hash="$1" + local commit_sha="$2" + local branch="$3" + + local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" + local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" + local branch_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${branch}" + + log_info "Building multi-platform image..." + log_info "Platforms: ${DOCKER_PLATFORMS}" + log_info "Tags: tree-${tree_hash}, sha-${commit_sha}, ${branch}" + + # Ensure buildx is available + if ! docker buildx version >/dev/null 2>&1; then + log_error "Docker buildx is not available. Please install Docker with buildx support." + exit 1 + fi + + # Create or use existing buildx builder + if ! docker buildx ls | grep -q "haystack-builder"; then + log_info "Creating buildx builder..." + docker buildx create --name haystack-builder --use + else + docker buildx use haystack-builder + fi + + # Build the image with all tags + local build_args=( + "--platform=${DOCKER_PLATFORMS}" + "--tag=${tree_tag}" + "--tag=${sha_tag}" + "--tag=${branch_tag}" + ) + + # Add push flag if requested + if [ "${DOCKER_PUSH}" = "true" ]; then + build_args+=("--push") + else + build_args+=("--load") + log_warn "Building for local platform only (--load mode). Set DOCKER_PUSH=true for multi-platform push." + build_args=("--tag=${tree_tag}" "--tag=${sha_tag}" "--tag=${branch_tag}") + fi + + docker buildx build "${build_args[@]}" . + + log_info "Build complete!" +} + +# Add additional tags to an existing image +add_tags() { + local source_tag="$1" + local commit_sha="$2" + local branch="$3" + + local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" + local branch_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${branch}" + + log_info "Image already exists with tree hash, adding new tags..." + + if [ "${DOCKER_PUSH}" = "true" ]; then + # Pull the existing image + docker pull "$source_tag" + + # Tag with commit SHA + docker tag "$source_tag" "$sha_tag" + docker push "$sha_tag" + + # Tag with branch name + docker tag "$source_tag" "$branch_tag" + docker push "$branch_tag" + + log_info "Tags added and pushed: sha-${commit_sha}, ${branch}" + else + log_info "Skipping tag operations (DOCKER_PUSH=false)" + fi +} + +# Check if git working directory is clean +check_git_clean() { + if ! git diff --quiet || ! git diff --cached --quiet; then + log_error "Git working directory is not clean!" + log_error "Please commit or stash your changes before building." + log_info "Uncommitted changes:" + git status --short + return 1 + fi + + # Check for untracked files (excluding ignored files) + if [ -n "$(git ls-files --others --exclude-standard)" ]; then + log_warn "Untracked files detected:" + git ls-files --others --exclude-standard + log_error "Please add or ignore untracked files before building." + return 1 + fi + + return 0 +} + +# Main execution +main() { + # Check if git is clean (unless explicitly skipped) + if [ "${SKIP_GIT_CHECK}" != "true" ]; then + if ! check_git_clean; then + log_error "Refusing to build with uncommitted changes." + log_info "This ensures Docker images only contain committed code." + log_info "To bypass for local testing: SKIP_GIT_CHECK=true make docker-build" + exit 1 + fi + else + log_warn "Skipping git clean check (SKIP_GIT_CHECK=true)" + fi + + # Get current hashes + local tree_hash + local commit_sha + local branch + + tree_hash=$(get_tree_hash) + commit_sha=$(get_commit_sha) + branch=$(get_branch_name) + + log_info "Tree hash: ${tree_hash}" + log_info "Commit SHA: ${commit_sha}" + log_info "Branch: ${branch}" + + # Check if image with tree hash already exists + local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" + + if image_exists "$tree_tag"; then + log_info "Image already exists for tree hash: ${tree_hash}" + add_tags "$tree_tag" "$commit_sha" "$branch" + else + log_info "No existing image for tree hash: ${tree_hash}" + build_image "$tree_hash" "$commit_sha" "$branch" + fi + + log_info "Done!" +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/scripts/docker-tags.sh b/scripts/docker-tags.sh new file mode 100755 index 0000000..791aede --- /dev/null +++ b/scripts/docker-tags.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# docker-tags.sh - Manage Docker image tags without rebuilding +# +# This script handles tag promotion and management for existing images + +set -euo pipefail + +# Load configuration from environment or use defaults +DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" +DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Get the current git commit SHA (short) +get_commit_sha() { + git rev-parse --short HEAD +} + +# Check if a Docker image exists +image_exists() { + local image="$1" + + if docker manifest inspect "$image" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# Promote an image with additional tags using buildx imagetools +promote_image() { + local source_ref="$1" + shift + local new_tags=("$@") + + # Ensure source exists + if ! image_exists "$source_ref"; then + log_error "Source image does not exist: $source_ref" + exit 1 + fi + + log_info "Promoting image: $source_ref" + + # Build the tag arguments for imagetools create + local tag_args=() + for tag in "${new_tags[@]}"; do + local full_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${tag}" + tag_args+=("--tag" "$full_tag") + log_info " Adding tag: $tag" + done + + # Use buildx imagetools to create new tags without rebuilding + # This works across platforms and registries + docker buildx imagetools create \ + "${tag_args[@]}" \ + "$source_ref" + + log_info "Promotion complete!" +} + +# List all tags for the repository +list_tags() { + log_info "Fetching tags for ${DOCKER_REPO}..." + + # This works for Docker Hub + # For other registries, might need different approach + if [ "$DOCKER_REGISTRY" = "docker.io" ]; then + curl -s "https://hub.docker.com/v2/repositories/${DOCKER_REPO}/tags?page_size=100" | \ + jq -r '.results[].name' | \ + sort + else + log_warn "Tag listing not implemented for registry: $DOCKER_REGISTRY" + log_info "Use: docker search ${DOCKER_REGISTRY}/${DOCKER_REPO}" + fi +} + +# Find image by commit SHA +find_image_by_commit() { + local commit_sha="$1" + local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" + + if image_exists "$sha_tag"; then + echo "$sha_tag" + return 0 + else + return 1 + fi +} + +# Main command dispatcher +main() { + local command="${1:-}" + shift || true + + case "$command" in + promote) + if [ $# -lt 2 ]; then + log_error "Usage: $0 promote [...]" + exit 1 + fi + promote_image "$@" + ;; + + release) + # Special case for releases - find image by current commit + if [ $# -lt 1 ]; then + log_error "Usage: $0 release [...]" + exit 1 + fi + + local commit_sha + commit_sha=$(get_commit_sha) + local source_image + + if source_image=$(find_image_by_commit "$commit_sha"); then + log_info "Found image for commit $commit_sha: $source_image" + promote_image "$source_image" "$@" + else + log_error "No image found for commit SHA: $commit_sha" + log_error "Build the image first with: make docker-build" + exit 1 + fi + ;; + + list) + list_tags + ;; + + exists) + if [ $# -lt 1 ]; then + log_error "Usage: $0 exists " + exit 1 + fi + + local check_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:$1" + if image_exists "$check_tag"; then + log_info "Image exists: $check_tag" + exit 0 + else + log_info "Image does not exist: $check_tag" + exit 1 + fi + ;; + + *) + cat < [arguments] + +Commands: + promote [...] + Add new tags to an existing image + + release [...] + Promote the current commit's image to a release + + list + List all available tags + + exists + Check if a tag exists + +Environment variables: + DOCKER_REGISTRY Registry to use (default: docker.io) + DOCKER_REPO Repository name (default: nomasters/haystack) + +Examples: + $0 promote sha-abc123 v1.0.0 latest + $0 release v1.0.0 latest + $0 exists v1.0.0 +EOF + exit 1 + ;; + esac +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/scripts/tree-hash.sh b/scripts/tree-hash.sh new file mode 100755 index 0000000..d2cffa4 --- /dev/null +++ b/scripts/tree-hash.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# tree-hash.sh - Calculate deterministic hash of source files +# +# This script generates a hash based on the content of files that affect +# the build output. It ensures we can detect when a rebuild is actually needed. + +set -euo pipefail + +# Files and directories that affect the build +# Only includes actual source code directories and build-critical files +BUILD_PATHS=( + "cmd" + "needle" + "storage" + "server" + "client" + "go.mod" + "go.sum" + "Dockerfile" +) + +# Function to get git tree hash for specific paths +calculate_tree_hash() { + # Use git ls-tree to get a deterministic hash of file contents + # This is better than sha256sum because it's consistent across platforms + + # Get all files from our specific paths + local files="" + for path in "${BUILD_PATHS[@]}"; do + if [ -e "$path" ]; then + if [ -d "$path" ]; then + # For directories, get all files within + files="$files $(git ls-files "$path" 2>/dev/null || true)" + else + # For files, add directly + files="$files $path" + fi + fi + done + + # Remove duplicates and sort for deterministic output + files=$(echo $files | tr ' ' '\n' | sort -u | tr '\n' ' ') + + if [ -z "$files" ]; then + echo "Error: No source files found" >&2 + exit 1 + fi + + # Calculate hash using git's internal tree hashing + # This gives us a content-based hash that's consistent + echo $files | tr ' ' '\n' | xargs git hash-object | git hash-object --stdin +} + +# Main execution +main() { + # Ensure we're in a git repository + if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo "Error: Not in a git repository" >&2 + exit 1 + fi + + # Calculate and output the hash (first 12 characters for brevity) + local full_hash=$(calculate_tree_hash) + echo "${full_hash:0:12}" +} + +# Run main function +main "$@" \ No newline at end of file From 459dd48e856fc83310bd2757233bc6fb1a6d0f7f Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:16:36 -0600 Subject: [PATCH 2/8] fix: update linting, formatting, etc --- .github/workflows/go.yaml | 9 +++++++++ Makefile | 11 ++++++++--- README.md | 10 +++++++++- cmd/cli.go | 1 - cmd/client.go | 2 +- cmd/serve.go | 2 +- scripts/benchmark.sh | 19 ++++++++++++------- scripts/docker-build.sh | 3 ++- scripts/tree-hash.sh | 7 ++++--- 9 files changed, 46 insertions(+), 18 deletions(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 0bf082a..acbd792 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -29,6 +29,15 @@ jobs: exit 1 fi + - name: Run Go vet + run: make lint + + - name: Run Shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + make shellcheck + - name: Run Gosec Security Scanner uses: securego/gosec@master with: diff --git a/Makefile b/Makefile index fee02f9..6faea2e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: coverage test inspect install sec-scan lint fmt check +.PHONY: coverage test inspect install sec-scan lint fmt check shellcheck .PHONY: tree-hash docker-build docker-push docker-exists docker-promote docker-release docker-list # Docker configuration with defaults (override with environment variables) @@ -22,12 +22,17 @@ sec-scan: gosec -fmt=json -out=gosec-report.json -stdout -verbose=text ./... lint: - golangci-lint run ./... + go vet ./... fmt: go fmt ./... -check: fmt lint test +# Check shell scripts for issues +shellcheck: + @echo "Checking shell scripts with shellcheck..." + @shellcheck scripts/*.sh + +check: fmt lint test shellcheck @echo "All checks passed!" update-deps: diff --git a/README.md b/README.md index a6b4c7e..4f2d673 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,7 @@ make docker-exists make docker-promote TAGS="v0.1.0" ``` -### Testing +### Testing & Quality Checks ```bash # Run tests with race detection @@ -237,6 +237,14 @@ make test # Generate coverage report make coverage +# Run all quality checks (fmt, vet, test, shellcheck) +make check + +# Individual checks +make fmt # Format Go code +make lint # Run go vet +make shellcheck # Check shell scripts + # Run specific tests go test -v -run TestName ./path/to/package ``` diff --git a/cmd/cli.go b/cmd/cli.go index 8ad2db1..1a77659 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -41,4 +41,3 @@ COMMANDS: Use "haystack --help" for more information about a command. `) } - diff --git a/cmd/client.go b/cmd/client.go index d14dc01..f464551 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -331,4 +331,4 @@ func getEndpoint() string { return endpoint } return "localhost:1337" -} \ No newline at end of file +} diff --git a/cmd/serve.go b/cmd/serve.go index 4615adc..ead03f2 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -146,4 +146,4 @@ DESCRIPTION: if !*quiet { fmt.Println("Server stopped") } -} \ No newline at end of file +} diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index b38ebbe..7726ec0 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -37,7 +37,10 @@ run_benchmark() { # Extract key metrics echo "Key Results:" >> "$output_file" echo "============" >> "$output_file" - grep -E "(ops/sec|ns/op|B/op|allocs/op)" "$output_file" | tail -10 >> "$output_file" || true + # Avoid reading and writing same file in pipeline - use temp file + grep -E "(ops/sec|ns/op|B/op|allocs/op)" "$output_file" | tail -10 > "$output_file.tmp" || true + cat "$output_file.tmp" >> "$output_file" + rm -f "$output_file.tmp" else echo -e "${RED}✗ $name failed${NC}" echo "Error details saved to $output_file" @@ -85,7 +88,7 @@ cat > "$SUMMARY_FILE" << EOF ## Test Environment - **CPU:** $(sysctl -n machdep.cpu.brand_string 2>/dev/null || grep "model name" /proc/cpuinfo | head -1 | cut -d: -f2 | xargs || echo "Unknown") -- **Memory:** $(echo $(($(sysctl -n hw.memsize 2>/dev/null || grep MemTotal /proc/meminfo | awk '{print $2 * 1024}' || echo 0) / 1024 / 1024 / 1024)))GB +- **Memory:** $(($(sysctl -n hw.memsize 2>/dev/null || grep MemTotal /proc/meminfo | awk '{print $2 * 1024}' || echo 0) / 1024 / 1024 / 1024))GB - **Go Version:** $(go version) ## Key Performance Metrics @@ -164,11 +167,13 @@ if [ "$MIN_LATENCY" != "0" ]; then echo "- **Lowest Latency:** ${MIN_LATENCY} ns/op" >> "$SUMMARY_FILE" fi -echo "" >> "$SUMMARY_FILE" -echo "## Files Generated" >> "$SUMMARY_FILE" -echo "" >> "$SUMMARY_FILE" -echo "- Individual benchmark results: \`${RESULTS_DIR}/*.txt\`" >> "$SUMMARY_FILE" -echo "- This summary: \`${SUMMARY_FILE}\`" >> "$SUMMARY_FILE" +{ + echo "" + echo "## Files Generated" + echo "" + echo "- Individual benchmark results: \`${RESULTS_DIR}/*.txt\`" + echo "- This summary: \`${SUMMARY_FILE}\`" +} >> "$SUMMARY_FILE" echo -e "${GREEN}✓ Benchmark suite completed!${NC}" echo -e "${BLUE}Summary report: $SUMMARY_FILE${NC}" diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 426ebfa..cce34df 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -37,7 +37,8 @@ log_error() { # Get the current tree hash get_tree_hash() { - local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" "${script_dir}/tree-hash.sh" } diff --git a/scripts/tree-hash.sh b/scripts/tree-hash.sh index d2cffa4..a9e2aac 100755 --- a/scripts/tree-hash.sh +++ b/scripts/tree-hash.sh @@ -40,7 +40,7 @@ calculate_tree_hash() { done # Remove duplicates and sort for deterministic output - files=$(echo $files | tr ' ' '\n' | sort -u | tr '\n' ' ') + files=$(echo "$files" | tr ' ' '\n' | sort -u | tr '\n' ' ') if [ -z "$files" ]; then echo "Error: No source files found" >&2 @@ -49,7 +49,7 @@ calculate_tree_hash() { # Calculate hash using git's internal tree hashing # This gives us a content-based hash that's consistent - echo $files | tr ' ' '\n' | xargs git hash-object | git hash-object --stdin + echo "$files" | tr ' ' '\n' | xargs git hash-object | git hash-object --stdin } # Main execution @@ -61,7 +61,8 @@ main() { fi # Calculate and output the hash (first 12 characters for brevity) - local full_hash=$(calculate_tree_hash) + local full_hash + full_hash=$(calculate_tree_hash) echo "${full_hash:0:12}" } From e1cea51db6e7e811567ecf159a9a65745ff7a13a Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:20:58 -0600 Subject: [PATCH 3/8] fix: builder tooling --- scripts/docker-build.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index cce34df..66e6ac6 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -85,11 +85,21 @@ build_image() { fi # Create or use existing buildx builder - if ! docker buildx ls | grep -q "haystack-builder"; then + # First check if builder exists and is valid + if docker buildx ls | grep -q "haystack-builder"; then + # Try to inspect the builder to see if it's valid + if docker buildx inspect haystack-builder >/dev/null 2>&1; then + log_info "Using existing buildx builder..." + docker buildx use haystack-builder + else + log_warn "Existing builder is invalid, removing..." + docker buildx rm haystack-builder 2>/dev/null || true + log_info "Creating new buildx builder..." + docker buildx create --name haystack-builder --use + fi + else log_info "Creating buildx builder..." docker buildx create --name haystack-builder --use - else - docker buildx use haystack-builder fi # Build the image with all tags From cabfe98b1cb2db13019e52682180dc9aff823561 Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:23:19 -0600 Subject: [PATCH 4/8] fix: patch build and deploy script --- scripts/docker-build.sh | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 66e6ac6..a20f647 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -84,24 +84,16 @@ build_image() { exit 1 fi - # Create or use existing buildx builder - # First check if builder exists and is valid + # Ensure clean buildx state - always remove and recreate for consistency + # This avoids "existing instance" errors if docker buildx ls | grep -q "haystack-builder"; then - # Try to inspect the builder to see if it's valid - if docker buildx inspect haystack-builder >/dev/null 2>&1; then - log_info "Using existing buildx builder..." - docker buildx use haystack-builder - else - log_warn "Existing builder is invalid, removing..." - docker buildx rm haystack-builder 2>/dev/null || true - log_info "Creating new buildx builder..." - docker buildx create --name haystack-builder --use - fi - else - log_info "Creating buildx builder..." - docker buildx create --name haystack-builder --use + log_info "Removing existing buildx builder..." + docker buildx rm haystack-builder 2>/dev/null || true fi + log_info "Creating buildx builder..." + docker buildx create --name haystack-builder --use --bootstrap + # Build the image with all tags local build_args=( "--platform=${DOCKER_PLATFORMS}" From 96f175f70ca32629f9ab0b4006cd038019ffa189 Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:26:59 -0600 Subject: [PATCH 5/8] fix: only support 64bit --- Makefile | 2 +- README.md | 2 +- scripts/docker-build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6faea2e..2f8458c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # Docker configuration with defaults (override with environment variables) export DOCKER_REGISTRY ?= docker.io export DOCKER_REPO ?= nomasters/haystack -export DOCKER_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 +export DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 export DOCKER_PUSH ?= false export SKIP_GIT_CHECK ?= false diff --git a/README.md b/README.md index 4f2d673..25d675e 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Haystack follows a modular design with clear separation of concerns: ### Docker Images -Haystack uses a hermetic build system with content-based tagging. Images are built once and promoted through their lifecycle. +Haystack uses a hermetic build system with content-based tagging. Images are built once and promoted through their lifecycle. Supports 64-bit platforms only (amd64, arm64). #### Pull and Run diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index a20f647..6e1831d 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -12,7 +12,7 @@ set -euo pipefail # Load configuration from environment or use defaults DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" -DOCKER_PLATFORMS="${DOCKER_PLATFORMS:-linux/amd64,linux/arm64,linux/arm/v7}" +DOCKER_PLATFORMS="${DOCKER_PLATFORMS:-linux/amd64,linux/arm64}" DOCKER_PUSH="${DOCKER_PUSH:-false}" SKIP_GIT_CHECK="${SKIP_GIT_CHECK:-false}" From 2f0719c1e9759135e053bbeab4edccbcde1abaa7 Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:36:27 -0600 Subject: [PATCH 6/8] fix: update tagging tools for docker --- README.md | 3 +-- scripts/docker-build.sh | 39 +++++++++++---------------------------- scripts/docker-tags.sh | 8 ++++---- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 25d675e..8f1fb26 100644 --- a/README.md +++ b/README.md @@ -166,9 +166,8 @@ docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve -H 0.0.0.0 -p 1337 #### Available Tags -- `sha-`: Specific commit SHA +- `commit-`: Specific Git commit - `tree-`: Content-based hash (same source = same tag) -- `main`: Latest build from main branch - `v0.1.0`: Specific release version (immutable) ### Deploy to Fly.io diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 6e1831d..bcf352f 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -47,11 +47,6 @@ get_commit_sha() { git rev-parse --short HEAD } -# Get the current branch name -get_branch_name() { - git rev-parse --abbrev-ref HEAD -} - # Check if a Docker image exists in the registry image_exists() { local image="$1" @@ -68,15 +63,13 @@ image_exists() { build_image() { local tree_hash="$1" local commit_sha="$2" - local branch="$3" local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" - local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" - local branch_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${branch}" + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" log_info "Building multi-platform image..." log_info "Platforms: ${DOCKER_PLATFORMS}" - log_info "Tags: tree-${tree_hash}, sha-${commit_sha}, ${branch}" + log_info "Tags: tree-${tree_hash}, commit-${commit_sha}" # Ensure buildx is available if ! docker buildx version >/dev/null 2>&1; then @@ -98,8 +91,7 @@ build_image() { local build_args=( "--platform=${DOCKER_PLATFORMS}" "--tag=${tree_tag}" - "--tag=${sha_tag}" - "--tag=${branch_tag}" + "--tag=${commit_tag}" ) # Add push flag if requested @@ -108,7 +100,7 @@ build_image() { else build_args+=("--load") log_warn "Building for local platform only (--load mode). Set DOCKER_PUSH=true for multi-platform push." - build_args=("--tag=${tree_tag}" "--tag=${sha_tag}" "--tag=${branch_tag}") + build_args=("--tag=${tree_tag}" "--tag=${commit_tag}") fi docker buildx build "${build_args[@]}" . @@ -120,26 +112,20 @@ build_image() { add_tags() { local source_tag="$1" local commit_sha="$2" - local branch="$3" - local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" - local branch_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${branch}" + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" - log_info "Image already exists with tree hash, adding new tags..." + log_info "Image already exists with tree hash, adding new commit tag..." if [ "${DOCKER_PUSH}" = "true" ]; then # Pull the existing image docker pull "$source_tag" # Tag with commit SHA - docker tag "$source_tag" "$sha_tag" - docker push "$sha_tag" - - # Tag with branch name - docker tag "$source_tag" "$branch_tag" - docker push "$branch_tag" + docker tag "$source_tag" "$commit_tag" + docker push "$commit_tag" - log_info "Tags added and pushed: sha-${commit_sha}, ${branch}" + log_info "Tag added and pushed: commit-${commit_sha}" else log_info "Skipping tag operations (DOCKER_PUSH=false)" fi @@ -183,25 +169,22 @@ main() { # Get current hashes local tree_hash local commit_sha - local branch tree_hash=$(get_tree_hash) commit_sha=$(get_commit_sha) - branch=$(get_branch_name) log_info "Tree hash: ${tree_hash}" log_info "Commit SHA: ${commit_sha}" - log_info "Branch: ${branch}" # Check if image with tree hash already exists local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" if image_exists "$tree_tag"; then log_info "Image already exists for tree hash: ${tree_hash}" - add_tags "$tree_tag" "$commit_sha" "$branch" + add_tags "$tree_tag" "$commit_sha" else log_info "No existing image for tree hash: ${tree_hash}" - build_image "$tree_hash" "$commit_sha" "$branch" + build_image "$tree_hash" "$commit_sha" fi log_info "Done!" diff --git a/scripts/docker-tags.sh b/scripts/docker-tags.sh index 791aede..697dfb6 100755 --- a/scripts/docker-tags.sh +++ b/scripts/docker-tags.sh @@ -95,10 +95,10 @@ list_tags() { # Find image by commit SHA find_image_by_commit() { local commit_sha="$1" - local sha_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:sha-${commit_sha}" + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" - if image_exists "$sha_tag"; then - echo "$sha_tag" + if image_exists "$commit_tag"; then + echo "$commit_tag" return 0 else return 1 @@ -134,7 +134,7 @@ main() { log_info "Found image for commit $commit_sha: $source_image" promote_image "$source_image" "$@" else - log_error "No image found for commit SHA: $commit_sha" + log_error "No image found for commit: $commit_sha" log_error "Build the image first with: make docker-build" exit 1 fi From 32e45a49dd4c0a955017c01316a0e216fe229252 Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:40:38 -0600 Subject: [PATCH 7/8] fix: tagging tooling --- scripts/docker-build.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index bcf352f..fff08c1 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -118,12 +118,11 @@ add_tags() { log_info "Image already exists with tree hash, adding new commit tag..." if [ "${DOCKER_PUSH}" = "true" ]; then - # Pull the existing image - docker pull "$source_tag" - - # Tag with commit SHA - docker tag "$source_tag" "$commit_tag" - docker push "$commit_tag" + # Use imagetools to create new tag from existing image in registry + # This preserves multi-platform manifests and doesn't require local pull + docker buildx imagetools create \ + --tag "$commit_tag" \ + "$source_tag" log_info "Tag added and pushed: commit-${commit_sha}" else From af3aa2b3fdf4679f6bb90cae6fd6154bcc4b779c Mon Sep 17 00:00:00 2001 From: Nathan Toups Date: Fri, 8 Aug 2025 22:45:24 -0600 Subject: [PATCH 8/8] fix: removed broken scan tool --- .github/workflows/go.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index acbd792..7977756 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -38,11 +38,6 @@ jobs: sudo apt-get install -y shellcheck make shellcheck - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: ./... - test: runs-on: ubuntu-latest needs: quality-checks @@ -52,7 +47,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.4" + go-version: "1.24.6" - name: Test run: make test