diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index a671469..7977756 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.4" + go-version: "1.24.6" - name: Check formatting run: | @@ -29,10 +29,14 @@ jobs: exit 1 fi - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: ./... + - name: Run Go vet + run: make lint + + - name: Run Shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + make shellcheck test: runs-on: ubuntu-latest @@ -43,7 +47,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.4" + go-version: "1.24.6" - name: Test run: make test @@ -55,3 +59,28 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: bash <(curl -s https://codecov.io/bash) + + docker-build: + runs-on: ubuntu-latest + needs: [quality-checks, test] + # Only build and push Docker images on main branch after tests pass + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build info + run: make docker-info + + - name: Build and push Docker image + run: make docker-push + env: + DOCKER_PUSH: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..3415d9a --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,43 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + promote-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Build info + run: | + echo "Promoting release: ${{ steps.version.outputs.VERSION }}" + make docker-info + + - name: Promote Docker image + run: | + # The image should already exist from the main branch build + # We're just adding the version tag to it (immutable tags - no latest) + make docker-promote TAGS="${{ steps.version.outputs.VERSION }}" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + generate_release_notes: true + draft: false + prerelease: false \ No newline at end of file diff --git a/Makefile b/Makefile index 32fff9f..2f8458c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,14 @@ -.PHONEY: coverage test inspect install sec-scan lint fmt check +.PHONY: coverage test inspect install sec-scan lint fmt check shellcheck +.PHONY: tree-hash docker-build docker-push docker-exists docker-promote docker-release docker-list +# Docker configuration with defaults (override with environment variables) +export DOCKER_REGISTRY ?= docker.io +export DOCKER_REPO ?= nomasters/haystack +export DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 +export DOCKER_PUSH ?= false +export SKIP_GIT_CHECK ?= false + +# Go targets test: go test -v ./... @@ -13,16 +22,76 @@ sec-scan: gosec -fmt=json -out=gosec-report.json -stdout -verbose=text ./... lint: - golangci-lint run ./... + go vet ./... fmt: go fmt ./... -check: fmt lint test +# Check shell scripts for issues +shellcheck: + @echo "Checking shell scripts with shellcheck..." + @shellcheck scripts/*.sh + +check: fmt lint test shellcheck @echo "All checks passed!" update-deps: go get -u && go mod tidy install: - go install github.com/nomasters/haystack/cmd/haystack \ No newline at end of file + go install github.com/nomasters/haystack/cmd/haystack + +# Docker targets - Local-first hermetic builds +# These commands work identically on your machine and in CI + +# Calculate the tree hash of source files +tree-hash: + @./scripts/tree-hash.sh + +# Build Docker image (idempotent - only builds if tree hash changed) +# Requires clean git working directory (override with SKIP_GIT_CHECK=true) +docker-build: + @./scripts/docker-build.sh + +# Build and push Docker image to registry +docker-push: + @DOCKER_PUSH=true ./scripts/docker-build.sh + +# Check if image exists for current tree hash +docker-exists: + @TREE_HASH=$$(./scripts/tree-hash.sh) && \ + ./scripts/docker-tags.sh exists "tree-$$TREE_HASH" + +# Promote current commit's image with additional tags +# Usage: make docker-promote TAGS="v1.0.0" +docker-promote: + @if [ -z "$(TAGS)" ]; then \ + echo "Error: TAGS variable required. Usage: make docker-promote TAGS=\"v1.0.0\""; \ + exit 1; \ + fi + @./scripts/docker-tags.sh release $(TAGS) + +# Release workflow - build if needed, then tag +# Usage: make docker-release VERSION=v1.0.0 +docker-release: docker-push + @if [ -z "$(VERSION)" ]; then \ + echo "Error: VERSION variable required. Usage: make docker-release VERSION=v1.0.0"; \ + exit 1; \ + fi + @./scripts/docker-tags.sh release $(VERSION) + +# List all Docker tags in the registry +docker-list: + @./scripts/docker-tags.sh list + +# Clean up local Docker buildx builders +docker-clean: + @docker buildx rm haystack-builder 2>/dev/null || true + +# Show current build information +docker-info: + @echo "Tree hash: $$(./scripts/tree-hash.sh)" + @echo "Commit SHA: $$(git rev-parse --short HEAD)" + @echo "Branch: $$(git rev-parse --abbrev-ref HEAD)" + @echo "Registry: $${DOCKER_REGISTRY:-docker.io}" + @echo "Repository: $${DOCKER_REPO:-nomasters/haystack}" \ No newline at end of file diff --git a/README.md b/README.md index 59a8b2f..8f1fb26 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,291 @@ # haystack -an ephemeral, quiet and tiny content addressed key/value store. +A tiny, ephemeral, quiet, content-addressed key/value store. -Design goals: -- no auth. the submitter declares the hash of the content and submits the content. if the hash of the content on the server matches the submitted hash, the content is accepted and any requestor with the hash can request the content -- ephemeral. key/value is short lived, no keys will live longer than a window (TBD) and this is defined by the server. Content can be resubmitted to reset the counter -- consistent payload and response to make implementation details indistinguishable to an adversary -- quiet. No network traffic happens that isn't absolutely needed. +## Overview +Haystack is a minimalist key/value store built for simplicity and efficiency. It operates over UDP with fixed-size messages, uses content-addressing via SHA256 hashes, and automatically expires data after a configurable time window. +## Key Features -goals: -- easy to use client and server -- fast -- simple +- **Content-Addressed**: Keys are SHA256 hashes of the content itself - no separate key management needed +- **Fixed-Size Messages**: All network messages are exactly 192 bytes (writes) or 32 bytes (reads) +- **Ephemeral Storage**: Data automatically expires after a configurable TTL (default 24 hours) +- **UDP-Only**: Lightweight, stateless protocol with minimal overhead +- **No Authentication**: Content validity is proven by SHA256 hash matching +- **Zero Server Confirmation**: Write operations receive no response by design -### Needle: the Haystack message +## Installation -A message in haystack is called a Needle. A needle includes a 32 bytes sha256 hash of a 160 byte payload. -All payloads are a fixed length. Both the client and server can verify the hash of the payload by verifying a 192 byte needle. +### From Source +```bash +# Clone the repository +git clone https://github.com/nomasters/haystack.git +cd haystack + +# Build and install +make install +``` + +### Using Go + +```bash +go install github.com/nomasters/haystack@latest +``` + +## Quick Start + +### Start a Server + +```bash +# Default configuration (localhost:1337) +haystack serve + +# Custom host and port +haystack serve -H 0.0.0.0 -p 9000 +``` + +### Client Operations + +```bash +# Set a value (returns the SHA256 hash) +haystack client set "hello world" + +# Get a value using its hash +haystack client get + +# Use pipes for content +echo "my message" | haystack client set - +``` + +### Go Client Library + +```go +package main + +import ( + "context" + "fmt" + "github.com/nomasters/haystack/client" +) + +func main() { + // Create a client + c, err := client.New("localhost:1337") + if err != nil { + panic(err) + } + defer c.Close() + + // Store data + hash, err := c.Set(context.Background(), []byte("hello world")) + if err != nil { + panic(err) + } + + // Retrieve data + data, err := c.Get(context.Background(), hash) + if err != nil { + panic(err) + } + + fmt.Printf("Retrieved: %s\n", data) +} +``` + +## The Needle Protocol + +A message in Haystack is called a "Needle". Each needle consists of a 32-byte SHA256 hash followed by a 160-byte payload, totaling exactly 192 bytes. + +```text +| hash | payload | +|----------------|--------------------------------------------------------------------------------| +| 32 bytes | 160 bytes | +``` + +This fixed size enables: + +- Single UDP packet transmission +- Consistent network behavior +- Message chaining for larger payloads + +### Message Chaining Example + +The 160-byte payload is large enough to contain encrypted data and a reference to the next chunk: + +```text +| needle | +|--------------------------------------------------------------------------------------------------| +| hash | payload | +|----------------|---------------------------------------------------------------------------------| +| | nonce | encrypted payload | +| |------------|--------------------------------------------------------------------| +| | | next key | padded message | +| | |----------------|---------------------------------------------------| +| 32 bytes | 24 bytes | 32 bytes | 104 bytes | +``` + +## Protocol Operations + +### Read Request (32 bytes) + +Send a 32-byte SHA256 hash to retrieve the associated needle. If found, the server responds with the complete 192-byte needle. + +### Write Request (192 bytes) + +Send a complete 192-byte needle. The server validates that the hash matches the payload and stores it if valid. No response is sent (by design). + +## Architecture + +Haystack follows a modular design with clear separation of concerns: + +- **Needle Package**: Core message structure and validation +- **Storage Package**: Interface-based storage abstraction with TTL support +- **Server Package**: High-performance UDP server with zero-copy buffer pools +- **Client Package**: Production-ready Go client with connection pooling and retry logic + +## Deployment + +### Docker Images + +Haystack uses a hermetic build system with content-based tagging. Images are built once and promoted through their lifecycle. Supports 64-bit platforms only (amd64, arm64). + +#### Pull and Run + +```bash +# Run the latest build from main branch +docker run -p 1337:1337/udp nomasters/haystack:main serve + +# Run a specific version (immutable tags) +docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve + +# Run with custom configuration +docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve -H 0.0.0.0 -p 1337 +``` + +#### Available Tags + +- `commit-`: Specific Git commit +- `tree-`: Content-based hash (same source = same tag) +- `v0.1.0`: Specific release version (immutable) + +### Deploy to Fly.io + +Create a `fly.toml` configuration: + +```toml +app = "haystack" + +[build] + image = "nomasters/haystack:v0.1.0" + +[[services]] + internal_port = 1337 + protocol = "udp" + + [[services.ports]] + port = 1337 +``` + +Then deploy: + +```bash +fly deploy +``` + +## Development + +### Building from Source + +```bash +# Install locally +make install +``` + +### Docker Development + +The build system uses content-based hashing to ensure hermetic, reproducible builds. By default, builds require a clean git working directory to ensure only committed code is deployed. + +```bash +# Show current build info +make docker-info + +# Build locally (requires clean git working directory) +make docker-build + +# Build with uncommitted changes (local testing only) +SKIP_GIT_CHECK=true make docker-build + +# Build and push to registry (always requires clean git) +make docker-push + +# Check if image exists for current source +make docker-exists + +# Promote image with version tag +make docker-promote TAGS="v0.1.0" ``` -hash | payload ----------|---------- -32 bytes | 160 bytes + +### Testing & Quality Checks + +```bash +# Run tests with race detection +make test + +# Generate coverage report +make coverage + +# Run all quality checks (fmt, vet, test, shellcheck) +make check + +# Individual checks +make fmt # Format Go code +make lint # Run go vet +make shellcheck # Check shell scripts + +# Run specific tests +go test -v -run TestName ./path/to/package ``` -The message size is small, but designed to allow _single UDP packet_ message transmission. This is meant to be light weight and efficient. +### Project Structure + +```text +haystack/ +├── needle/ # Core message format +├── storage/ # Storage interface and implementations +├── server/ # UDP server implementation +├── client/ # Go client library +└── cmd/ # CLI implementation +``` -While it is small, it is large enough to "chain" messages together. Such patterns must be configured client-side, but a hypothetical payload with an encrypted message +## Design Philosophy -This is large enough for the value to contain something like: +Haystack embraces several core principles: -| nonce | encrypted payload | -| |------------------------------| -| | next key | padded message | -|----------|-------------|----------------| -| 24 bytes | 32 bytes | bytes 104 | +1. **Simplicity**: Minimal protocol, clear semantics, predictable behavior +2. **Performance**: Zero-copy operations, buffer pooling, efficient packet handling +3. **Reliability**: Content-addressed storage ensures data integrity +4. **Efficiency**: Fixed-size messages and stateless UDP protocol +## Use Cases +- Temporary data storage with automatic expiration +- Distributed caching +- Message passing between services +- Content-addressed storage needs +- Lightweight key/value operations -### Reads and Writes +## Contributing -A Haystack server accepts two and only two byte length requests 32 bytes and 192 bytes. +Contributions are welcome! Please ensure: -#### Read Requests +- All tests pass with race detection enabled +- Code follows standard Go conventions +- New features include appropriate tests +- Changes maintain backward compatibility -A 32 byte request is a read request. It implies that the requestor would like the stored value of the Needle. If the Haystack server has a Needle that matches the request hash, it will respond with the full Needle in the response. +## License -#### Write Requests +The Unlicense - This is free and unencumbered software released into the public domain. See [LICENSE](LICENSE) for details. -A write request my be 192 bytes. The server will verify that these bytes are a valid Needles (that the final 160 bytes sha256 hash match the first 32 bytes hash included in the payload). If this Needle is valid, it is stored. The server provides no response for this operation. If a client wants to confirm that a write was completed successfully, it should submit a read request to confirm. \ No newline at end of file diff --git a/cmd/cli.go b/cmd/cli.go index 8ad2db1..1a77659 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -41,4 +41,3 @@ COMMANDS: Use "haystack --help" for more information about a command. `) } - diff --git a/cmd/client.go b/cmd/client.go index d14dc01..f464551 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -331,4 +331,4 @@ func getEndpoint() string { return endpoint } return "localhost:1337" -} \ No newline at end of file +} diff --git a/cmd/serve.go b/cmd/serve.go index 4615adc..ead03f2 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -146,4 +146,4 @@ DESCRIPTION: if !*quiet { fmt.Println("Server stopped") } -} \ No newline at end of file +} diff --git a/haystack_test.go b/haystack_test.go deleted file mode 100644 index 691492e..0000000 --- a/haystack_test.go +++ /dev/null @@ -1 +0,0 @@ -package haystack diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index b38ebbe..7726ec0 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -37,7 +37,10 @@ run_benchmark() { # Extract key metrics echo "Key Results:" >> "$output_file" echo "============" >> "$output_file" - grep -E "(ops/sec|ns/op|B/op|allocs/op)" "$output_file" | tail -10 >> "$output_file" || true + # Avoid reading and writing same file in pipeline - use temp file + grep -E "(ops/sec|ns/op|B/op|allocs/op)" "$output_file" | tail -10 > "$output_file.tmp" || true + cat "$output_file.tmp" >> "$output_file" + rm -f "$output_file.tmp" else echo -e "${RED}✗ $name failed${NC}" echo "Error details saved to $output_file" @@ -85,7 +88,7 @@ cat > "$SUMMARY_FILE" << EOF ## Test Environment - **CPU:** $(sysctl -n machdep.cpu.brand_string 2>/dev/null || grep "model name" /proc/cpuinfo | head -1 | cut -d: -f2 | xargs || echo "Unknown") -- **Memory:** $(echo $(($(sysctl -n hw.memsize 2>/dev/null || grep MemTotal /proc/meminfo | awk '{print $2 * 1024}' || echo 0) / 1024 / 1024 / 1024)))GB +- **Memory:** $(($(sysctl -n hw.memsize 2>/dev/null || grep MemTotal /proc/meminfo | awk '{print $2 * 1024}' || echo 0) / 1024 / 1024 / 1024))GB - **Go Version:** $(go version) ## Key Performance Metrics @@ -164,11 +167,13 @@ if [ "$MIN_LATENCY" != "0" ]; then echo "- **Lowest Latency:** ${MIN_LATENCY} ns/op" >> "$SUMMARY_FILE" fi -echo "" >> "$SUMMARY_FILE" -echo "## Files Generated" >> "$SUMMARY_FILE" -echo "" >> "$SUMMARY_FILE" -echo "- Individual benchmark results: \`${RESULTS_DIR}/*.txt\`" >> "$SUMMARY_FILE" -echo "- This summary: \`${SUMMARY_FILE}\`" >> "$SUMMARY_FILE" +{ + echo "" + echo "## Files Generated" + echo "" + echo "- Individual benchmark results: \`${RESULTS_DIR}/*.txt\`" + echo "- This summary: \`${SUMMARY_FILE}\`" +} >> "$SUMMARY_FILE" echo -e "${GREEN}✓ Benchmark suite completed!${NC}" echo -e "${BLUE}Summary report: $SUMMARY_FILE${NC}" diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh new file mode 100755 index 0000000..fff08c1 --- /dev/null +++ b/scripts/docker-build.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# +# docker-build.sh - Hermetic Docker build with content-based tagging +# +# This script implements idempotent Docker builds that: +# 1. Check if an image with the current tree hash already exists +# 2. Build only if necessary +# 3. Tag with both tree hash and commit SHA + +set -euo pipefail + +# Load configuration from environment or use defaults +DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" +DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" +DOCKER_PLATFORMS="${DOCKER_PLATFORMS:-linux/amd64,linux/arm64}" +DOCKER_PUSH="${DOCKER_PUSH:-false}" +SKIP_GIT_CHECK="${SKIP_GIT_CHECK:-false}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Get the current tree hash +get_tree_hash() { + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + "${script_dir}/tree-hash.sh" +} + +# Get the current git commit SHA (short) +get_commit_sha() { + git rev-parse --short HEAD +} + +# Check if a Docker image exists in the registry +image_exists() { + local image="$1" + + # Try to inspect the manifest + if docker manifest inspect "$image" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# Build multi-platform Docker image +build_image() { + local tree_hash="$1" + local commit_sha="$2" + + local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" + + log_info "Building multi-platform image..." + log_info "Platforms: ${DOCKER_PLATFORMS}" + log_info "Tags: tree-${tree_hash}, commit-${commit_sha}" + + # Ensure buildx is available + if ! docker buildx version >/dev/null 2>&1; then + log_error "Docker buildx is not available. Please install Docker with buildx support." + exit 1 + fi + + # Ensure clean buildx state - always remove and recreate for consistency + # This avoids "existing instance" errors + if docker buildx ls | grep -q "haystack-builder"; then + log_info "Removing existing buildx builder..." + docker buildx rm haystack-builder 2>/dev/null || true + fi + + log_info "Creating buildx builder..." + docker buildx create --name haystack-builder --use --bootstrap + + # Build the image with all tags + local build_args=( + "--platform=${DOCKER_PLATFORMS}" + "--tag=${tree_tag}" + "--tag=${commit_tag}" + ) + + # Add push flag if requested + if [ "${DOCKER_PUSH}" = "true" ]; then + build_args+=("--push") + else + build_args+=("--load") + log_warn "Building for local platform only (--load mode). Set DOCKER_PUSH=true for multi-platform push." + build_args=("--tag=${tree_tag}" "--tag=${commit_tag}") + fi + + docker buildx build "${build_args[@]}" . + + log_info "Build complete!" +} + +# Add additional tags to an existing image +add_tags() { + local source_tag="$1" + local commit_sha="$2" + + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" + + log_info "Image already exists with tree hash, adding new commit tag..." + + if [ "${DOCKER_PUSH}" = "true" ]; then + # Use imagetools to create new tag from existing image in registry + # This preserves multi-platform manifests and doesn't require local pull + docker buildx imagetools create \ + --tag "$commit_tag" \ + "$source_tag" + + log_info "Tag added and pushed: commit-${commit_sha}" + else + log_info "Skipping tag operations (DOCKER_PUSH=false)" + fi +} + +# Check if git working directory is clean +check_git_clean() { + if ! git diff --quiet || ! git diff --cached --quiet; then + log_error "Git working directory is not clean!" + log_error "Please commit or stash your changes before building." + log_info "Uncommitted changes:" + git status --short + return 1 + fi + + # Check for untracked files (excluding ignored files) + if [ -n "$(git ls-files --others --exclude-standard)" ]; then + log_warn "Untracked files detected:" + git ls-files --others --exclude-standard + log_error "Please add or ignore untracked files before building." + return 1 + fi + + return 0 +} + +# Main execution +main() { + # Check if git is clean (unless explicitly skipped) + if [ "${SKIP_GIT_CHECK}" != "true" ]; then + if ! check_git_clean; then + log_error "Refusing to build with uncommitted changes." + log_info "This ensures Docker images only contain committed code." + log_info "To bypass for local testing: SKIP_GIT_CHECK=true make docker-build" + exit 1 + fi + else + log_warn "Skipping git clean check (SKIP_GIT_CHECK=true)" + fi + + # Get current hashes + local tree_hash + local commit_sha + + tree_hash=$(get_tree_hash) + commit_sha=$(get_commit_sha) + + log_info "Tree hash: ${tree_hash}" + log_info "Commit SHA: ${commit_sha}" + + # Check if image with tree hash already exists + local tree_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" + + if image_exists "$tree_tag"; then + log_info "Image already exists for tree hash: ${tree_hash}" + add_tags "$tree_tag" "$commit_sha" + else + log_info "No existing image for tree hash: ${tree_hash}" + build_image "$tree_hash" "$commit_sha" + fi + + log_info "Done!" +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/scripts/docker-tags.sh b/scripts/docker-tags.sh new file mode 100755 index 0000000..697dfb6 --- /dev/null +++ b/scripts/docker-tags.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# docker-tags.sh - Manage Docker image tags without rebuilding +# +# This script handles tag promotion and management for existing images + +set -euo pipefail + +# Load configuration from environment or use defaults +DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" +DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Get the current git commit SHA (short) +get_commit_sha() { + git rev-parse --short HEAD +} + +# Check if a Docker image exists +image_exists() { + local image="$1" + + if docker manifest inspect "$image" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# Promote an image with additional tags using buildx imagetools +promote_image() { + local source_ref="$1" + shift + local new_tags=("$@") + + # Ensure source exists + if ! image_exists "$source_ref"; then + log_error "Source image does not exist: $source_ref" + exit 1 + fi + + log_info "Promoting image: $source_ref" + + # Build the tag arguments for imagetools create + local tag_args=() + for tag in "${new_tags[@]}"; do + local full_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:${tag}" + tag_args+=("--tag" "$full_tag") + log_info " Adding tag: $tag" + done + + # Use buildx imagetools to create new tags without rebuilding + # This works across platforms and registries + docker buildx imagetools create \ + "${tag_args[@]}" \ + "$source_ref" + + log_info "Promotion complete!" +} + +# List all tags for the repository +list_tags() { + log_info "Fetching tags for ${DOCKER_REPO}..." + + # This works for Docker Hub + # For other registries, might need different approach + if [ "$DOCKER_REGISTRY" = "docker.io" ]; then + curl -s "https://hub.docker.com/v2/repositories/${DOCKER_REPO}/tags?page_size=100" | \ + jq -r '.results[].name' | \ + sort + else + log_warn "Tag listing not implemented for registry: $DOCKER_REGISTRY" + log_info "Use: docker search ${DOCKER_REGISTRY}/${DOCKER_REPO}" + fi +} + +# Find image by commit SHA +find_image_by_commit() { + local commit_sha="$1" + local commit_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:commit-${commit_sha}" + + if image_exists "$commit_tag"; then + echo "$commit_tag" + return 0 + else + return 1 + fi +} + +# Main command dispatcher +main() { + local command="${1:-}" + shift || true + + case "$command" in + promote) + if [ $# -lt 2 ]; then + log_error "Usage: $0 promote [...]" + exit 1 + fi + promote_image "$@" + ;; + + release) + # Special case for releases - find image by current commit + if [ $# -lt 1 ]; then + log_error "Usage: $0 release [...]" + exit 1 + fi + + local commit_sha + commit_sha=$(get_commit_sha) + local source_image + + if source_image=$(find_image_by_commit "$commit_sha"); then + log_info "Found image for commit $commit_sha: $source_image" + promote_image "$source_image" "$@" + else + log_error "No image found for commit: $commit_sha" + log_error "Build the image first with: make docker-build" + exit 1 + fi + ;; + + list) + list_tags + ;; + + exists) + if [ $# -lt 1 ]; then + log_error "Usage: $0 exists " + exit 1 + fi + + local check_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:$1" + if image_exists "$check_tag"; then + log_info "Image exists: $check_tag" + exit 0 + else + log_info "Image does not exist: $check_tag" + exit 1 + fi + ;; + + *) + cat < [arguments] + +Commands: + promote [...] + Add new tags to an existing image + + release [...] + Promote the current commit's image to a release + + list + List all available tags + + exists + Check if a tag exists + +Environment variables: + DOCKER_REGISTRY Registry to use (default: docker.io) + DOCKER_REPO Repository name (default: nomasters/haystack) + +Examples: + $0 promote sha-abc123 v1.0.0 latest + $0 release v1.0.0 latest + $0 exists v1.0.0 +EOF + exit 1 + ;; + esac +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/scripts/tree-hash.sh b/scripts/tree-hash.sh new file mode 100755 index 0000000..a9e2aac --- /dev/null +++ b/scripts/tree-hash.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# tree-hash.sh - Calculate deterministic hash of source files +# +# This script generates a hash based on the content of files that affect +# the build output. It ensures we can detect when a rebuild is actually needed. + +set -euo pipefail + +# Files and directories that affect the build +# Only includes actual source code directories and build-critical files +BUILD_PATHS=( + "cmd" + "needle" + "storage" + "server" + "client" + "go.mod" + "go.sum" + "Dockerfile" +) + +# Function to get git tree hash for specific paths +calculate_tree_hash() { + # Use git ls-tree to get a deterministic hash of file contents + # This is better than sha256sum because it's consistent across platforms + + # Get all files from our specific paths + local files="" + for path in "${BUILD_PATHS[@]}"; do + if [ -e "$path" ]; then + if [ -d "$path" ]; then + # For directories, get all files within + files="$files $(git ls-files "$path" 2>/dev/null || true)" + else + # For files, add directly + files="$files $path" + fi + fi + done + + # Remove duplicates and sort for deterministic output + files=$(echo "$files" | tr ' ' '\n' | sort -u | tr '\n' ' ') + + if [ -z "$files" ]; then + echo "Error: No source files found" >&2 + exit 1 + fi + + # Calculate hash using git's internal tree hashing + # This gives us a content-based hash that's consistent + echo "$files" | tr ' ' '\n' | xargs git hash-object | git hash-object --stdin +} + +# Main execution +main() { + # Ensure we're in a git repository + if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo "Error: Not in a git repository" >&2 + exit 1 + fi + + # Calculate and output the hash (first 12 characters for brevity) + local full_hash + full_hash=$(calculate_tree_hash) + echo "${full_hash:0:12}" +} + +# Run main function +main "$@" \ No newline at end of file