diff --git a/.gitignore b/.gitignore index cedc7da..9b823f1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,7 @@ haystack benchmark_results* -gosec-report.json \ No newline at end of file +gosec-report.json + +# Scratch directory for testing +scratch/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0107ca8..9901c85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,10 @@ RUN CGO_ENABLED=0 go build \ -o haystack \ ./cmd/haystack -FROM scratch +FROM cgr.dev/chainguard/static:latest COPY --from=builder /build/haystack /haystack -# scratch doesn't have mkdir, so WORKDIR creates the directory WORKDIR /data EXPOSE 1337/udp diff --git a/README.md b/README.md index 8f1fb26..9419c8b 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,11 @@ go install github.com/nomasters/haystack@latest # Default configuration (localhost:1337) haystack serve -# Custom host and port -haystack serve -H 0.0.0.0 -p 9000 +# Custom address +haystack serve -a 0.0.0.0:9000 + +# Using environment variables +HAYSTACK_ADDR=0.0.0.0:9000 haystack serve ``` ### Client Operations @@ -161,7 +164,7 @@ docker run -p 1337:1337/udp nomasters/haystack:main serve docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve # Run with custom configuration -docker run -p 1337:1337/udp nomasters/haystack:v0.1.0 serve -H 0.0.0.0 -p 1337 +docker run -p 1337:1337/udp -e HAYSTACK_ADDR=0.0.0.0:1337 nomasters/haystack:v0.1.0 serve ``` #### Available Tags diff --git a/client/client.go b/client/client.go index e088161..c09778b 100644 --- a/client/client.go +++ b/client/client.go @@ -110,14 +110,14 @@ func (c *Client) SetBytes(ctx context.Context, data []byte) error { } defer c.connPool.Put(conn) - // Set write timeout + // Set timeout (using SetDeadline like GET does) if deadline, ok := ctx.Deadline(); ok { - if err := conn.SetWriteDeadline(deadline); err != nil { - c.logger.Errorf("Failed to set write deadline: %v", err) + if err := conn.SetDeadline(deadline); err != nil { + c.logger.Errorf("Failed to set deadline: %v", err) } } else { - if err := conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)); err != nil { - c.logger.Errorf("Failed to set write timeout: %v", err) + if err := conn.SetDeadline(time.Now().Add(c.writeTimeout)); err != nil { + c.logger.Errorf("Failed to set timeout: %v", err) } } @@ -300,6 +300,7 @@ func (p *connectionPool) MarkBad(conn net.Conn) { // createConn creates a new connection. func (p *connectionPool) createConn() (*pooledConn, error) { + // Use regular Dial for connected UDP socket conn, err := net.Dial("udp", p.address) if err != nil { return nil, err diff --git a/client/client_test.go b/client/client_test.go index 0fd8d0e..4714790 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -55,6 +55,7 @@ func TestClient_SetAndGet(t *testing.T) { retrievedNeedle, err := client.Get(ctx, hash) if err != nil { t.Errorf("GET operation failed: %v", err) + return // Exit early if GET failed } if retrievedNeedle.Hash() != testNeedle.Hash() { @@ -723,6 +724,8 @@ func (m *mockLogger) Error(v ...any) {} func (m *mockLogger) Errorf(format string, v ...any) {} func (m *mockLogger) Info(v ...any) {} func (m *mockLogger) Infof(format string, v ...any) {} +func (m *mockLogger) Debug(v ...any) {} +func (m *mockLogger) Debugf(format string, v ...any) {} func TestPool_OverflowAndClosedScenarios(t *testing.T) { serverAddr := startTestServer(t) diff --git a/cmd/client.go b/cmd/client.go index f464551..5653a3e 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -142,8 +142,22 @@ EXAMPLES: os.Exit(1) } - // Output hash based on format + // Verify the SET by immediately doing a GET + // This ensures the packet was transmitted and stored hash := n.Hash() + verifyCtx, verifyCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer verifyCancel() + + verifiedNeedle, err := c.Get(verifyCtx, hash) + if err != nil { + // SET might have succeeded but verification failed + // Still output the hash but warn the user + fmt.Fprintf(os.Stderr, "Warning: Could not verify SET operation (GET failed): %v\n", err) + } else if verifiedNeedle.Hash() != hash { + fmt.Fprintf(os.Stderr, "Warning: SET verification returned different hash\n") + } + + // Output hash based on format hashHex := hex.EncodeToString(hash[:]) switch *format { case "hex": diff --git a/cmd/serve.go b/cmd/serve.go index ead03f2..848e568 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -20,12 +20,12 @@ func runServe(args []string) { fs := flag.NewFlagSet("serve", flag.ExitOnError) // Define flags - port := fs.String("port", "1337", "Port for the server listener") - fs.StringVar(port, "p", "1337", "Port for the server listener (shorthand)") - host := fs.String("host", "", "Hostname of server listener") - storageType := fs.String("storage", "memory", "Storage backend: memory or mmap") - dataDir := fs.String("data-dir", "./data", "Data directory for mmap storage") - quiet := fs.Bool("quiet", false, "Disable logging output") + addr := fs.String("addr", getAddr(), "Server address (host:port)") + fs.StringVar(addr, "a", getAddr(), "Server address (host:port) (shorthand)") + storageType := fs.String("storage", getStorage(), "Storage backend: memory or mmap") + dataDir := fs.String("data-dir", getDataDir(), "Data directory for mmap storage") + logLevel := fs.String("log-level", getLogLevel(), "Log level: debug, info, error, or silent") + quiet := fs.Bool("quiet", false, "Disable logging output (same as --log-level=silent)") fs.BoolVar(quiet, "q", false, "Disable logging output (shorthand)") help := fs.Bool("help", false, "Show server command help") fs.BoolVar(help, "h", false, "Show server command help (shorthand)") @@ -38,18 +38,24 @@ USAGE: haystack serve [options] OPTIONS: - -p, --port Port for the server listener (default: 1337) - --host Hostname of server listener (default: "") - --storage Storage backend: memory or mmap (default: memory) - --data-dir Data directory for mmap storage (default: ./data) - -q, --quiet Disable logging output + -a, --addr Server address (host:port) (default: %s) + --storage Storage backend: memory or mmap (default: %s) + --data-dir Data directory for mmap storage (default: %s) + --log-level Log level: debug, info, error, or silent (default: %s) + -q, --quiet Disable logging output (same as --log-level=silent) -h, --help Show this help message +ENVIRONMENT VARIABLES: + HAYSTACK_ADDR Server address (overridden by --addr) + HAYSTACK_STORAGE Storage backend (overridden by --storage) + HAYSTACK_DATA_DIR Data directory (overridden by --data-dir) + HAYSTACK_LOG_LEVEL Log level (overridden by --log-level) + DESCRIPTION: Server mode is used to run long-lived haystack servers. Memory storage keeps data in RAM only. MMAP storage persists data to disk using memory-mapped files. -`) +`, getAddr(), getStorage(), getDataDir(), getLogLevel()) } // Parse flags @@ -63,16 +69,19 @@ DESCRIPTION: return } - // Build address - addr := *host + ":" + *port + // Determine effective log level + effectiveLogLevel := *logLevel + if *quiet { + effectiveLogLevel = "silent" + } - // Set up logger based on quiet flag + // Set up logger based on log level var log logger.Logger - if *quiet { + if effectiveLogLevel == "silent" { log = logger.NewNoOp() } else { - log = logger.New() - fmt.Printf("listening on: %s\n", addr) + log = logger.NewWithLevel(effectiveLogLevel) + fmt.Printf("listening on: %s (log level: %s)\n", *addr, effectiveLogLevel) } // Create storage backend based on type @@ -116,7 +125,7 @@ DESCRIPTION: // Handle graceful shutdown go func() { - if err := srv.ListenAndServe(addr); err != nil { + if err := srv.ListenAndServe(*addr); err != nil { log.Fatalf("Server error: %v", err) } }() @@ -147,3 +156,35 @@ DESCRIPTION: fmt.Println("Server stopped") } } + +// getAddr returns the server address from environment or default +func getAddr() string { + if addr := os.Getenv("HAYSTACK_ADDR"); addr != "" { + return addr + } + return ":1337" +} + +// getStorage returns the storage type from environment or default +func getStorage() string { + if storage := os.Getenv("HAYSTACK_STORAGE"); storage != "" { + return storage + } + return "memory" +} + +// getDataDir returns the data directory from environment or default +func getDataDir() string { + if dataDir := os.Getenv("HAYSTACK_DATA_DIR"); dataDir != "" { + return dataDir + } + return "./data" +} + +// getLogLevel returns the log level from environment or default +func getLogLevel() string { + if level := os.Getenv("HAYSTACK_LOG_LEVEL"); level != "" { + return level + } + return "info" +} diff --git a/examples/deployments/fly.io/.gitignore b/examples/deployments/fly.io/.gitignore new file mode 100644 index 0000000..d1b57c8 --- /dev/null +++ b/examples/deployments/fly.io/.gitignore @@ -0,0 +1,5 @@ +# Generated files +fly.toml.generated + +# Fly.io state +.fly/ \ No newline at end of file diff --git a/examples/deployments/fly.io/README.md b/examples/deployments/fly.io/README.md new file mode 100644 index 0000000..ecfa6d4 --- /dev/null +++ b/examples/deployments/fly.io/README.md @@ -0,0 +1,219 @@ +# Haystack Fly.io Deployment + +This is a reference implementation for deploying Haystack to [Fly.io](https://fly.io), a platform that provides global app deployment with excellent support for UDP services. + +## Overview + +This deployment uses Haystack's hermetic build system to ensure: +- Only committed code is deployed +- Deployments are idempotent (same tree hash = no redeploy) +- Images are pulled from Docker Hub, not built on Fly.io +- Automatic tracking of deployed versions + +**Storage Configuration:** +- Uses mmap storage backend for persistence (configured via `HAYSTACK_STORAGE` environment variable) +- 10GB persistent volume mounted at `/data` (configured via `HAYSTACK_DATA_DIR` environment variable) +- Data survives container restarts and deployments +- Efficient memory-mapped file access + +## Prerequisites + +1. **Fly.io Account and CLI** + ```bash + # Install flyctl + brew install flyctl # macOS + # or + curl -L https://fly.io/install.sh | sh # Linux + + # Authenticate + fly auth login + ``` + +2. **Docker Hub** + - Haystack images must be pushed to Docker Hub + - Default repository: `nomasters/haystack` + - Or configure your own with `DOCKER_REPO` environment variable + +3. **Built and Pushed Images** + ```bash + # From the repository root + cd ../../../ # Go to haystack root + make docker-push + ``` + +## Quick Start + +1. **Deploy with defaults:** + ```bash + ./deploy.sh + ``` + + Note: On first deployment, Fly.io will automatically create a 10GB persistent volume for mmap storage. + +2. **Custom configuration:** + ```bash + FLY_APP_NAME=my-haystack FLY_REGION=lax ./deploy.sh + ``` + +## How It Works + +The `deploy.sh` script: + +1. **Calculates tree hash** from current HEAD commit +2. **Verifies Docker image** exists with that tree hash +3. **Checks current deployment** to see if already running same version +4. **Generates fly.toml** with the correct image tag +5. **Deploys only if needed** (new tree hash or first deployment) + +## Configuration + +### Environment Variables + +- `FLY_APP_NAME`: Fly.io app name (default: `haystack-kv`) +- `FLY_REGION`: Primary region (default: `ord` - Chicago) +- `DOCKER_REPO`: Docker repository (default: `nomasters/haystack`) +- `DOCKER_REGISTRY`: Docker registry (default: `docker.io`) + +### Regions + +Common Fly.io regions: +- `ord` - Chicago +- `lax` - Los Angeles +- `sea` - Seattle +- `ewr` - New Jersey +- `lhr` - London +- `ams` - Amsterdam +- `nrt` - Tokyo +- `syd` - Sydney + +## Testing Your Deployment + +Once deployed, test your Haystack instance: + +```bash +# Get your app's URL +fly status --app haystack-kv + +# Test with haystack client +echo "hello world" | haystack client set --endpoint haystack-kv.fly.dev:1337 + +# Get the hash and retrieve +haystack client get --endpoint haystack-kv.fly.dev:1337 +``` + +## UDP on Fly.io + +Important notes about UDP on Fly.io: +- UDP services get a dedicated IPv4 address +- No load balancing for UDP (each instance has its own IP) +- UDP ports are exposed globally +- Firewall rules can be configured if needed + +## Scaling + +For production deployments, you may want multiple instances: + +```bash +# Scale to multiple regions +fly scale count 3 --app haystack-kv + +# Set specific regions +fly regions set ord lax lhr --app haystack-kv +``` + +## Monitoring + +```bash +# View logs +fly logs --app haystack-kv + +# Check status +fly status --app haystack-kv + +# SSH into instance (for debugging) +fly ssh console --app haystack-kv + +# Check volume usage +fly ssh console --app haystack-kv -C "df -h /data" + +# View mmap storage files +fly ssh console --app haystack-kv -C "ls -la /data" +``` + +## Customization + +### Using Your Own Docker Images + +1. Fork the Haystack repository +2. Configure your Docker Hub repository: + ```bash + export DOCKER_REPO=yourusername/haystack + ``` +3. Build and push your images: + ```bash + make docker-push + ``` +4. Deploy: + ```bash + DOCKER_REPO=yourusername/haystack ./deploy.sh + ``` + +### Modifying fly.toml + +The `fly.toml` file can be customized for your needs: +- Adjust VM resources (`cpus`, `memory_mb`) +- Modify environment variables (`HAYSTACK_ADDR`, `HAYSTACK_STORAGE`, `HAYSTACK_DATA_DIR`) +- Configure persistent volumes +- Set up multiple regions + +## Troubleshooting + +### Image Not Found + +If you get "Docker image not found": +```bash +# Check if image exists locally +docker images | grep haystack + +# Push to Docker Hub +cd ../../../ # Go to repo root +make docker-push +``` + +### App Already Exists + +If the app name is taken: +```bash +FLY_APP_NAME=my-unique-haystack ./deploy.sh +``` + +### Connection Issues + +UDP services on Fly.io: +- Make sure you're using port 1337 +- Use the full hostname: `your-app.fly.dev:1337` +- Check firewall rules if applicable + +## Clean Up + +To remove your deployment: +```bash +fly apps destroy haystack-kv +``` + +## Security Considerations + +- Haystack has no authentication by design +- Consider Fly.io's firewall features if you need access control +- Data is ephemeral (TTL-based expiration) +- No encryption in transit (add if needed for your use case) + +## Support + +- [Fly.io Documentation](https://fly.io/docs/) +- [Haystack Repository](https://github.com/nomasters/haystack) +- [Fly.io Community](https://community.fly.io/) + +## License + +This deployment example is provided under the same license as Haystack (The Unlicense). \ No newline at end of file diff --git a/examples/deployments/fly.io/deploy.sh b/examples/deployments/fly.io/deploy.sh new file mode 100755 index 0000000..5891cf2 --- /dev/null +++ b/examples/deployments/fly.io/deploy.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# deploy.sh - Deploy Haystack to Fly.io using tree hash from HEAD +# +# This script: +# 1. Gets the tree hash from HEAD +# 2. Verifies the Docker image exists +# 3. Substitutes TREE_HASH in fly.toml +# 4. Deploys to Fly.io + +set -euo pipefail + +# Configuration +DOCKER_REPO="${DOCKER_REPO:-nomasters/haystack}" +DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Get tree hash from HEAD +get_tree_hash() { + local repo_root + repo_root=$(git rev-parse --show-toplevel) + + if [ -f "${repo_root}/scripts/tree-hash.sh" ]; then + (cd "$repo_root" && ./scripts/tree-hash.sh) + else + log_error "Cannot find scripts/tree-hash.sh" + exit 1 + fi +} + +# Main deployment +main() { + log_info "=== Haystack Fly.io Deployment ===" + + # Get the tree hash + local tree_hash + tree_hash=$(get_tree_hash) + log_info "Tree hash: ${tree_hash}" + + # Build the image tag + local image_tag="${DOCKER_REGISTRY}/${DOCKER_REPO}:tree-${tree_hash}" + + # Check if the Docker image exists + log_info "Checking for image: ${image_tag}" + if ! docker manifest inspect "${image_tag}" >/dev/null 2>&1; then + log_error "Image not found: ${image_tag}" + log_error "Run 'make docker-push' from repository root first" + exit 1 + fi + log_info "✓ Image found" + + # Substitute TREE_HASH in fly.toml + log_info "Generating fly.toml with tree hash: ${tree_hash}" + sed "s/TREE_HASH/${tree_hash}/g" fly.toml > fly.toml.generated + + # Show what we're deploying + log_info "Deploying image: ${image_tag}" + + # Deploy to Fly.io + if fly deploy --config fly.toml.generated; then + log_info "✓ Deployment successful!" + + # Clean up generated file + rm -f fly.toml.generated + + # Show app info + fly status + else + log_error "Deployment failed!" + rm -f fly.toml.generated + exit 1 + fi +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/examples/deployments/fly.io/fly.toml b/examples/deployments/fly.io/fly.toml new file mode 100644 index 0000000..43daedf --- /dev/null +++ b/examples/deployments/fly.io/fly.toml @@ -0,0 +1,41 @@ +# fly.toml app configuration file generated for haystack-example-trunk on 2025-08-09T20:54:42-06:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'haystack-example-trunk' +primary_region = 'dfw' + +[build] +image = 'nomasters/haystack:tree-TREE_HASH' + +[deploy] +strategy = 'immediate' + +# Environment variables for haystack configuration +[env] +HAYSTACK_ADDR = 'fly-global-services:1337' +HAYSTACK_STORAGE = 'memory' +HAYSTACK_DATA_DIR = '/data' +HAYSTACK_LOG_LEVEL = 'debug' + +# Persistent volume for mmap storage +[mounts] +source = 'haystack_data' +destination = '/data' +initial_size = '10gb' + +# Simple command now that configuration is via environment variables +[processes] +app = 'serve' + +[[services]] +protocol = 'udp' +internal_port = 1337 +processes = ['app'] + +[[services.ports]] +port = 1337 + +[[vm]] +size = 'shared-cpu-1x' diff --git a/logger/logger.go b/logger/logger.go index b9a9fef..4df61e5 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -14,15 +14,33 @@ type Logger interface { Errorf(format string, v ...any) Info(v ...any) Infof(format string, v ...any) - // Debug(v ...any) - // Debugf(format string, v ...any) + Debug(v ...any) + Debugf(format string, v ...any) } // New returns a SlogLogger reference that satisfies the Logger interface. func New() *SlogLogger { + return NewWithLevel("info") +} + +// NewWithLevel returns a SlogLogger with the specified log level. +// Valid levels: "debug", "info", "error" +func NewWithLevel(level string) *SlogLogger { + var logLevel slog.Level + switch level { + case "debug": + logLevel = slog.LevelDebug + case "error": + logLevel = slog.LevelError + case "info": + fallthrough + default: + logLevel = slog.LevelInfo + } + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ AddSource: false, - Level: slog.LevelInfo, + Level: logLevel, }) logger := slog.New(handler) return &SlogLogger{logger: logger} @@ -65,6 +83,16 @@ func (s *SlogLogger) Infof(format string, v ...any) { s.logger.Info(fmt.Sprintf(format, v...)) } +// Debug starts a new message at the debug level in the logger +func (s *SlogLogger) Debug(v ...any) { + s.logger.Debug(fmt.Sprint(v...)) +} + +// Debugf starts a formatted message at the debug level in the logger +func (s *SlogLogger) Debugf(format string, v ...any) { + s.logger.Debug(fmt.Sprintf(format, v...)) +} + // NoOpLogger is a logger that does nothing - useful for silent mode type NoOpLogger struct{} @@ -94,3 +122,9 @@ func (n *NoOpLogger) Info(v ...any) {} // Infof does nothing in NoOp mode func (n *NoOpLogger) Infof(format string, v ...any) {} + +// Debug does nothing in NoOp mode +func (n *NoOpLogger) Debug(v ...any) {} + +// Debugf does nothing in NoOp mode +func (n *NoOpLogger) Debugf(format string, v ...any) {} diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index fff08c1..1806df6 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -81,11 +81,21 @@ build_image() { # This avoids "existing instance" errors if docker buildx ls | grep -q "haystack-builder"; then log_info "Removing existing buildx builder..." - docker buildx rm haystack-builder 2>/dev/null || true + docker buildx rm --force haystack-builder || { + log_warn "Failed to remove builder, attempting to use existing one..." + docker buildx use haystack-builder + } fi - log_info "Creating buildx builder..." - docker buildx create --name haystack-builder --use --bootstrap + # Only create if it doesn't exist after removal attempt + if ! docker buildx ls | grep -q "haystack-builder"; then + log_info "Creating buildx builder..." + docker buildx create --name haystack-builder --use --bootstrap + else + log_info "Using existing haystack-builder..." + docker buildx use haystack-builder + docker buildx inspect --bootstrap + fi # Build the image with all tags local build_args=( diff --git a/server/server.go b/server/server.go index aadf7a9..62b0d0a 100644 --- a/server/server.go +++ b/server/server.go @@ -158,12 +158,15 @@ func (s *Server) processPacket() error { switch n { case needle.HashLength: // GET operation: 32-byte hash query + s.logger.Infof("GET request from %s", addr) return s.handleGet(buf[:n], addr) case needle.NeedleLength: // SET operation: 192-byte needle storage + s.logger.Infof("SET request from %s", addr) return s.handleSet(buf[:n], addr) default: - // Invalid packet size, silently drop + // Invalid packet size, log and drop + s.logger.Debugf("Invalid packet size %d from %s", n, addr) return nil } } @@ -177,12 +180,19 @@ func (s *Server) handleGet(hashBytes []byte, addr net.Addr) error { // Retrieve needle from storage n, err := s.storage.Get(hash) if err != nil { + s.logger.Debugf("GET failed for hash %x: %v", hash, err) return fmt.Errorf("failed to get needle: %w", err) } // Send the full needle as response - _, err = s.conn.WriteTo(n.Bytes(), addr) - return err + bytesWritten, err := s.conn.WriteTo(n.Bytes(), addr) + if err != nil { + s.logger.Errorf("Failed to send GET response to %s: %v", addr, err) + return err + } + + s.logger.Debugf("GET response sent to %s (%d bytes)", addr, bytesWritten) + return nil } // handleSet processes a needle storage request. diff --git a/wc b/wc new file mode 100644 index 0000000..d88ddca Binary files /dev/null and b/wc differ