diff --git a/.env.dist b/.env.dist index e7d1fdd..97bb40f 100644 --- a/.env.dist +++ b/.env.dist @@ -1,3 +1,4 @@ APP_PORT= ROOT_DIRECTORY= -ALLOWED_FILE_TYPES= \ No newline at end of file +ALLOWED_FILE_TYPES= +SIGNING_SECRET= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 49eeb4c..ece7a43 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ *.so *.dylib +# Built binaries +file-storage + # Test binary, built with `go test -c` *.test diff --git a/Readme.md b/Readme.md index 4e39862..8148651 100644 --- a/Readme.md +++ b/Readme.md @@ -2,6 +2,13 @@ This is a file storage server for the MAXIT project. +## Features + +- **S3-like API**: Bucket and object management +- **Signed URLs**: Time-limited, secure access to PDF files +- **Simple deployment**: Single binary with filesystem storage +- **HMAC-SHA256 signatures**: Cryptographically secure URL signing + ## Build Prerequisites: @@ -36,12 +43,34 @@ To set up and run the File Storage API, follow these steps: cp .env.dist .env ``` Update the `.env` file with the necessary environment variables. + + **Important**: Set `SIGNING_SECRET` for signed URL support: + ```bash + SIGNING_SECRET=your-secret-key-here + ``` + 4. **Run the Application**: To run the application, you can use the prepared `Makefile`. jut run: ```bash make ``` +## Features + +### Signed URLs for Secure File Access + +The file storage service supports signed, time-limited URLs for secure PDF access. See [SIGNED_URLS.md](./SIGNED_URLS.md) for detailed documentation. + +Quick example: +```go +storage, _ := filestorage.NewFileStorage(filestorage.FileStorageConfig{ + URL: "https://storage.example.com", +}) + +// Generate a URL valid for 1 hour +signedURL, _ := storage.GetSignedFileURL("bucket", "file.pdf", 1*time.Hour, "secret") +``` + ## Endpoints OpenAPI 3.0 specification: [api.raml](./api.raml) diff --git a/SIGNED_URLS.md b/SIGNED_URLS.md new file mode 100644 index 0000000..b06637a --- /dev/null +++ b/SIGNED_URLS.md @@ -0,0 +1,230 @@ +# Signed URLs for Secure File Access + +This file storage service supports signed, time-limited URLs for secure access to PDF files. This allows you to generate temporary URLs that expire after a specified duration, providing secure file delivery without requiring authentication at the storage level. + +## Features + +- **Time-limited access**: URLs expire after a configurable TTL (Time To Live) +- **Cryptographic signatures**: Uses HMAC-SHA256 to ensure URL integrity +- **Automatic validation**: Middleware validates signatures and expiration on every request +- **PDF-specific**: Currently enforces signed URLs only for PDF files +- **No authentication required**: Clients don't need credentials, just the signed URL + +## Configuration + +Set the `SIGNING_SECRET` environment variable in your `.env` file: + +```bash +SIGNING_SECRET=your-secret-key-here +``` + +**Important**: Use a strong, randomly generated secret in production. The secret must be kept confidential and should be rotated periodically. + +## Usage + +### Generating Signed URLs (Client-side) + +Use the `GetSignedFileURL` method to generate a signed URL: + +```go +package main + +import ( + "fmt" + "time" + "github.com/mini-maxit/file-storage/pkg/filestorage" +) + +func main() { + // Create file storage client + config := filestorage.FileStorageConfig{ + URL: "https://storage.example.com", + } + storage, err := filestorage.NewFileStorage(config) + if err != nil { + panic(err) + } + + // Generate a signed URL valid for 1 hour + bucketName := "task-pdfs" + objectKey := "task-123/description.pdf" + ttl := 1 * time.Hour + signingSecret := "your-secret-key-here" // Must match server secret + + signedURL, err := storage.GetSignedFileURL(bucketName, objectKey, ttl, signingSecret) + if err != nil { + panic(err) + } + + fmt.Printf("Signed URL: %s\n", signedURL) + // Output: https://storage.example.com/buckets/task-pdfs/task-123/description.pdf?expires=1234567890&signature=... +} +``` + +### URL Structure + +A signed URL includes: +- **Base path**: `/buckets/{bucketName}/{objectKey}` +- **expires**: Unix timestamp indicating when the URL expires +- **signature**: HMAC-SHA256 signature of the path and expiration + +Example: +``` +https://storage.example.com/buckets/task-pdfs/task-123/description.pdf?expires=1737804000&signature=abc123... +``` + +### Server-side Validation + +The server automatically validates: +1. **Signature authenticity**: Ensures the URL hasn't been tampered with +2. **Expiration**: Rejects URLs past their expiration time +3. **Method**: Only GET requests are allowed with signed URLs + +### Error Responses + +| Scenario | HTTP Status | Response | +|----------|------------|----------| +| Valid signed URL | 200 OK | File content | +| Expired URL | 403 Forbidden | "Forbidden: URL has expired" | +| Invalid/tampered signature | 403 Forbidden | "Forbidden: invalid signature" | +| Missing signature (PDF) | 403 Forbidden | "Forbidden: signature required for PDF file access" | +| Metadata-only request | 200 OK | Metadata JSON (no signature required) | + +## Security Considerations + +1. **Secret Management**: + - Store the signing secret securely (environment variables, secrets manager) + - Never commit secrets to version control + - Use different secrets for different environments + - Rotate secrets periodically + +2. **TTL Selection**: + - Use the shortest TTL practical for your use case + - For temporary downloads: 5-15 minutes + - For email links: 1-24 hours + - For public sharing: consider access control implications + +3. **HTTPS Only**: + - Always serve files over HTTPS to prevent URL interception + - The signature protects against tampering, but HTTPS protects against eavesdropping + +4. **Signature in Query Parameters**: + - Allows CDN caching + - Ensures URLs are self-contained and shareable + - Note: Query parameters may appear in server logs + +## File Type Restrictions + +Currently, signature validation is enforced only for PDF files (`.pdf` extension). Other file types can be accessed without signatures, though signatures are still validated if provided. + +To extend this to all files or specific file types, modify the `isPDFFile` function in `internal/api/http/middleware/signature.go`. + +## Implementation Details + +### URL Signing Algorithm + +The signature is generated using HMAC-SHA256: + +``` +stringToSign = "{path}:{expiresTimestamp}" +signature = base64_url_encode(HMAC_SHA256(secret, stringToSign)) +``` + +### Middleware Flow + +1. Request arrives at server +2. Middleware checks if it's a GET request to an object endpoint +3. If signature parameters are present, validates them +4. If path is a PDF and no signature is present, rejects with 403 +5. If validation passes, forwards to the handler +6. Handler serves the file + +## Testing + +Run the test suite to verify signed URL functionality: + +```bash +# Test URL signer +go test ./pkg/urlsigner/ + +# Test middleware +go test ./internal/api/http/middleware/ + +# Test client library +go test ./pkg/filestorage/ + +# Run all tests +go test ./... +``` + +## Examples + +### Example 1: Generate and Use a Signed URL + +```go +// Server configuration +os.Setenv("SIGNING_SECRET", "my-secret-key") + +// Client generates signed URL +storage, _ := filestorage.NewFileStorage(filestorage.FileStorageConfig{ + URL: "https://storage.example.com", +}) + +signedURL, _ := storage.GetSignedFileURL("docs", "manual.pdf", 15*time.Minute, "my-secret-key") + +// User clicks the link and downloads the file +// URL is valid for 15 minutes +``` + +### Example 2: Backend Integration + +```go +// In your backend service that manages tasks +func generateTaskPDFLink(taskID string) (string, error) { + storage, err := filestorage.NewFileStorage(filestorage.FileStorageConfig{ + URL: os.Getenv("FILE_STORAGE_URL"), + }) + if err != nil { + return "", err + } + + bucketName := "task-pdfs" + objectKey := fmt.Sprintf("task-%s/description.pdf", taskID) + ttl := 1 * time.Hour + secret := os.Getenv("SIGNING_SECRET") + + return storage.GetSignedFileURL(bucketName, objectKey, ttl, secret) +} +``` + +## Troubleshooting + +### "Forbidden: invalid signature" +- Ensure the signing secret matches on both client and server +- Check that the URL hasn't been modified after generation +- Verify the path is exactly as signed (including case) + +### "Forbidden: URL has expired" +- The URL's TTL has passed +- Generate a new signed URL +- Consider increasing TTL if users need more time + +### "Forbidden: signature required for PDF file access" +- PDF files require signed URLs +- Generate a signed URL before accessing the file +- For metadata access, use `?metadataOnly=true` + +## Migration Notes + +If you have existing clients accessing PDFs without signatures: +1. Deploy the server with signature validation +2. Update clients to generate signed URLs +3. Consider a grace period with optional signatures before enforcing +4. Monitor logs for signature validation failures + +## API Compatibility + +This feature maintains backward compatibility: +- Metadata requests (`?metadataOnly=true`) don't require signatures +- Non-PDF files don't require signatures (currently) +- Upload, delete, and bucket operations are unaffected diff --git a/cmd/app/integration_test.go b/cmd/app/integration_test.go new file mode 100644 index 0000000..ff3d541 --- /dev/null +++ b/cmd/app/integration_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/mini-maxit/file-storage/internal/api/http/server" + "github.com/mini-maxit/file-storage/internal/api/services" + "github.com/mini-maxit/file-storage/internal/config" + "github.com/mini-maxit/file-storage/internal/logger" + "github.com/mini-maxit/file-storage/pkg/urlsigner" +) + +func TestSignedURLIntegration(t *testing.T) { + // Setup + tempDir := t.TempDir() + testSecret := "test-signing-secret" + + // Create config + cfg := &config.Config{ + Port: "8080", + RootDirectory: tempDir, + SigningSecret: testSecret, + } + + // Create file service + fileService := services.NewFileService(cfg) + + // Create URL signer + signer := urlsigner.NewURLSigner(testSecret) + + // Initialize logger + logger.InitializeLogger() + log := logger.NewNamedLogger("test") + + // Create server + srv := server.NewServer(fileService, signer, log) + + // Create a test bucket + createBucket(t, srv, "test-bucket") + + // Upload a PDF file + uploadPDF(t, srv, "test-bucket", "test.pdf", []byte("PDF content")) + + // Test 1: Valid signed URL should work + t.Run("ValidSignedURL", func(t *testing.T) { + path := "/buckets/test-bucket/test.pdf" + signedURL, err := signer.SignURL(path, 1*time.Hour) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, signedURL, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Expected 200, got %d: %s", w.Code, w.Body.String()) + } + + body, _ := io.ReadAll(w.Body) + if string(body) != "PDF content" { + t.Errorf("Expected 'PDF content', got %s", string(body)) + } + }) + + // Test 2: Expired signed URL should fail + t.Run("ExpiredSignedURL", func(t *testing.T) { + path := "/buckets/test-bucket/test.pdf" + signedURL, err := signer.SignURL(path, -1*time.Hour) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, signedURL, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("Expected 403, got %d", w.Code) + } + }) + + // Test 3: Invalid signature should fail + t.Run("InvalidSignature", func(t *testing.T) { + path := "/buckets/test-bucket/test.pdf" + expires := time.Now().Add(1 * time.Hour).Unix() + invalidURL := fmt.Sprintf("%s?expires=%d&signature=invalid", path, expires) + + req := httptest.NewRequest(http.MethodGet, invalidURL, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("Expected 403, got %d", w.Code) + } + }) + + // Test 4: Missing signature for PDF should fail + t.Run("MissingSignature", func(t *testing.T) { + path := "/buckets/test-bucket/test.pdf" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("Expected 403, got %d", w.Code) + } + }) + + // Test 5: Metadata request should work without signature + t.Run("MetadataWithoutSignature", func(t *testing.T) { + path := "/buckets/test-bucket/test.pdf?metadataOnly=true" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Expected 200 for metadata request, got %d: %s", w.Code, w.Body.String()) + } + }) + + // Test 6: Non-PDF file should work without signature + t.Run("NonPDFWithoutSignature", func(t *testing.T) { + uploadFile(t, srv, "test-bucket", "test.txt", []byte("Text content")) + + path := "/buckets/test-bucket/test.txt" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Expected 200 for non-PDF file, got %d: %s", w.Code, w.Body.String()) + } + }) +} + +// Helper function to create a bucket +func createBucket(t *testing.T, srv http.Handler, bucketName string) { + body := fmt.Sprintf(`{"name":"%s"}`, bucketName) + req := httptest.NewRequest(http.MethodPost, "/buckets", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusCreated && w.Code != http.StatusConflict { + t.Fatalf("Failed to create bucket: %d - %s", w.Code, w.Body.String()) + } +} + +// Helper function to upload a PDF +func uploadPDF(t *testing.T, srv http.Handler, bucketName, fileName string, content []byte) { + uploadFile(t, srv, bucketName, fileName, content) +} + +// Helper function to upload any file +func uploadFile(t *testing.T, srv http.Handler, bucketName, fileName string, content []byte) { + var b bytes.Buffer + writer := multipart.NewWriter(&b) + + part, err := writer.CreateFormFile("file", fileName) + if err != nil { + t.Fatalf("Failed to create form file: %v", err) + } + + _, err = part.Write(content) + if err != nil { + t.Fatalf("Failed to write file content: %v", err) + } + + writer.Close() + + url := fmt.Sprintf("/buckets/%s/%s", bucketName, fileName) + req := httptest.NewRequest(http.MethodPut, url, &b) + req.Header.Set("Content-Type", writer.FormDataContentType()) + w := httptest.NewRecorder() + + srv.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Failed to upload file: %d - %s", w.Code, w.Body.String()) + } +} diff --git a/cmd/app/main.go b/cmd/app/main.go index b300c78..2a6648c 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/mini-maxit/file-storage/internal/api/services" "github.com/mini-maxit/file-storage/internal/logger" + "github.com/mini-maxit/file-storage/pkg/urlsigner" "os" "github.com/joho/godotenv" @@ -29,11 +30,14 @@ func main() { fileService := services.NewFileService(_config) + // Create URL signer with the configured secret + signer := urlsigner.NewURLSigner(_config.SigningSecret) + logger.InitializeLogger() log := logger.NewNamedLogger("server") addr := ":" + _config.Port - _server := server.NewServer(fileService, log) + _server := server.NewServer(fileService, signer, log) err = _server.Run(addr) if err != nil { logrus.Fatalf("server stopped: %v", err) diff --git a/go.mod b/go.mod index 06e569c..9a7dbf2 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.1 go.uber.org/zap v1.27.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( @@ -14,6 +15,5 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/sys v0.5.0 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/api/http/middleware/signature.go b/internal/api/http/middleware/signature.go new file mode 100644 index 0000000..0a0da7c --- /dev/null +++ b/internal/api/http/middleware/signature.go @@ -0,0 +1,73 @@ +package middleware + +import ( + "net/http" + "strings" + + "github.com/mini-maxit/file-storage/pkg/urlsigner" + "go.uber.org/zap" +) + +// SignatureValidationMiddleware validates signed URLs for GET requests to object endpoints +func SignatureValidationMiddleware(next http.Handler, signer *urlsigner.URLSigner, log *zap.SugaredLogger) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Only validate GET requests to object endpoints (not metadata-only requests) + // Pattern: /buckets/{bucketName}/{objectKey} + if r.Method == http.MethodGet && isObjectEndpoint(r.URL.Path) { + query := r.URL.Query() + + // Check if this is a metadata-only request (no signature required) + metadataOnly := strings.ToLower(query.Get("metadataOnly")) == "true" + + // Check if signature parameters are present + expires := query.Get("expires") + signature := query.Get("signature") + + // If either expires or signature is present, validate both + if expires != "" || signature != "" { + // Validate the signature + err := signer.ValidateSignedURL(r.URL.Path, expires, signature) + if err != nil { + log.Warnf("Signature validation failed for %s: %v", r.URL.Path, err) + http.Error(w, "Forbidden: "+err.Error(), http.StatusForbidden) + return + } + log.Debugf("Signature validated successfully for %s", r.URL.Path) + } else if !metadataOnly && isPDFFile(r.URL.Path) { + // Only require signatures for PDF file downloads (not metadata) + log.Warnf("Missing signature for PDF file access: %s", r.URL.Path) + http.Error(w, "Forbidden: signature required for PDF file access", http.StatusForbidden) + return + } + } + + // Continue with the next handler + next.ServeHTTP(w, r) + }) +} + +// isObjectEndpoint checks if the path is an object endpoint +// Pattern: /buckets/{bucketName}/{objectKey} +func isObjectEndpoint(path string) bool { + // Remove leading slash + path = strings.TrimPrefix(path, "/") + + // Split by / + parts := strings.SplitN(path, "/", 3) + + // Must have at least 3 parts: buckets, bucketName, objectKey + // and the first part must be "buckets" + if len(parts) >= 3 && parts[0] == "buckets" { + // Check that it's not a special endpoint like upload-multiple or remove-multiple + if parts[2] != "upload-multiple" && parts[2] != "remove-multiple" { + return true + } + } + + return false +} + +// isPDFFile checks if the path ends with .pdf extension +func isPDFFile(path string) bool { + return strings.HasSuffix(strings.ToLower(path), ".pdf") +} diff --git a/internal/api/http/middleware/signature_test.go b/internal/api/http/middleware/signature_test.go new file mode 100644 index 0000000..00fe0b7 --- /dev/null +++ b/internal/api/http/middleware/signature_test.go @@ -0,0 +1,324 @@ +package middleware + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/mini-maxit/file-storage/pkg/urlsigner" + "go.uber.org/zap" +) + +func TestSignatureValidationMiddleware_ValidSignature(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that returns 200 OK + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Sign a URL + path := "/buckets/test-bucket/test-file.pdf" + signedURL, err := signer.SignURL(path, 1*time.Hour) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Create a request with the signed URL + req := httptest.NewRequest(http.MethodGet, signedURL, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was successful + if w.Code != http.StatusOK { + t.Errorf("Expected status 200, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_ExpiredSignature(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that should not be reached + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("Handler should not be called for expired signature") + w.WriteHeader(http.StatusOK) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Sign a URL with negative TTL (already expired) + path := "/buckets/test-bucket/test-file.pdf" + signedURL, err := signer.SignURL(path, -1*time.Hour) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Create a request with the expired signed URL + req := httptest.NewRequest(http.MethodGet, signedURL, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was forbidden + if w.Code != http.StatusForbidden { + t.Errorf("Expected status 403, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_InvalidSignature(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that should not be reached + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("Handler should not be called for invalid signature") + w.WriteHeader(http.StatusOK) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a URL with invalid signature + path := "/buckets/test-bucket/test-file.pdf" + expires := fmt.Sprintf("%d", time.Now().Add(1*time.Hour).Unix()) + invalidSignedURL := fmt.Sprintf("%s?expires=%s&signature=invalid-signature", path, expires) + + // Create a request with the invalid signed URL + req := httptest.NewRequest(http.MethodGet, invalidSignedURL, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was forbidden + if w.Code != http.StatusForbidden { + t.Errorf("Expected status 403, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_MissingSignature(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that should not be reached for PDF files without signature + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("Handler should not be called for PDF without signature") + w.WriteHeader(http.StatusOK) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a request without signature for a PDF file + path := "/buckets/test-bucket/test-file.pdf" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was forbidden + if w.Code != http.StatusForbidden { + t.Errorf("Expected status 403, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_MissingSignatureNonPDF(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that returns 200 OK + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a request without signature for a non-PDF file + path := "/buckets/test-bucket/test-file.txt" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was successful (no signature required for non-PDF files) + if w.Code != http.StatusOK { + t.Errorf("Expected status 200 for non-PDF file, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_MetadataOnlyRequest(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that returns 200 OK + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a metadata-only request (no signature required) + path := "/buckets/test-bucket/test-file.pdf?metadataOnly=true" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was successful (no signature required for metadata-only) + if w.Code != http.StatusOK { + t.Errorf("Expected status 200 for metadata-only request, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_NonGetRequest(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that returns 200 OK + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a PUT request (signature validation only applies to GET) + path := "/buckets/test-bucket/test-file.pdf" + req := httptest.NewRequest(http.MethodPut, path, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was successful (no signature required for non-GET) + if w.Code != http.StatusOK { + t.Errorf("Expected status 200 for PUT request, got %d", w.Code) + } +} + +func TestSignatureValidationMiddleware_NonObjectEndpoint(t *testing.T) { + // Create a test signer + signer := urlsigner.NewURLSigner("test-secret") + + // Create a test logger + logger, _ := zap.NewDevelopment() + sugaredLogger := logger.Sugar() + + // Create a simple handler that returns 200 OK + nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Wrap the handler with the signature validation middleware + middleware := SignatureValidationMiddleware(nextHandler, signer, sugaredLogger) + + // Create a request to a non-object endpoint (e.g., bucket listing) + path := "/buckets/test-bucket" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + + // Execute the middleware + middleware.ServeHTTP(w, req) + + // Check that the request was successful (no signature required for bucket endpoints) + if w.Code != http.StatusOK { + t.Errorf("Expected status 200 for bucket endpoint, got %d", w.Code) + } +} + +func TestIsObjectEndpoint(t *testing.T) { + tests := []struct { + path string + expected bool + }{ + {"/buckets/test-bucket/test-file.pdf", true}, + {"/buckets/test-bucket/folder/test-file.pdf", true}, + {"/buckets/test-bucket", false}, + {"/buckets", false}, + {"/buckets/test-bucket/upload-multiple", false}, + {"/buckets/test-bucket/remove-multiple", false}, + {"/other/path", false}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + result := isObjectEndpoint(tt.path) + if result != tt.expected { + t.Errorf("isObjectEndpoint(%s) = %v, expected %v", tt.path, result, tt.expected) + } + }) + } +} + +func TestIsPDFFile(t *testing.T) { + tests := []struct { + path string + expected bool + }{ + {"/buckets/test-bucket/test-file.pdf", true}, + {"/buckets/test-bucket/test-file.PDF", true}, + {"/buckets/test-bucket/folder/document.pdf", true}, + {"/buckets/test-bucket/test-file.txt", false}, + {"/buckets/test-bucket/image.jpg", false}, + {"/buckets/test-bucket/archive.zip", false}, + {"/buckets/test-bucket/pdffile", false}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + result := isPDFFile(tt.path) + if result != tt.expected { + t.Errorf("isPDFFile(%s) = %v, expected %v", tt.path, result, tt.expected) + } + }) + } +} diff --git a/internal/api/http/server/router.go b/internal/api/http/server/router.go index c7e58fd..b0a5df9 100644 --- a/internal/api/http/server/router.go +++ b/internal/api/http/server/router.go @@ -12,6 +12,7 @@ import ( "github.com/mini-maxit/file-storage/internal/api/services" "github.com/mini-maxit/file-storage/internal/logger" "github.com/mini-maxit/file-storage/pkg/filestorage/entities" + "github.com/mini-maxit/file-storage/pkg/urlsigner" "go.uber.org/zap" ) @@ -21,6 +22,11 @@ type Server struct { logger *zap.SugaredLogger } +// ServeHTTP implements http.Handler interface for the Server +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + // Run starts the HTTP server. func (s *Server) Run(addr string) error { s.logger.Infof("Server is running on %s", addr) @@ -362,7 +368,7 @@ func deleteObjectHandler(fs *services.FileService, w http.ResponseWriter, bucket // NewServer sets up the routes, wraps the mux with HTTP logging middleware, // and returns the Server object. -func NewServer(fs *services.FileService, appLog *zap.SugaredLogger) *Server { +func NewServer(fs *services.FileService, signer *urlsigner.URLSigner, appLog *zap.SugaredLogger) *Server { // Create the base mux for our file storage API endpoints. mux := http.NewServeMux() @@ -434,8 +440,9 @@ func NewServer(fs *services.FileService, appLog *zap.SugaredLogger) *Server { // Retrieve an HTTP-specific logger. httpLog := logger.NewHttpLogger() - // Wrap our mux with HTTP logging middleware. - loggedMux := middleware.LoggingMiddleware(mux, httpLog) + // Wrap our mux with signature validation middleware first, then logging + signedMux := middleware.SignatureValidationMiddleware(mux, signer, httpLog) + loggedMux := middleware.LoggingMiddleware(signedMux, httpLog) return &Server{ mux: loggedMux, diff --git a/internal/config/config.go b/internal/config/config.go index e02d02e..d21fe25 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,9 +13,11 @@ import ( // It includes: // - Port: the port on which the server will run (defaults to "8080"). // - RootDirectory: the directory where tasks/files will be stored (defaults to "tasks/"). +// - SigningSecret: the secret key used for signing URLs (HMAC). type Config struct { Port string RootDirectory string + SigningSecret string } // NewConfig loads the application's configuration from environment variables or sets defaults @@ -37,8 +39,17 @@ func NewConfig() *Config { rootDirectory = "file-storage-media" } + signingSecret := os.Getenv("SIGNING_SECRET") + if signingSecret == "" { + log.Println("ERROR: SIGNING_SECRET not set. This is a security risk!") + log.Println("Please set SIGNING_SECRET environment variable for secure signed URL support.") + log.Println("Using insecure default for development only - DO NOT USE IN PRODUCTION") + signingSecret = "insecure-default-secret-change-this-immediately" + } + return &Config{ Port: port, RootDirectory: rootDirectory, + SigningSecret: signingSecret, } } diff --git a/pkg/filestorage/file_storage.go b/pkg/filestorage/file_storage.go index e8ebc44..c8ab7ca 100644 --- a/pkg/filestorage/file_storage.go +++ b/pkg/filestorage/file_storage.go @@ -15,6 +15,7 @@ import ( "time" "github.com/mini-maxit/file-storage/pkg/filestorage/entities" + "github.com/mini-maxit/file-storage/pkg/urlsigner" ) type FileStorage interface { @@ -30,6 +31,7 @@ type FileStorage interface { GetFile(bucketName string, objectKey string) ([]byte, error) GetFileURL(bucketName string, objectKey string) string + GetSignedFileURL(bucketName string, objectKey string, ttl time.Duration, signingSecret string) (string, error) GetFileMetadata(bucketName string, objectKey string) (*entities.Object, error) UploadFile(bucketName string, objectKey string, file *os.File) error DeleteFile(bucketName string, objectKey string) error @@ -449,6 +451,37 @@ func (fs *fileStorage) GetFileURL(bucketName string, objectKey string) string { return fs.config.URL + apiPrefix } +// GetSignedFileURL returns a signed URL with expiration for accessing a file +// bucketName: the name of the bucket +// objectKey: the key/path of the object +// ttl: time-to-live duration for the signed URL +// signingSecret: the secret key used to sign the URL (must match server-side secret) +func (fs *fileStorage) GetSignedFileURL(bucketName string, objectKey string, ttl time.Duration, signingSecret string) (string, error) { + if signingSecret == "" { + return "", errors.New("signing secret is required") + } + + // Create the path for the object + apiPrefix := fmt.Sprintf("/buckets/%s/%s", bucketName, objectKey) + + // Create a URL signer with the provided secret + signer := urlsigner.NewURLSigner(signingSecret) + + // Sign the URL path + signedPath, err := signer.SignURL(apiPrefix, ttl) + if err != nil { + slog.Error("Error signing URL", "error", err) + return "", &ErrClient{ + Message: "failed to sign URL", + Err: err, + Context: map[string]any{"bucket_name": bucketName, "object_key": objectKey}, + } + } + + // Return the full URL with the signed path + return fs.config.URL + signedPath, nil +} + func (fs *fileStorage) GetFileMetadata(bucketName string, objectKey string) (*entities.Object, error) { apiPrefix := fmt.Sprintf("/buckets/%s/%s?metadataOnly=true", bucketName, objectKey) apiURL := fs.config.URL + apiPrefix diff --git a/pkg/filestorage/file_storage_test.go b/pkg/filestorage/file_storage_test.go index 4b04b6b..8c2d9c5 100644 --- a/pkg/filestorage/file_storage_test.go +++ b/pkg/filestorage/file_storage_test.go @@ -465,3 +465,63 @@ func TestDeleteMultipleFiles(t *testing.T) { t.Fatalf("Failed to delete multiple files: %v", err) } } + +func TestGetSignedFileURL(t *testing.T) { + config := filestorage.FileStorageConfig{ + URL: "https://example.com", + } + storage, err := filestorage.NewFileStorage(config) + if err != nil { + t.Fatalf("Failed to create file storage: %v", err) + } + + bucketName := "test-bucket" + objectKey := "test-file.pdf" + ttl := 1 * time.Hour + signingSecret := "test-secret-key" + + signedURL, err := storage.GetSignedFileURL(bucketName, objectKey, ttl, signingSecret) + if err != nil { + t.Fatalf("Failed to generate signed URL: %v", err) + } + + // Verify the signed URL contains the base URL + if !strings.Contains(signedURL, config.URL) { + t.Errorf("Signed URL should contain base URL %s, got %s", config.URL, signedURL) + } + + // Verify the signed URL contains expires and signature parameters + if !strings.Contains(signedURL, "expires=") { + t.Error("Signed URL should contain expires parameter") + } + + if !strings.Contains(signedURL, "signature=") { + t.Error("Signed URL should contain signature parameter") + } + + // Verify the path is correct + expectedPath := fmt.Sprintf("/buckets/%s/%s", bucketName, objectKey) + if !strings.Contains(signedURL, expectedPath) { + t.Errorf("Signed URL should contain path %s", expectedPath) + } +} + +func TestGetSignedFileURL_EmptySecret(t *testing.T) { + config := filestorage.FileStorageConfig{ + URL: "https://example.com", + } + storage, err := filestorage.NewFileStorage(config) + if err != nil { + t.Fatalf("Failed to create file storage: %v", err) + } + + bucketName := "test-bucket" + objectKey := "test-file.pdf" + ttl := 1 * time.Hour + + _, err = storage.GetSignedFileURL(bucketName, objectKey, ttl, "") + if err == nil { + t.Error("Expected error for empty signing secret") + } +} + diff --git a/pkg/urlsigner/urlsigner.go b/pkg/urlsigner/urlsigner.go new file mode 100644 index 0000000..11a435f --- /dev/null +++ b/pkg/urlsigner/urlsigner.go @@ -0,0 +1,99 @@ +package urlsigner + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "net/url" + "strconv" + "time" +) + +// URLSigner handles signing and validating URLs with HMAC signatures +type URLSigner struct { + secret []byte +} + +// NewURLSigner creates a new URLSigner with the given secret +func NewURLSigner(secret string) *URLSigner { + return &URLSigner{ + secret: []byte(secret), + } +} + +// SignURL generates a signed URL with expiration +// path: the request path (e.g., "/buckets/mybucket/myfile.pdf") +// ttl: time-to-live duration for the URL +func (s *URLSigner) SignURL(path string, ttl time.Duration) (string, error) { + if path == "" { + return "", errors.New("path cannot be empty") + } + + // Calculate expiration timestamp + expiresAt := time.Now().Add(ttl).Unix() + + // Create the string to sign: "path:expiration" + stringToSign := fmt.Sprintf("%s:%d", path, expiresAt) + + // Generate HMAC signature + signature := s.generateSignature(stringToSign) + + // Build query parameters + values := url.Values{} + values.Set("expires", strconv.FormatInt(expiresAt, 10)) + values.Set("signature", signature) + + // Return path with query parameters + return fmt.Sprintf("%s?%s", path, values.Encode()), nil +} + +// ValidateSignedURL validates a signed URL +// path: the request path without query parameters +// expiresStr: the expires query parameter value +// signature: the signature query parameter value +func (s *URLSigner) ValidateSignedURL(path string, expiresStr string, signature string) error { + if path == "" { + return errors.New("path cannot be empty") + } + + if expiresStr == "" { + return errors.New("missing expires parameter") + } + + if signature == "" { + return errors.New("missing signature parameter") + } + + // Parse expiration timestamp + expiresAt, err := strconv.ParseInt(expiresStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid expires parameter: %w", err) + } + + // Check if URL has expired + if time.Now().Unix() > expiresAt { + return errors.New("URL has expired") + } + + // Recreate the string that was signed + stringToSign := fmt.Sprintf("%s:%d", path, expiresAt) + + // Generate expected signature + expectedSignature := s.generateSignature(stringToSign) + + // Compare signatures using constant-time comparison to prevent timing attacks + if !hmac.Equal([]byte(signature), []byte(expectedSignature)) { + return errors.New("invalid signature") + } + + return nil +} + +// generateSignature creates an HMAC-SHA256 signature and returns it as base64 URL-encoded string +func (s *URLSigner) generateSignature(data string) string { + h := hmac.New(sha256.New, s.secret) + h.Write([]byte(data)) + return base64.URLEncoding.EncodeToString(h.Sum(nil)) +} diff --git a/pkg/urlsigner/urlsigner_test.go b/pkg/urlsigner/urlsigner_test.go new file mode 100644 index 0000000..324d56d --- /dev/null +++ b/pkg/urlsigner/urlsigner_test.go @@ -0,0 +1,233 @@ +package urlsigner + +import ( + "net/url" + "strings" + "testing" + "time" +) + +func TestSignURL(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + ttl := 1 * time.Hour + + signedURL, err := signer.SignURL(path, ttl) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + if signedURL == "" { + t.Fatal("Signed URL should not be empty") + } + + // Verify the signed URL contains expires and signature parameters + if !strings.Contains(signedURL, "expires=") { + t.Error("Signed URL should contain expires parameter") + } + + if !strings.Contains(signedURL, "signature=") { + t.Error("Signed URL should contain signature parameter") + } + + if !strings.HasPrefix(signedURL, path) { + t.Errorf("Signed URL should start with original path %s, got %s", path, signedURL) + } +} + +func TestSignURLEmptyPath(t *testing.T) { + signer := NewURLSigner("test-secret") + _, err := signer.SignURL("", 1*time.Hour) + if err == nil { + t.Error("Expected error for empty path") + } +} + +func TestValidateSignedURL_Valid(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + ttl := 1 * time.Hour + + signedURL, err := signer.SignURL(path, ttl) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Parse the signed URL to extract query parameters + parts := strings.Split(signedURL, "?") + if len(parts) != 2 { + t.Fatalf("Invalid signed URL format: %s", signedURL) + } + + // Parse query parameters properly + params, err := url.ParseQuery(parts[1]) + if err != nil { + t.Fatalf("Failed to parse query parameters: %v", err) + } + + expiresStr := params.Get("expires") + signature := params.Get("signature") + + // Validate the signed URL + err = signer.ValidateSignedURL(path, expiresStr, signature) + if err != nil { + t.Errorf("Failed to validate valid signed URL: %v", err) + } +} + +func TestValidateSignedURL_Expired(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + + // Create a URL that expired 1 hour ago + ttl := -1 * time.Hour + + signedURL, err := signer.SignURL(path, ttl) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Parse the signed URL to extract query parameters + parts := strings.Split(signedURL, "?") + if len(parts) != 2 { + t.Fatalf("Invalid signed URL format: %s", signedURL) + } + + // Parse query parameters properly + params, err := url.ParseQuery(parts[1]) + if err != nil { + t.Fatalf("Failed to parse query parameters: %v", err) + } + + expiresStr := params.Get("expires") + signature := params.Get("signature") + + // Validate the signed URL - should fail due to expiration + err = signer.ValidateSignedURL(path, expiresStr, signature) + if err == nil { + t.Error("Expected error for expired URL") + } + + if !strings.Contains(err.Error(), "expired") { + t.Errorf("Expected 'expired' error, got: %v", err) + } +} + +func TestValidateSignedURL_InvalidSignature(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + ttl := 1 * time.Hour + + signedURL, err := signer.SignURL(path, ttl) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Parse the signed URL to extract query parameters + parts := strings.Split(signedURL, "?") + if len(parts) != 2 { + t.Fatalf("Invalid signed URL format: %s", signedURL) + } + + // Parse query parameters properly + params, err := url.ParseQuery(parts[1]) + if err != nil { + t.Fatalf("Failed to parse query parameters: %v", err) + } + + expiresStr := params.Get("expires") + invalidSignature := "invalid-signature" + + // Validate with invalid signature + err = signer.ValidateSignedURL(path, expiresStr, invalidSignature) + if err == nil { + t.Error("Expected error for invalid signature") + } + + if !strings.Contains(err.Error(), "signature") { + t.Errorf("Expected 'signature' error, got: %v", err) + } +} + +func TestValidateSignedURL_MissingExpires(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + + err := signer.ValidateSignedURL(path, "", "some-signature") + if err == nil { + t.Error("Expected error for missing expires parameter") + } +} + +func TestValidateSignedURL_MissingSignature(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + + err := signer.ValidateSignedURL(path, "1234567890", "") + if err == nil { + t.Error("Expected error for missing signature parameter") + } +} + +func TestValidateSignedURL_TamperedPath(t *testing.T) { + signer := NewURLSigner("test-secret") + path := "/buckets/test-bucket/test-file.pdf" + ttl := 1 * time.Hour + + signedURL, err := signer.SignURL(path, ttl) + if err != nil { + t.Fatalf("Failed to sign URL: %v", err) + } + + // Parse the signed URL to extract query parameters + parts := strings.Split(signedURL, "?") + if len(parts) != 2 { + t.Fatalf("Invalid signed URL format: %s", signedURL) + } + + // Parse query parameters properly + params, err := url.ParseQuery(parts[1]) + if err != nil { + t.Fatalf("Failed to parse query parameters: %v", err) + } + + expiresStr := params.Get("expires") + signature := params.Get("signature") + + // Validate with a different path (tampered) + tamperedPath := "/buckets/test-bucket/different-file.pdf" + err = signer.ValidateSignedURL(tamperedPath, expiresStr, signature) + if err == nil { + t.Error("Expected error for tampered path") + } + + if !strings.Contains(err.Error(), "signature") { + t.Errorf("Expected 'signature' error, got: %v", err) + } +} + +func TestDifferentSecretsProduceDifferentSignatures(t *testing.T) { + path := "/buckets/test-bucket/test-file.pdf" + ttl := 1 * time.Hour + + signer1 := NewURLSigner("secret1") + signer2 := NewURLSigner("secret2") + + signedURL1, _ := signer1.SignURL(path, ttl) + signedURL2, _ := signer2.SignURL(path, ttl) + + // Extract signatures + parts1 := strings.Split(signedURL1, "signature=") + parts2 := strings.Split(signedURL2, "signature=") + + if len(parts1) < 2 || len(parts2) < 2 { + t.Fatal("Failed to extract signatures") + } + + sig1 := parts1[1] + sig2 := parts2[1] + + if sig1 == sig2 { + t.Error("Different secrets should produce different signatures") + } +}