From f50e7b3f8eb0fba47a17efc388d1e589408e3705 Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Sat, 3 Jan 2026 07:11:56 +0200 Subject: [PATCH 1/4] adds npm proxy --- README.md | 34 +++- config.yaml | 2 + main.go | 5 + pkg/handlers/npm.go | 335 +++++++++++++++++++++++++++++++ pkg/misc/download_conditional.go | 120 +++++++++++ pkg/types/config.go | 1 + 6 files changed, 494 insertions(+), 3 deletions(-) create mode 100644 pkg/handlers/npm.go create mode 100644 pkg/misc/download_conditional.go diff --git a/README.md b/README.md index 4fdf4ae..d619cf1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ server: pypi: pypi.org: https://pypi.org/simple rubygems: - rubygems.org: https://rubygems.org + rubygems: https://rubygems.org galaxy: ansible: url: https://galaxy.ansible.com @@ -20,6 +20,8 @@ server: get_helm: https://get.helm.sh goproxy: golang: https://proxy.golang.org + npm: + npmjs: https://registry.npmjs.org ``` ## Usage @@ -74,8 +76,6 @@ source "https://rubygems.org" do end ``` - - ### Static files Access cached static files: @@ -105,3 +105,31 @@ The proxy supports all standard GOPROXY protocol endpoints: - `/{module}/@v/{version}.mod` - go.mod file for the version - `/{module}/@v/{version}.zip` - source code archive - `/{module}/@latest` - latest version info + +### NPM + +To use HUB as an npm registry proxy, set the `registry` to your HUB instance: + +```bash +npm config set registry http://localhost:6587/npm/npmjs +``` + +For a scoped registry: + +```bash +npm config set @my-scope:registry http://localhost:6587/npm/npmjs +``` + +Or it can be used in the file `.npmrc` + +```ini +registry=http://localhost:6587/npm/npmjs +``` + +The proxy supports the following npm registry endpoints: + +- `/{package}` - package metadata (packument) +- `/@scope%2F{name}` - scoped package metadata (packument) +- `/{package}/-/{tarball}.tgz` - package tarball +- `/@scope/{name}/-/{tarball}.tgz` - scoped package tarball +- `/-/v1/search` - search (cached for 10 minutes) diff --git a/config.yaml b/config.yaml index 6428b93..4cde012 100644 --- a/config.yaml +++ b/config.yaml @@ -15,3 +15,5 @@ server: get_helm: https://get.helm.sh goproxy: golang: https://proxy.golang.org + npm: + npmjs: https://registry.npmjs.org diff --git a/main.go b/main.go index 78db222..23148ec 100644 --- a/main.go +++ b/main.go @@ -186,6 +186,11 @@ func startServer(c *cli.Context) error { }).Name = fmt.Sprintf("goproxy::%s", k) } + for k := range cfg.Server.NPM { + n := e.Group(fmt.Sprintf("/npm/%s", k)) + n.GET("/*", handlers.NpmProxy(k)).Name = fmt.Sprintf("npm::%s", k) + } + for k, v := range cfg.Server.Galaxy { g := e.Group(fmt.Sprintf("/galaxy/%s", k)) if v.URL != "" && v.Dir != "" { diff --git a/pkg/handlers/npm.go b/pkg/handlers/npm.go new file mode 100644 index 0000000..d41d436 --- /dev/null +++ b/pkg/handlers/npm.go @@ -0,0 +1,335 @@ +package handlers + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/psvmcc/hub/pkg/misc" + "github.com/psvmcc/hub/pkg/types" + + "github.com/labstack/echo/v4" + "go.uber.org/zap" +) + +type npmCacheMeta struct { + ETag string `json:"etag"` + LastModified string `json:"last_modified"` +} + +const npmSearchTTL = 10 * time.Minute + +func NpmProxy(key string) echo.HandlerFunc { + return func(c echo.Context) error { + cfg := c.Get("cfg").(types.ConfigFile) + logger := c.Get("logger").(*zap.SugaredLogger) + loggerNS := "npm" + + rawPath := strings.TrimPrefix(c.Param("*"), "/") + rawPath = strings.TrimSuffix(rawPath, "/") + if rawPath == "" { + return c.String(http.StatusNotFound, "") + } + + cleaned := path.Clean("/" + rawPath) + cleaned = strings.TrimPrefix(cleaned, "/") + if cleaned == "" { + return c.String(http.StatusNotFound, "") + } + + if isNpmSearchPath(cleaned) { + return handleNpmSearch(c, cfg, logger, loggerNS, key) + } + if isNpmTarballPath(cleaned) { + return handleNpmTarball(c, cfg, logger, loggerNS, key, cleaned) + } + return handleNpmMetadata(c, cfg, logger, loggerNS, key, cleaned) + } +} + +func handleNpmMetadata(c echo.Context, cfg types.ConfigFile, logger *zap.SugaredLogger, loggerNS, key, rawPath string) error { + decodedPath, err := url.PathUnescape(rawPath) + if err != nil { + decodedPath = rawPath + } + decodedPath = path.Clean("/" + decodedPath) + packageName := strings.TrimSuffix(strings.TrimPrefix(decodedPath, "/"), "/") + if packageName == "" { + return c.String(http.StatusNotFound, "") + } + + acceptKey, upstreamAccept := npmAcceptHeader(c.Request().Header.Get("Accept")) + query := c.QueryString() + queryHash := "" + if query != "" { + sum := sha256.Sum256([]byte(query)) + queryHash = hex.EncodeToString(sum[:]) + } + + filenameBase := fmt.Sprintf("packument.%s", acceptKey) + if queryHash != "" { + filenameBase = fmt.Sprintf("%s.%s", filenameBase, queryHash) + } + packagePath := filepath.FromSlash(packageName) + cacheDir := filepath.Join(cfg.Dir, "npm", key, "metadata", packagePath) + dataFile := filepath.Join(cacheDir, filenameBase+".json") + metaFile := filepath.Join(cacheDir, filenameBase+".meta.json") + + upstreamBase := strings.TrimSuffix(cfg.Server.NPM[key], "/") + upstreamName := npmEncodePackageName(packageName) + upstreamURL := fmt.Sprintf("%s/%s", upstreamBase, upstreamName) + if query != "" { + upstreamURL = upstreamURL + "?" + query + } + + headers := types.RequestHeaders{ + "User-Agent": "npm", + "Accept": upstreamAccept, + } + + cacheExists := fileExists(dataFile) + meta := npmCacheMeta{} + if cacheExists { + meta, _ = readNpmCacheMeta(metaFile) + } + + status, newETag, newLastModified, notModified, err := misc.DownloadFileConditional(upstreamURL, dataFile, headers, meta.ETag, meta.LastModified) + if err != nil { + logger.Named(loggerNS).Errorf("[Downloading] %s", err) + if !cacheExists { + return c.String(status, "Please check logs...") + } + c.Response().Header().Add("X-Cache-Status", "STALE") + } else { + if notModified { + c.Response().Header().Add("X-Cache-Status", "HIT") + } else { + c.Response().Header().Add("X-Cache-Status", "MISS") + } + if newETag != "" || newLastModified != "" { + if newETag != "" { + meta.ETag = newETag + } + if newLastModified != "" { + meta.LastModified = newLastModified + } + if writeErr := writeNpmCacheMeta(metaFile, meta); writeErr != nil { + logger.Named(loggerNS).Errorf("Cache meta write error: %s", writeErr) + } + } + } + + payload, err := os.ReadFile(filepath.Clean(dataFile)) + if err != nil { + logger.Named(loggerNS).Errorf("Cache read error: %s", err) + return c.String(http.StatusBadRequest, "Metadata error") + } + + var packument map[string]any + if err := json.Unmarshal(payload, &packument); err != nil { + logger.Named(loggerNS).Errorf("Metadata unmarshal error: %s", err) + return c.String(http.StatusBadRequest, "Metadata error") + } + + baseURL := fmt.Sprintf("%s://%s", c.Scheme(), c.Request().Host) + rewrote := rewriteNpmTarballs(packument, baseURL, key) + if !rewrote { + logger.Named(loggerNS).Debugf("No tarball URLs rewritten for %s", packageName) + } + + updated, err := json.Marshal(packument) + if err != nil { + logger.Named(loggerNS).Errorf("Metadata marshal error: %s", err) + return c.String(http.StatusInternalServerError, "Metadata error") + } + + return c.Blob(http.StatusOK, upstreamAccept, updated) +} + +func handleNpmTarball(c echo.Context, cfg types.ConfigFile, logger *zap.SugaredLogger, loggerNS, key, rawPath string) error { + upstreamBase := strings.TrimSuffix(cfg.Server.NPM[key], "/") + upstreamURL := fmt.Sprintf("%s/%s", upstreamBase, rawPath) + dest := filepath.Join(cfg.Dir, "npm", key, "tarballs", filepath.FromSlash(rawPath)) + + headers := types.RequestHeaders{ + "User-Agent": "npm", + } + + if _, err := os.Stat(dest); err == nil { + c.Response().Header().Add("X-Cache-Status", "HIT") + return c.File(dest) + } + + status, err := misc.DownloadFile(upstreamURL, dest, headers) + if err != nil { + logger.Named(loggerNS).Errorf("[Downloading] %s", err) + if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { + logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + return c.String(status, "Please check logs...") + } + logger.Named(loggerNS).Debugf("Remote %s served from local file %s", upstreamURL, dest) + return c.File(dest) + } + + c.Response().Header().Add("X-Cache-Status", "MISS") + logger.Named(loggerNS).Debugf("Remote %s saved as %s", upstreamURL, dest) + return c.File(dest) +} + +func handleNpmSearch(c echo.Context, cfg types.ConfigFile, logger *zap.SugaredLogger, loggerNS, key string) error { + query := c.QueryString() + hash := "empty" + if query != "" { + sum := sha256.Sum256([]byte(query)) + hash = hex.EncodeToString(sum[:]) + } + + dest := filepath.Join(cfg.Dir, "npm", key, "search", hash+".json") + info, err := os.Stat(dest) + if err == nil && time.Since(info.ModTime()) < npmSearchTTL { + c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Set("Content-Type", "application/json") + return c.File(dest) + } + + upstreamBase := strings.TrimSuffix(cfg.Server.NPM[key], "/") + upstreamURL := fmt.Sprintf("%s/-/v1/search", upstreamBase) + if query != "" { + upstreamURL = upstreamURL + "?" + query + } + + headers := types.RequestHeaders{ + "User-Agent": "npm", + "Accept": "application/json", + } + + status, err := misc.DownloadFile(upstreamURL, dest, headers) + if err != nil { + logger.Named(loggerNS).Errorf("[Downloading] %s", err) + if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { + logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + return c.String(status, "Please check logs...") + } + c.Response().Header().Add("X-Cache-Status", "STALE") + c.Response().Header().Set("Content-Type", "application/json") + return c.File(dest) + } + + if err := os.Chtimes(dest, time.Now(), time.Now()); err != nil { + logger.Named(loggerNS).Errorf("Cache timestamp update error: %s", err) + } + c.Response().Header().Add("X-Cache-Status", "MISS") + c.Response().Header().Set("Content-Type", "application/json") + return c.File(dest) +} + +func isNpmTarballPath(p string) bool { + return strings.Contains(p, "/-/") && (strings.HasSuffix(p, ".tgz") || strings.HasSuffix(p, ".tar.gz")) +} + +func isNpmSearchPath(p string) bool { + return strings.TrimSuffix(p, "/") == "-/v1/search" +} + +func npmAcceptHeader(accept string) (cacheKey, upstreamAccept string) { + if strings.Contains(accept, "application/vnd.npm.install-v1+json") || accept == "" || strings.Contains(accept, "*/*") { + return "corgi", "application/vnd.npm.install-v1+json" + } + if strings.Contains(accept, "application/json") { + return "full", "application/json" + } + sum := sha256.Sum256([]byte(accept)) + return "accept-" + hex.EncodeToString(sum[:8]), accept +} + +func npmEncodePackageName(name string) string { + if strings.HasPrefix(name, "@") && strings.Contains(name, "/") { + return strings.ReplaceAll(name, "/", "%2F") + } + return name +} + +func rewriteNpmTarballs(packument map[string]any, baseURL, key string) bool { + versionsRaw, ok := packument["versions"] + if !ok { + return false + } + versions, ok := versionsRaw.(map[string]any) + if !ok { + return false + } + + updated := false + for _, v := range versions { + versionInfo, ok := v.(map[string]any) + if !ok { + continue + } + distRaw, ok := versionInfo["dist"] + if !ok { + continue + } + dist, ok := distRaw.(map[string]any) + if !ok { + continue + } + tarballRaw, ok := dist["tarball"] + if !ok { + continue + } + tarballURL, ok := tarballRaw.(string) + if !ok || tarballURL == "" { + continue + } + parsed, err := url.Parse(tarballURL) + if err != nil || parsed.Path == "" { + continue + } + rewritePath := parsed.Path + if parsed.RawQuery != "" { + rewritePath = rewritePath + "?" + parsed.RawQuery + } + dist["tarball"] = fmt.Sprintf("%s/npm/%s%s", baseURL, key, rewritePath) + updated = true + } + + return updated +} + +func readNpmCacheMeta(metaPath string) (npmCacheMeta, error) { + meta := npmCacheMeta{} + data, err := os.ReadFile(filepath.Clean(metaPath)) + if err != nil { + return meta, err + } + if err := json.Unmarshal(data, &meta); err != nil { + return meta, err + } + return meta, nil +} + +func writeNpmCacheMeta(metaPath string, meta npmCacheMeta) error { + file := filepath.Clean(metaPath) + data, err := json.Marshal(meta) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(file), 0o750); err != nil { + return err + } + return os.WriteFile(file, data, 0o600) +} + +func fileExists(filePath string) bool { + _, err := os.Stat(filePath) + return err == nil +} diff --git a/pkg/misc/download_conditional.go b/pkg/misc/download_conditional.go new file mode 100644 index 0000000..0ad814b --- /dev/null +++ b/pkg/misc/download_conditional.go @@ -0,0 +1,120 @@ +package misc + +import ( + "crypto/rand" + "fmt" + "io" + "math/big" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/psvmcc/hub/pkg/types" +) + +func DownloadFileConditional(url, destination string, headers types.RequestHeaders, etag, lastModified string) (code int, newETag, newLastModified string, notModified bool, err error) { + client := &http.Client{} + + var req *http.Request + var response *http.Response + + req, err = http.NewRequest("GET", url, http.NoBody) + if err != nil { + code = http.StatusBadRequest + return code, "", "", false, err + } + req.Header.Set("User-Agent", "hub") + for k, v := range headers { + req.Header.Set(k, v) + } + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + if lastModified != "" { + req.Header.Set("If-Modified-Since", lastModified) + } + + response, err = client.Do(req) + if err != nil { + code = http.StatusBadGateway + return code, "", "", false, err + } + defer response.Body.Close() + + newETag = response.Header.Get("ETag") + newLastModified = response.Header.Get("Last-Modified") + + if response.StatusCode == http.StatusNotModified { + code = response.StatusCode + return code, newETag, newLastModified, true, nil + } + + if response.StatusCode != http.StatusOK { + err = fmt.Errorf("upstream returned %s", response.Status) + code = response.StatusCode + return code, newETag, newLastModified, false, err + } + + if err = os.MkdirAll(filepath.Dir(destination), 0o750); err != nil { + err = fmt.Errorf("failed to create destination directory: %v", err) + code = http.StatusConflict + return code, newETag, newLastModified, false, err + } + + n, err := rand.Int(rand.Reader, big.NewInt(1000)) + if err != nil { + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + + tempFileName := fmt.Sprintf(".tmp.%s.%d.%d", filepath.Base(filepath.Clean(destination)), time.Now().UnixNano(), n.Int64()) + tempFilePath := filepath.Join(filepath.Dir(filepath.Clean(destination)), tempFileName) + tempFile, err := os.Create(filepath.Clean(tempFilePath)) + if err != nil { + err = fmt.Errorf("failed to create temporary file: %v", err) + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + defer os.Remove(tempFile.Name()) + + _, err = io.Copy(tempFile, response.Body) + if err != nil { + err = fmt.Errorf("failed to copy response body to file: %v", err) + code = http.StatusBadRequest + return code, newETag, newLastModified, false, err + } + + if newLastModified != "" { + var lastModifiedTime time.Time + if lastModifiedTime, err = time.Parse(http.TimeFormat, newLastModified); err == nil { + var fileInfo os.FileInfo + fileInfo, err = tempFile.Stat() + if err != nil { + err = fmt.Errorf("failed to get file info: %v", err) + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + if err = os.Chtimes(tempFilePath, fileInfo.ModTime(), lastModifiedTime); err != nil { + err = fmt.Errorf("failed to set last-modified time: %v", err) + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + } + } + err = tempFile.Close() + if err != nil { + err = fmt.Errorf("tempFile close error: %v", err) + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + + if err := os.Rename(tempFile.Name(), destination); err != nil { + err = fmt.Errorf("failed to rename temporary file to destination: %v", err) + code = http.StatusInternalServerError + return code, newETag, newLastModified, false, err + } + + code = http.StatusOK + return code, newETag, newLastModified, false, nil +} diff --git a/pkg/types/config.go b/pkg/types/config.go index 668ed74..d12e525 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -19,6 +19,7 @@ type ConfigFile struct { RUBYGEMS map[string]string `yaml:"rubygems"` Static map[string]string `yaml:"static"` GOPROXY map[string]string `yaml:"goproxy"` + NPM map[string]string `yaml:"npm"` } `yaml:"server"` } From 10dc658ac17e1d1f6c17df7475a5587185548e61 Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Sat, 3 Jan 2026 07:13:22 +0200 Subject: [PATCH 2/4] [chores] add modernize linter and replace interface with any --- .golangci.yml | 1 + pkg/templates/pypi.go | 4 ++-- pkg/types/galaxy_collection_version_info.go | 2 +- pkg/types/pypi.go | 16 ++++++++-------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 01b6694..831f3fa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,7 @@ linters: - staticcheck - unconvert - unused + - modernize settings: goconst: min-len: 2 diff --git a/pkg/templates/pypi.go b/pkg/templates/pypi.go index 6287ffe..13bd062 100644 --- a/pkg/templates/pypi.go +++ b/pkg/templates/pypi.go @@ -24,10 +24,10 @@ type TemplateRegistry struct { Templates *template.Template } -func (t *TemplateRegistry) Render(w io.Writer, name string, data interface{}, _ echo.Context) error { +func (t *TemplateRegistry) Render(w io.Writer, name string, data any, _ echo.Context) error { return t.Templates.ExecuteTemplate(w, name, data) } -func KindIs(v interface{}, kind string) bool { +func KindIs(v any, kind string) bool { return reflect.TypeOf(v).Kind().String() == kind } diff --git a/pkg/types/galaxy_collection_version_info.go b/pkg/types/galaxy_collection_version_info.go index e23bd1f..e509f0b 100644 --- a/pkg/types/galaxy_collection_version_info.go +++ b/pkg/types/galaxy_collection_version_info.go @@ -68,7 +68,7 @@ type GalaxyCollectionVersionInfo struct { Name string `json:"name"` MetadataSha256 string `json:"metadata_sha256"` } `json:"namespace"` - Signatures interface{} `json:"signatures"` + Signatures any `json:"signatures"` Metadata struct { Authors []string `json:"authors"` Contents []any `json:"contents"` diff --git a/pkg/types/pypi.go b/pkg/types/pypi.go index e2c1225..aeb36e4 100644 --- a/pkg/types/pypi.go +++ b/pkg/types/pypi.go @@ -10,17 +10,17 @@ import ( type PypiMetadata struct { Files []struct { - CoreMetadata interface{} `json:"core-metadata"` - DataDistInfoMetadata interface{} `json:"data-dist-info-metadata"` - Filename string `json:"filename"` + CoreMetadata any `json:"core-metadata"` + DataDistInfoMetadata any `json:"data-dist-info-metadata"` + Filename string `json:"filename"` Hashes struct { Sha256 string `json:"sha256"` } `json:"hashes"` - RequiresPython string `json:"requires-python"` - Size int `json:"size"` - UploadTime time.Time `json:"upload-time"` - URL string `json:"url"` - Yanked interface{} `json:"yanked"` + RequiresPython string `json:"requires-python"` + Size int `json:"size"` + UploadTime time.Time `json:"upload-time"` + URL string `json:"url"` + Yanked any `json:"yanked"` } `json:"files"` Meta struct { LastSerial int `json:"_last-serial"` From bd2161e08317c5974d4832ba726de3d1e2e030cb Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Sat, 3 Jan 2026 07:59:27 +0200 Subject: [PATCH 3/4] [fix] update cache status headers in handlers for better response tracking --- pkg/handlers/galaxy_local.go | 4 ++++ pkg/handlers/galaxy_proxy.go | 15 ++++++++++++--- pkg/handlers/goproxy.go | 11 ++++++++--- pkg/handlers/npm.go | 4 ++++ pkg/handlers/pypi.go | 8 +++++++- pkg/handlers/rubygems.go | 12 +++++++++--- pkg/handlers/static.go | 12 ++++++++++-- 7 files changed, 54 insertions(+), 12 deletions(-) diff --git a/pkg/handlers/galaxy_local.go b/pkg/handlers/galaxy_local.go index 0fa79f1..9f8c21a 100644 --- a/pkg/handlers/galaxy_local.go +++ b/pkg/handlers/galaxy_local.go @@ -45,6 +45,7 @@ func GalaxyLocalCollection(key string) echo.HandlerFunc { collection.HighestVersion.Href = fmt.Sprintf("/api/v3/collections/%s/%s/versions/%s/", namespace, name, collectionLocal.Latest.Version) collection.UpdatedAt = collectionLocal.Latest.Time.UTC() + c.Response().Header().Add("X-Cache-Status", "LOCAL") return c.JSON(http.StatusOK, collection) } } @@ -78,6 +79,7 @@ func GalaxyLocalCollectionVersions(key string) echo.HandlerFunc { collectionVersions.Data = append(collectionVersions.Data, verInfo) } + c.Response().Header().Add("X-Cache-Status", "LOCAL") return c.JSON(http.StatusOK, collectionVersions) } } @@ -177,6 +179,7 @@ func GalaxyLocalCollectionVersionInfo(key string) echo.HandlerFunc { collectionVersionInfo.Metadata.Dependencies = manifest.CollectionInfo.Dependencies collectionVersionInfo.Files = files + c.Response().Header().Add("X-Cache-Status", "LOCAL") return c.JSON(http.StatusOK, collectionVersionInfo) }(c) } @@ -201,6 +204,7 @@ func GalaxyLocalCollectionGet(key string) echo.HandlerFunc { logger.Named(loggerNS).Debugf("Collection not found: %s/%s", namespace, name) return c.String(http.StatusNotFound, "") } + c.Response().Header().Add("X-Cache-Status", "LOCAL") c.Response().Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s-%s-%s.tar.gz\"", namespace, name, version)) return c.File(dest) } diff --git a/pkg/handlers/galaxy_proxy.go b/pkg/handlers/galaxy_proxy.go index 0402d34..55a8c24 100644 --- a/pkg/handlers/galaxy_proxy.go +++ b/pkg/handlers/galaxy_proxy.go @@ -32,11 +32,13 @@ func GalaxyProxyCollection(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { + c.Response().Header().Add("X-Cache-Status", "MISS") logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) } var collection types.GalaxyCollection @@ -69,11 +71,13 @@ func GalaxyProxyCollectionVersions(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { + c.Response().Header().Add("X-Cache-Status", "MISS") logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) } var collectionVersions types.GalaxyCollectionVersions @@ -107,11 +111,13 @@ func GalaxyProxyCollectionVersionInfo(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(http.StatusNotFound, "") } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { + c.Response().Header().Add("X-Cache-Status", "MISS") logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) } var CollectionVersionInfo types.GalaxyCollectionVersionInfo @@ -148,6 +154,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc { _, err := misc.DownloadFile(url, versionFile, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(http.StatusBadRequest, "Downloading error") } logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, versionFile) @@ -167,6 +174,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc { status, err := misc.DownloadFile(url, dest, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } c.Response().Header().Add("X-Cache-Status", "MISS") @@ -183,6 +191,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc { status, err := misc.DownloadFile(url, dest, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } logger.Named(loggerNS).Debugf("Downloaded %s", url) diff --git a/pkg/handlers/goproxy.go b/pkg/handlers/goproxy.go index d921b94..ea44d71 100644 --- a/pkg/handlers/goproxy.go +++ b/pkg/handlers/goproxy.go @@ -28,9 +28,10 @@ func downloadAndCacheFile(c echo.Context, key, loggerNS, url, dest string) error logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "410 Gone\n") } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { c.Response().Header().Add("X-Cache-Status", "MISS") @@ -62,9 +63,10 @@ func GoProxyList(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "410 Gone\n") } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { c.Response().Header().Add("X-Cache-Status", "MISS") @@ -161,8 +163,10 @@ func GoProxyZip(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "410 Gone\n") } + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { c.Response().Header().Add("X-Cache-Status", "MISS") @@ -205,9 +209,10 @@ func GoProxyLatest(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "410 Gone\n") } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { c.Response().Header().Add("X-Cache-Status", "MISS") diff --git a/pkg/handlers/npm.go b/pkg/handlers/npm.go index d41d436..8fa1542 100644 --- a/pkg/handlers/npm.go +++ b/pkg/handlers/npm.go @@ -106,6 +106,7 @@ func handleNpmMetadata(c echo.Context, cfg types.ConfigFile, logger *zap.Sugared if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if !cacheExists { + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } c.Response().Header().Add("X-Cache-Status", "STALE") @@ -174,8 +175,10 @@ func handleNpmTarball(c echo.Context, cfg types.ConfigFile, logger *zap.SugaredL logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", upstreamURL, dest) return c.File(dest) } @@ -217,6 +220,7 @@ func handleNpmSearch(c echo.Context, cfg types.ConfigFile, logger *zap.SugaredLo logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } c.Response().Header().Add("X-Cache-Status", "STALE") diff --git a/pkg/handlers/pypi.go b/pkg/handlers/pypi.go index 4ecc7b2..c20c3e3 100644 --- a/pkg/handlers/pypi.go +++ b/pkg/handlers/pypi.go @@ -35,11 +35,13 @@ func PypiSimple(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err = os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } - c.Response().Header().Add("X-Cache-Status", "HIT") + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { + c.Response().Header().Add("X-Cache-Status", "MISS") logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) } @@ -109,12 +111,14 @@ func PypiPackages(key string) echo.HandlerFunc { _, err := misc.DownloadFile(url, indexDest, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(http.StatusBadRequest, "Downloading error") } logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, indexDest) err = pypiMetadata.ReadFromJSONFile(indexDest) if err != nil { logger.Named(loggerNS).Errorf("Unable to parse local json file %s, got error: %s", indexDest, err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(http.StatusBadRequest, "Metadata error") } } @@ -136,6 +140,7 @@ func PypiPackages(key string) echo.HandlerFunc { status, err := misc.DownloadFile(url, dest, headers) if err != nil { logger.Named(loggerNS).Errorf("Downloading %s error: %s", url, err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } logger.Named(loggerNS).Debugf("Local file %s not found", dest) @@ -152,6 +157,7 @@ func PypiPackages(key string) echo.HandlerFunc { status, err := misc.DownloadFile(url, dest, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, fmt.Sprintf("%v", err)) } logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) diff --git a/pkg/handlers/rubygems.go b/pkg/handlers/rubygems.go index 2fd35db..896acf0 100644 --- a/pkg/handlers/rubygems.go +++ b/pkg/handlers/rubygems.go @@ -52,8 +52,9 @@ func RubyGems(key string) echo.HandlerFunc { "User-Agent": "rubygems", } + cacheExists := true if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { - c.Response().Header().Add("X-Cache-Status", "MISS") + cacheExists = false } else { equal, err := misc.FilesEqual(url, dest) if err != nil { @@ -64,8 +65,6 @@ func RubyGems(key string) echo.HandlerFunc { c.Response().Header().Add("X-Cache-Status", "HIT") return c.File(dest) } - - c.Response().Header().Add("X-Cache-Status", "EXPIRE") } status, err := misc.DownloadFile(url, dest, headers) @@ -73,12 +72,19 @@ func RubyGems(key string) echo.HandlerFunc { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) return c.File(dest) } + if cacheExists { + c.Response().Header().Add("X-Cache-Status", "EXPIRED") + } else { + c.Response().Header().Add("X-Cache-Status", "MISS") + } logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) return c.File(dest) } diff --git a/pkg/handlers/static.go b/pkg/handlers/static.go index 30234d8..1287d1d 100644 --- a/pkg/handlers/static.go +++ b/pkg/handlers/static.go @@ -26,8 +26,9 @@ func Static(key string) echo.HandlerFunc { "User-Agent": "curl", } + cacheExists := true if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { - c.Response().Header().Add("X-Cache-Status", "MISS") + cacheExists = false } else { equal, err := misc.FilesEqual(url, dest) if err != nil { @@ -38,17 +39,24 @@ func Static(key string) echo.HandlerFunc { c.Response().Header().Add("X-Cache-Status", "HIT") return c.File(dest) } - c.Response().Header().Add("X-Cache-Status", "EXPIRE") } + status, err := misc.DownloadFile(url, dest, headers) if err != nil { logger.Named(loggerNS).Errorf("[Downloading] %s", err) if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { logger.Named(loggerNS).Errorf("[FS]: %s", err) + c.Response().Header().Add("X-Cache-Status", "ERROR") return c.String(status, "Please check logs...") } + c.Response().Header().Add("X-Cache-Status", "STALE") logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) } else { + if cacheExists { + c.Response().Header().Add("X-Cache-Status", "EXPIRED") + } else { + c.Response().Header().Add("X-Cache-Status", "MISS") + } logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) } return c.File(dest) From 4d37a767780dc8d9a0641dee3796813c2d8b0625 Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Sat, 3 Jan 2026 07:59:48 +0200 Subject: [PATCH 4/4] [chores] add cache status logging to server request handling --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 23148ec..b464e14 100644 --- a/main.go +++ b/main.go @@ -114,8 +114,12 @@ func startServer(c *cli.Context) error { c.Error(err) } stop := time.Now() + cacheStatus := res.Header().Get("X-Cache-Status") + if cacheStatus == "" { + cacheStatus = "UNKNOWN" + } message := fmt.Sprintf( - "[%s] %s %s requested from %s with status %d in %s [%s]", + "[%s] %s %s requested from %s with status %d in %s [%s] cache=%s", c.Request().Host, req.Method, req.RequestURI, @@ -123,6 +127,7 @@ func startServer(c *cli.Context) error { res.Status, stop.Sub(start).String(), c.Path(), + cacheStatus, ) logger := c.Get("logger").(*zap.SugaredLogger)