From 4b0bd9ac61b79813bb332358ff70f8ca420954d8 Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Wed, 24 Dec 2025 22:06:44 +0200 Subject: [PATCH 1/4] [chores] set user-agent header as canonical --- pkg/handlers/galaxy_proxy.go | 10 +++++----- pkg/handlers/goproxy.go | 8 ++++---- pkg/handlers/pypi.go | 14 +++++++------- pkg/handlers/static.go | 2 +- pkg/misc/download.go | 3 ++- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pkg/handlers/galaxy_proxy.go b/pkg/handlers/galaxy_proxy.go index 4db1f4e..0402d34 100644 --- a/pkg/handlers/galaxy_proxy.go +++ b/pkg/handlers/galaxy_proxy.go @@ -25,7 +25,7 @@ func GalaxyProxyCollection(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/galaxy/%s/index/%s/%s/index.json", cfg.Dir, key, namespace, name) headers := types.RequestHeaders{ - "UserAgent": "ansible-galaxy", + "User-Agent": "ansible-galaxy", } status, err := misc.DownloadFile(url, dest, headers) if err != nil { @@ -62,7 +62,7 @@ func GalaxyProxyCollectionVersions(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/galaxy/%s/index/%s/%s/versions/index/%s", cfg.Dir, key, namespace, name, c.QueryString()) headers := types.RequestHeaders{ - "UserAgent": "ansible-galaxy", + "User-Agent": "ansible-galaxy", } status, err := misc.DownloadFile(url, dest, headers) if err != nil { @@ -100,7 +100,7 @@ func GalaxyProxyCollectionVersionInfo(key string) echo.HandlerFunc { host := c.Request().Host headers := types.RequestHeaders{ - "UserAgent": "ansible-galaxy", + "User-Agent": "ansible-galaxy", } _, err := misc.DownloadFile(url, dest, headers) if err != nil { @@ -143,7 +143,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc { url := fmt.Sprintf("%s/api/v3/collections/%s/%s/versions/%s", cfg.Server.Galaxy[key].URL, namespace, name, version) headers := types.RequestHeaders{ - "UserAgent": "ansible-galaxy", + "User-Agent": "ansible-galaxy", } _, err := misc.DownloadFile(url, versionFile, headers) if err != nil { @@ -161,7 +161,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/galaxy/%s/binary/%s/%s/%s-%s-%s.tar.gz", cfg.Dir, key, namespace, name, namespace, name, version) headers := types.RequestHeaders{ - "UserAgent": "ansible-galaxy", + "User-Agent": "ansible-galaxy", } if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { status, err := misc.DownloadFile(url, dest, headers) diff --git a/pkg/handlers/goproxy.go b/pkg/handlers/goproxy.go index 843c1c2..d921b94 100644 --- a/pkg/handlers/goproxy.go +++ b/pkg/handlers/goproxy.go @@ -20,7 +20,7 @@ func downloadAndCacheFile(c echo.Context, key, loggerNS, url, dest string) error logger := c.Get("logger").(*zap.SugaredLogger) headers := types.RequestHeaders{ - "UserAgent": "go/goproxy", + "User-Agent": "go/goproxy", } status, err := misc.DownloadFile(url, dest, headers) @@ -54,7 +54,7 @@ func GoProxyList(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/goproxy/%s/%s/@v/list", cfg.Dir, key, module) headers := types.RequestHeaders{ - "UserAgent": "go/goproxy", + "User-Agent": "go/goproxy", } status, err := misc.DownloadFile(url, dest, headers) @@ -145,7 +145,7 @@ func GoProxyZip(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/goproxy/%s/%s/@v/%s.zip", cfg.Dir, key, modulePath, version) headers := types.RequestHeaders{ - "UserAgent": "go/goproxy", + "User-Agent": "go/goproxy", } // Check if file exists and has valid content @@ -188,7 +188,7 @@ func GoProxyLatest(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/goproxy/%s/%s/@latest", cfg.Dir, key, module) headers := types.RequestHeaders{ - "UserAgent": "go/goproxy", + "User-Agent": "go/goproxy", } // @latest should be fetched more frequently, so check if file is older than 1 hour diff --git a/pkg/handlers/pypi.go b/pkg/handlers/pypi.go index 3f5e4b1..4ecc7b2 100644 --- a/pkg/handlers/pypi.go +++ b/pkg/handlers/pypi.go @@ -26,8 +26,8 @@ func PypiSimple(key string) echo.HandlerFunc { scheme := c.Scheme() host := c.Request().Host headers := types.RequestHeaders{ - "UserAgent": "pypi", - "Accept": "application/vnd.pypi.simple.v1+json", + "User-Agent": "pypi", + "Accept": "application/vnd.pypi.simple.v1+json", } status, err := misc.DownloadFile(url, dest, headers) @@ -70,7 +70,7 @@ func PypiPackages(key string) echo.HandlerFunc { var url, sha string headers := types.RequestHeaders{ - "UserAgent": "pypi", + "User-Agent": "pypi", } indexFileInfo, err := os.Stat(indexDest) @@ -84,8 +84,8 @@ func PypiPackages(key string) echo.HandlerFunc { url = fmt.Sprintf("%s/%s/", cfg.Server.PYPI[key], name) headers = types.RequestHeaders{ - "UserAgent": "pypi", - "Accept": "application/vnd.pypi.simple.v1+json", + "User-Agent": "pypi", + "Accept": "application/vnd.pypi.simple.v1+json", } _, err := misc.DownloadFile(url, indexDest, headers) if err != nil { @@ -103,8 +103,8 @@ func PypiPackages(key string) echo.HandlerFunc { url = fmt.Sprintf("%s/%s/", cfg.Server.PYPI[key], name) headers = types.RequestHeaders{ - "UserAgent": "pypi", - "Accept": "application/vnd.pypi.simple.v1+json", + "User-Agent": "pypi", + "Accept": "application/vnd.pypi.simple.v1+json", } _, err := misc.DownloadFile(url, indexDest, headers) if err != nil { diff --git a/pkg/handlers/static.go b/pkg/handlers/static.go index b7140ab..30234d8 100644 --- a/pkg/handlers/static.go +++ b/pkg/handlers/static.go @@ -23,7 +23,7 @@ func Static(key string) echo.HandlerFunc { dest := fmt.Sprintf("%s/static/%s/%s", cfg.Dir, key, path) headers := types.RequestHeaders{ - "UserAgent": "curl", + "User-Agent": "curl", } if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { diff --git a/pkg/misc/download.go b/pkg/misc/download.go index 3e50556..fcec925 100644 --- a/pkg/misc/download.go +++ b/pkg/misc/download.go @@ -24,7 +24,7 @@ func DownloadFile(url, destination string, headers types.RequestHeaders) (code i code = http.StatusBadRequest return code, err } - req.Header.Set("UserAgent", "hub") + req.Header.Set("User-Agent", "hub") for k, v := range headers { req.Header.Set(k, v) @@ -38,6 +38,7 @@ func DownloadFile(url, destination string, headers types.RequestHeaders) (code i defer response.Body.Close() if response.StatusCode != http.StatusOK { + err = fmt.Errorf("upstream returned %s", response.Status) code = response.StatusCode return code, err } From b74fd96a311ab3c5044d1ce49ae1b8ffa5cdef2f Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Wed, 24 Dec 2025 22:08:49 +0200 Subject: [PATCH 2/4] add rubygems proxy, based on compact index --- README.md | 22 +++++++++++ config.yaml | 2 + main.go | 5 +++ pkg/handlers/rubygems.go | 85 ++++++++++++++++++++++++++++++++++++++++ pkg/types/config.go | 7 ++-- 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 pkg/handlers/rubygems.go diff --git a/README.md b/README.md index 386a302..cbc740e 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ dir: _data server: pypi: pypi.org: https://pypi.org/simple + rubygems: + rubygems.org: https://rubygems.org galaxy: ansible: url: https://galaxy.ansible.com @@ -38,6 +40,26 @@ Access cached Galaxy collections: http://localhost:6587/galaxy/ansible/api/v3/collections/{namespace}/{name}/ ``` +### RubyGems + +Use HUB as a RubyGems/Bundler source: + +```text +http://localhost:6587/rubygems/{repo_name} +``` + +Bundler/RubyGems clients fetch a set of plain HTTP resources. HUB proxies and caches any path under the configured upstream (wildcard route), including common endpoints: + +- Compact index: `/names`, `/versions`, `/info/` +- Legacy indexes: `/specs.4.8.gz`, `/latest_specs.4.8.gz`, `/prerelease_specs.4.8.gz`, `/quick/Marshal.4.8/*.gemspec.rz` +- Gem artifacts: `/gems/-.gem` + +Bundler mirror example: + +```bash +bundle config set --global mirror.https://rubygems.org http://localhost:6587/rubygems/rubygems +``` + ### Static files Access cached static files: diff --git a/config.yaml b/config.yaml index 405722c..6428b93 100644 --- a/config.yaml +++ b/config.yaml @@ -2,6 +2,8 @@ dir: _data server: pypi: pypi.org: https://pypi.org/simple + rubygems: + rubygems: https://rubygems.org galaxy: ansible: url: https://galaxy.ansible.com diff --git a/main.go b/main.go index 8541fd9..78db222 100644 --- a/main.go +++ b/main.go @@ -155,6 +155,11 @@ func startServer(c *cli.Context) error { p.GET("/packages/:name/:filename", handlers.PypiPackages(k)).Name = fmt.Sprintf("pypi::%s::packages", k) } + for k := range cfg.Server.RUBYGEMS { + r := e.Group(fmt.Sprintf("/rubygems/%s", k)) + r.GET("/*", handlers.RubyGems(k)).Name = fmt.Sprintf("rubygems::%s", k) + } + for k := range cfg.Server.Static { s := e.Group(fmt.Sprintf("/static/%s", k)) s.GET("/get/*", handlers.Static(k)).Name = fmt.Sprintf("static::%s", k) diff --git a/pkg/handlers/rubygems.go b/pkg/handlers/rubygems.go new file mode 100644 index 0000000..a795874 --- /dev/null +++ b/pkg/handlers/rubygems.go @@ -0,0 +1,85 @@ +package handlers + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/psvmcc/hub/pkg/misc" + "github.com/psvmcc/hub/pkg/types" + + "github.com/labstack/echo/v4" + "go.uber.org/zap" +) + +func RubyGems(key string) echo.HandlerFunc { + return func(c echo.Context) error { + cfg := c.Get("cfg").(types.ConfigFile) + logger := c.Get("logger").(*zap.SugaredLogger) + loggerNS := "rubygems" + + requestedPath := strings.TrimPrefix(c.Param("*"), "/") + upstreamPath := strings.TrimPrefix(path.Clean("/"+requestedPath), "/") + cacheKey := upstreamPath + if upstreamPath == "" || upstreamPath == "." { + upstreamPath = "" + cacheKey = "__root" + } + + query := c.QueryString() + cachePath := cacheKey + if query != "" { + sum := sha256.Sum256([]byte(query)) + cachePath = path.Join("_query", hex.EncodeToString(sum[:]), cacheKey) + } + + upstreamBase := strings.TrimSuffix(cfg.Server.RUBYGEMS[key], "/") + url := upstreamBase + "/" + if upstreamPath != "" { + url = url + upstreamPath + } + if query != "" { + url = url + "?" + query + } + + dest := fmt.Sprintf("%s/rubygems/%s/%s", cfg.Dir, key, cachePath) + + headers := types.RequestHeaders{ + "User-Agent": "rubygems", + } + + if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) { + c.Response().Header().Add("X-Cache-Status", "MISS") + } else { + equal, err := misc.FilesEqual(url, dest) + if err != nil { + logger.Named(loggerNS).Errorf("[FilesEqual]: %s", err) + } + + if equal { + c.Response().Header().Add("X-Cache-Status", "HIT") + return c.File(dest) + } + + c.Response().Header().Add("X-Cache-Status", "EXPIRE") + } + + status, err := misc.DownloadFile(url, dest, headers) + if err != nil { + logger.Named(loggerNS).Errorf("[Downloading] %s", err) + if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) { + logger.Named(loggerNS).Errorf("[FS]: %s", statErr) + return c.String(status, "Please check logs...") + } + logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest) + return c.File(dest) + } + + logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest) + return c.File(dest) + } +} diff --git a/pkg/types/config.go b/pkg/types/config.go index 93ba5b1..668ed74 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -15,9 +15,10 @@ type ConfigFile struct { URL string `yaml:"url"` Dir string `yaml:"dir"` } `yaml:"galaxy"` - PYPI map[string]string `yaml:"pypi"` - Static map[string]string `yaml:"static"` - GOPROXY map[string]string `yaml:"goproxy"` + PYPI map[string]string `yaml:"pypi"` + RUBYGEMS map[string]string `yaml:"rubygems"` + Static map[string]string `yaml:"static"` + GOPROXY map[string]string `yaml:"goproxy"` } `yaml:"server"` } From f784bc70219f57992c73ba9cdeb0ebf5c64bfb63 Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Wed, 24 Dec 2025 22:41:23 +0200 Subject: [PATCH 3/4] [chores] upd readme --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbc740e..4fdf4ae 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,25 @@ Bundler/RubyGems clients fetch a set of plain HTTP resources. HUB proxies and ca Bundler mirror example: ```bash -bundle config set --global mirror.https://rubygems.org http://localhost:6587/rubygems/rubygems +# all requests to rubygems.org must be forwarded to localhost:6587/rubygems/rubygems +bundle config --local mirror.https://rubygems.org http://localhost:6587/rubygems/rubygems ``` +Gemfile source example: + +```Gemfile +# by default all requests will be forwarded to proxy +source "http://localhost:6587/rubygems/rubygems" +gem "puma" + +# but for rack gem it will use some other upstream +source "https://rubygems.org" do + gem "rack" +end +``` + + + ### Static files Access cached static files: From 229fc314c5d43af92ee5195253f676de8afdcfad Mon Sep 17 00:00:00 2001 From: Dmitrij Shishkin Date: Wed, 24 Dec 2025 23:02:20 +0200 Subject: [PATCH 4/4] [chores] fix assignOp: replace `url = url + upstreamPath` with `url += upstreamPath` (gocritic) --- pkg/handlers/rubygems.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/handlers/rubygems.go b/pkg/handlers/rubygems.go index a795874..2fd35db 100644 --- a/pkg/handlers/rubygems.go +++ b/pkg/handlers/rubygems.go @@ -40,7 +40,7 @@ func RubyGems(key string) echo.HandlerFunc { upstreamBase := strings.TrimSuffix(cfg.Server.RUBYGEMS[key], "/") url := upstreamBase + "/" if upstreamPath != "" { - url = url + upstreamPath + url += upstreamPath } if query != "" { url = url + "?" + query