Add RubyGems proxy support

README.md

@@ -7,6 +7,8 @@ dir: _data
 server:
   pypi:
     pypi.org: https://pypi.org/simple
+  rubygems:
+    rubygems.org: https://rubygems.org
   galaxy:
     ansible:
       url: https://galaxy.ansible.com
@@ -38,6 +40,42 @@ Access cached Galaxy collections:
 http://localhost:6587/galaxy/ansible/api/v3/collections/{namespace}/{name}/
 ```
 
+### RubyGems
+
+Use HUB as a RubyGems/Bundler source:
+
+```text
+http://localhost:6587/rubygems/{repo_name}
+```
+
+Bundler/RubyGems clients fetch a set of plain HTTP resources. HUB proxies and caches any path under the configured upstream (wildcard route), including common endpoints:
+
+- Compact index: `/names`, `/versions`, `/info/<gem>`
+- Legacy indexes: `/specs.4.8.gz`, `/latest_specs.4.8.gz`, `/prerelease_specs.4.8.gz`, `/quick/Marshal.4.8/*.gemspec.rz`
+- Gem artifacts: `/gems/<name>-<version>.gem`
+
+Bundler mirror example:
+
+```bash
+# all requests to rubygems.org must be forwarded to localhost:6587/rubygems/rubygems
+bundle config --local mirror.https://rubygems.org http://localhost:6587/rubygems/rubygems
+```
+
+Gemfile source example:
+
+```Gemfile
+# by default all requests will be forwarded to proxy
+source "http://localhost:6587/rubygems/rubygems"
+gem "puma"
+
+# but for rack gem it will use some other upstream
+source "https://rubygems.org" do
+  gem "rack"
+end
+```
+
+
+
 ### Static files
 
 Access cached static files:

config.yaml

@@ -2,6 +2,8 @@ dir: _data
 server:
   pypi:
     pypi.org: https://pypi.org/simple
+  rubygems:
+    rubygems: https://rubygems.org
   galaxy:
     ansible:
       url: https://galaxy.ansible.com

main.go

@@ -155,6 +155,11 @@ func startServer(c *cli.Context) error {
 		p.GET("/packages/:name/:filename", handlers.PypiPackages(k)).Name = fmt.Sprintf("pypi::%s::packages", k)
 	}
 
+	for k := range cfg.Server.RUBYGEMS {
+		r := e.Group(fmt.Sprintf("/rubygems/%s", k))
+		r.GET("/*", handlers.RubyGems(k)).Name = fmt.Sprintf("rubygems::%s", k)
+	}
+
 	for k := range cfg.Server.Static {
 		s := e.Group(fmt.Sprintf("/static/%s", k))
 		s.GET("/get/*", handlers.Static(k)).Name = fmt.Sprintf("static::%s", k)

pkg/handlers/galaxy_proxy.go

@@ -25,7 +25,7 @@ func GalaxyProxyCollection(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/galaxy/%s/index/%s/%s/index.json", cfg.Dir, key, namespace, name)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "ansible-galaxy",
+			"User-Agent": "ansible-galaxy",
 		}
 		status, err := misc.DownloadFile(url, dest, headers)
 		if err != nil {
@@ -62,7 +62,7 @@ func GalaxyProxyCollectionVersions(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/galaxy/%s/index/%s/%s/versions/index/%s", cfg.Dir, key, namespace, name, c.QueryString())
 
 		headers := types.RequestHeaders{
-			"UserAgent": "ansible-galaxy",
+			"User-Agent": "ansible-galaxy",
 		}
 		status, err := misc.DownloadFile(url, dest, headers)
 		if err != nil {
@@ -100,7 +100,7 @@ func GalaxyProxyCollectionVersionInfo(key string) echo.HandlerFunc {
 		host := c.Request().Host
 
 		headers := types.RequestHeaders{
-			"UserAgent": "ansible-galaxy",
+			"User-Agent": "ansible-galaxy",
 		}
 		_, err := misc.DownloadFile(url, dest, headers)
 		if err != nil {
@@ -143,7 +143,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc {
 			url := fmt.Sprintf("%s/api/v3/collections/%s/%s/versions/%s", cfg.Server.Galaxy[key].URL, namespace, name, version)
 
 			headers := types.RequestHeaders{
-				"UserAgent": "ansible-galaxy",
+				"User-Agent": "ansible-galaxy",
 			}
 			_, err := misc.DownloadFile(url, versionFile, headers)
 			if err != nil {
@@ -161,7 +161,7 @@ func GalaxyProxyCollectionGet(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/galaxy/%s/binary/%s/%s/%s-%s-%s.tar.gz", cfg.Dir, key, namespace, name, namespace, name, version)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "ansible-galaxy",
+			"User-Agent": "ansible-galaxy",
 		}
 		if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) {
 			status, err := misc.DownloadFile(url, dest, headers)

pkg/handlers/goproxy.go

@@ -20,7 +20,7 @@ func downloadAndCacheFile(c echo.Context, key, loggerNS, url, dest string) error
 	logger := c.Get("logger").(*zap.SugaredLogger)
 
 	headers := types.RequestHeaders{
-		"UserAgent": "go/goproxy",
+		"User-Agent": "go/goproxy",
 	}
 
 	status, err := misc.DownloadFile(url, dest, headers)
@@ -54,7 +54,7 @@ func GoProxyList(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/goproxy/%s/%s/@v/list", cfg.Dir, key, module)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "go/goproxy",
+			"User-Agent": "go/goproxy",
 		}
 
 		status, err := misc.DownloadFile(url, dest, headers)
@@ -145,7 +145,7 @@ func GoProxyZip(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/goproxy/%s/%s/@v/%s.zip", cfg.Dir, key, modulePath, version)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "go/goproxy",
+			"User-Agent": "go/goproxy",
 		}
 
 		// Check if file exists and has valid content
@@ -188,7 +188,7 @@ func GoProxyLatest(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/goproxy/%s/%s/@latest", cfg.Dir, key, module)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "go/goproxy",
+			"User-Agent": "go/goproxy",
 		}
 
 		// @latest should be fetched more frequently, so check if file is older than 1 hour

pkg/handlers/pypi.go

@@ -26,8 +26,8 @@ func PypiSimple(key string) echo.HandlerFunc {
 		scheme := c.Scheme()
 		host := c.Request().Host
 		headers := types.RequestHeaders{
-			"UserAgent": "pypi",
-			"Accept":    "application/vnd.pypi.simple.v1+json",
+			"User-Agent": "pypi",
+			"Accept":     "application/vnd.pypi.simple.v1+json",
 		}
 
 		status, err := misc.DownloadFile(url, dest, headers)
@@ -70,7 +70,7 @@ func PypiPackages(key string) echo.HandlerFunc {
 		var url, sha string
 
 		headers := types.RequestHeaders{
-			"UserAgent": "pypi",
+			"User-Agent": "pypi",
 		}
 
 		indexFileInfo, err := os.Stat(indexDest)
@@ -84,8 +84,8 @@ func PypiPackages(key string) echo.HandlerFunc {
 				url = fmt.Sprintf("%s/%s/", cfg.Server.PYPI[key], name)
 
 				headers = types.RequestHeaders{
-					"UserAgent": "pypi",
-					"Accept":    "application/vnd.pypi.simple.v1+json",
+					"User-Agent": "pypi",
+					"Accept":     "application/vnd.pypi.simple.v1+json",
 				}
 				_, err := misc.DownloadFile(url, indexDest, headers)
 				if err != nil {
@@ -103,8 +103,8 @@ func PypiPackages(key string) echo.HandlerFunc {
 			url = fmt.Sprintf("%s/%s/", cfg.Server.PYPI[key], name)
 
 			headers = types.RequestHeaders{
-				"UserAgent": "pypi",
-				"Accept":    "application/vnd.pypi.simple.v1+json",
+				"User-Agent": "pypi",
+				"Accept":     "application/vnd.pypi.simple.v1+json",
 			}
 			_, err := misc.DownloadFile(url, indexDest, headers)
 			if err != nil {

pkg/handlers/rubygems.go

@@ -0,0 +1,85 @@
+package handlers
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/psvmcc/hub/pkg/misc"
+	"github.com/psvmcc/hub/pkg/types"
+
+	"github.com/labstack/echo/v4"
+	"go.uber.org/zap"
+)
+
+func RubyGems(key string) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		cfg := c.Get("cfg").(types.ConfigFile)
+		logger := c.Get("logger").(*zap.SugaredLogger)
+		loggerNS := "rubygems"
+
+		requestedPath := strings.TrimPrefix(c.Param("*"), "/")
+		upstreamPath := strings.TrimPrefix(path.Clean("/"+requestedPath), "/")
+		cacheKey := upstreamPath
+		if upstreamPath == "" || upstreamPath == "." {
+			upstreamPath = ""
+			cacheKey = "__root"
+		}
+
+		query := c.QueryString()
+		cachePath := cacheKey
+		if query != "" {
+			sum := sha256.Sum256([]byte(query))
+			cachePath = path.Join("_query", hex.EncodeToString(sum[:]), cacheKey)
+		}
+
+		upstreamBase := strings.TrimSuffix(cfg.Server.RUBYGEMS[key], "/")
+		url := upstreamBase + "/"
+		if upstreamPath != "" {
+			url += upstreamPath
+		}
+		if query != "" {
+			url = url + "?" + query
+		}
+
+		dest := fmt.Sprintf("%s/rubygems/%s/%s", cfg.Dir, key, cachePath)
+
+		headers := types.RequestHeaders{
+			"User-Agent": "rubygems",
+		}
+
+		if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) {
+			c.Response().Header().Add("X-Cache-Status", "MISS")
+		} else {
+			equal, err := misc.FilesEqual(url, dest)
+			if err != nil {
+				logger.Named(loggerNS).Errorf("[FilesEqual]: %s", err)
+			}
+
+			if equal {
+				c.Response().Header().Add("X-Cache-Status", "HIT")
+				return c.File(dest)
+			}
+
+			c.Response().Header().Add("X-Cache-Status", "EXPIRE")
+		}
+
+		status, err := misc.DownloadFile(url, dest, headers)
+		if err != nil {
+			logger.Named(loggerNS).Errorf("[Downloading] %s", err)
+			if _, statErr := os.Stat(dest); errors.Is(statErr, os.ErrNotExist) {
+				logger.Named(loggerNS).Errorf("[FS]: %s", statErr)
+				return c.String(status, "Please check logs...")
+			}
+			logger.Named(loggerNS).Debugf("Remote %s served from local file %s", url, dest)
+			return c.File(dest)
+		}
+
+		logger.Named(loggerNS).Debugf("Remote %s saved as %s", url, dest)
+		return c.File(dest)
+	}
+}

pkg/handlers/static.go

@@ -23,7 +23,7 @@ func Static(key string) echo.HandlerFunc {
 		dest := fmt.Sprintf("%s/static/%s/%s", cfg.Dir, key, path)
 
 		headers := types.RequestHeaders{
-			"UserAgent": "curl",
+			"User-Agent": "curl",
 		}
 
 		if _, err := os.Stat(dest); errors.Is(err, os.ErrNotExist) {

pkg/misc/download.go

@@ -24,7 +24,7 @@ func DownloadFile(url, destination string, headers types.RequestHeaders) (code i
 		code = http.StatusBadRequest
 		return code, err
 	}
-	req.Header.Set("UserAgent", "hub")
+	req.Header.Set("User-Agent", "hub")
 
 	for k, v := range headers {
 		req.Header.Set(k, v)
@@ -38,6 +38,7 @@ func DownloadFile(url, destination string, headers types.RequestHeaders) (code i
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
+		err = fmt.Errorf("upstream returned %s", response.Status)
 		code = response.StatusCode
 		return code, err
 	}

pkg/types/config.go

@@ -15,9 +15,10 @@ type ConfigFile struct {
 			URL string `yaml:"url"`
 			Dir string `yaml:"dir"`
 		} `yaml:"galaxy"`
-		PYPI    map[string]string `yaml:"pypi"`
-		Static  map[string]string `yaml:"static"`
-		GOPROXY map[string]string `yaml:"goproxy"`
+		PYPI     map[string]string `yaml:"pypi"`
+		RUBYGEMS map[string]string `yaml:"rubygems"`
+		Static   map[string]string `yaml:"static"`
+		GOPROXY  map[string]string `yaml:"goproxy"`
 	} `yaml:"server"`
 }