From 19fa76e1c1d0caa9cf830c457142ea01dfd44785 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Wed, 28 May 2025 14:52:54 +0700 Subject: [PATCH 1/3] config: Normalize URL for fallback mirrors as well The function `NormalizeURL` is used to ensure that a mirror URL ends with a trailing slash. It's applied when the mirror configuration is written to the database (cf. `setMirror` in `rpc/rpc.go`). Mirrorbits then relies on this fact when it constructs the redirect URLs, in `pagerenderer.go`: ``` path := strings.TrimPrefix(results.FileInfo.Path, "/") [...] http.Redirect([...], results.MirrorList[0].AbsoluteURL+path, http.StatusFound) ``` However, it turns out that `NormalizeURL` is not applied to the fallback URLs. Meaning that if your fallback is `http://mirror.org` and you request the file `foobar`, and if ever the fallback mechanism kicks in, mirrorbits will happily redirect you to `http://mirror.orgfoobar`... Easy to fix! One less papercut! --- config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/config.go b/config/config.go index 38dd0920..78e7e588 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/etix/mirrorbits/core" + "github.com/etix/mirrorbits/utils" "github.com/op/go-logging" "gopkg.in/yaml.v3" ) @@ -175,6 +176,9 @@ func ReloadConfig() error { if c.RepositoryScanInterval < 0 { c.RepositoryScanInterval = 0 } + for i := range c.Fallbacks { + c.Fallbacks[i].URL = utils.NormalizeURL(c.Fallbacks[i].URL) + } for _, rule := range c.AllowOutdatedFiles { if len(rule.Prefix) > 0 && rule.Prefix[0] != '/' { return fmt.Errorf("AllowOutdatedFiles.Prefix must start with '/'") From aed7cffa9349c1a8b0a573bea4e21956398dd7b5 Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Wed, 28 May 2025 17:52:06 +0700 Subject: [PATCH 2/3] networ, util: Move Is{Primary,Additional}Country into the network package The previous commit introduced a import cycle: ``` package github.com/etix/mirrorbits imports github.com/etix/mirrorbits/cli from main.go imports github.com/etix/mirrorbits/filesystem from commands.go imports github.com/etix/mirrorbits/config from hash.go imports github.com/etix/mirrorbits/utils from config.go imports github.com/etix/mirrorbits/network from utils.go imports github.com/etix/mirrorbits/config from geoip.go: import cycle not allowed ``` This is because we now import `utils` from within `config`. Let's move the two network-related utils to `network/utils.go`, so that we don't need to import `network` in `utils.go` anymore. That's enough to break the import cycle. --- http/selection.go | 6 +++--- network/utils.go | 24 ++++++++++++++++++++++ network/utils_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++ utils/utils.go | 25 ----------------------- utils/utils_test.go | 47 ------------------------------------------- 5 files changed, 73 insertions(+), 75 deletions(-) diff --git a/http/selection.go b/http/selection.go index 7e33cf84..ad312e55 100644 --- a/http/selection.go +++ b/http/selection.go @@ -87,13 +87,13 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f if m.Distance <= closestMirror*GetConfig().WeightDistributionRange { score := (float32(baseScore) - m.Distance) - if !utils.IsPrimaryCountry(clientInfo, m.CountryFields) { + if !network.IsPrimaryCountry(clientInfo, m.CountryFields) { score /= 2 } m.ComputedScore += int(score) - } else if utils.IsPrimaryCountry(clientInfo, m.CountryFields) { + } else if network.IsPrimaryCountry(clientInfo, m.CountryFields) { m.ComputedScore += int(float32(baseScore) - (m.Distance * 5)) - } else if utils.IsAdditionalCountry(clientInfo, m.CountryFields) { + } else if network.IsAdditionalCountry(clientInfo, m.CountryFields) { m.ComputedScore += int(float32(baseScore) - closestMirror) } diff --git a/network/utils.go b/network/utils.go index a0587552..db0e697a 100644 --- a/network/utils.go +++ b/network/utils.go @@ -42,3 +42,27 @@ func ExtractRemoteIP(XForwardedFor string) string { } return "" } + +// IsPrimaryCountry returns true if the clientInfo country is the primary country +func IsPrimaryCountry(clientInfo GeoIPRecord, list []string) bool { + if !clientInfo.IsValid() { + return false + } + if len(list) > 0 && list[0] == clientInfo.CountryCode { + return true + } + return false +} + +// IsAdditionalCountry returns true if the clientInfo country is in list +func IsAdditionalCountry(clientInfo GeoIPRecord, list []string) bool { + if !clientInfo.IsValid() { + return false + } + for i, b := range list { + if i > 0 && b == clientInfo.CountryCode { + return true + } + } + return false +} diff --git a/network/utils_test.go b/network/utils_test.go index 5209d626..f1653205 100644 --- a/network/utils_test.go +++ b/network/utils_test.go @@ -35,3 +35,49 @@ func TestExtractRemoteIP(t *testing.T) { t.Fatalf("Expected '192.168.0.1', got %s", r) } } + +func TestIsPrimaryCountry(t *testing.T) { + var b bool + list := []string{"FR", "DE", "GR"} + + clientInfo := GeoIPRecord{ + CountryCode: "FR", + } + + b = IsPrimaryCountry(clientInfo, list) + if !b { + t.Fatal("Expected true, got false") + } + + clientInfo = GeoIPRecord{ + CountryCode: "GR", + } + + b = IsPrimaryCountry(clientInfo, list) + if b { + t.Fatal("Expected false, got true") + } +} + +func TestIsAdditionalCountry(t *testing.T) { + var b bool + list := []string{"FR", "DE", "GR"} + + clientInfo := GeoIPRecord{ + CountryCode: "FR", + } + + b = IsAdditionalCountry(clientInfo, list) + if b { + t.Fatal("Expected false, got true") + } + + clientInfo = GeoIPRecord{ + CountryCode: "GR", + } + + b = IsAdditionalCountry(clientInfo, list) + if !b { + t.Fatal("Expected true, got false") + } +} diff --git a/utils/utils.go b/utils/utils.go index e672052b..8c1cfaee 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -11,7 +11,6 @@ import ( "time" "github.com/etix/mirrorbits/core" - "github.com/etix/mirrorbits/network" ) const ( @@ -93,30 +92,6 @@ func IsInSlice(a string, list []string) bool { return false } -// IsAdditionalCountry returns true if the clientInfo country is in list -func IsAdditionalCountry(clientInfo network.GeoIPRecord, list []string) bool { - if !clientInfo.IsValid() { - return false - } - for i, b := range list { - if i > 0 && b == clientInfo.CountryCode { - return true - } - } - return false -} - -// IsPrimaryCountry returns true if the clientInfo country is the primary country -func IsPrimaryCountry(clientInfo network.GeoIPRecord, list []string) bool { - if !clientInfo.IsValid() { - return false - } - if len(list) > 0 && list[0] == clientInfo.CountryCode { - return true - } - return false -} - // IsStopped returns true if a stop has been requested func IsStopped(stop <-chan struct{}) bool { select { diff --git a/utils/utils_test.go b/utils/utils_test.go index dfddcc7d..05a1af3c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/etix/mirrorbits/core" - "github.com/etix/mirrorbits/network" ) func TestHasAnyPrefix(t *testing.T) { @@ -103,52 +102,6 @@ func TestIsInSlice(t *testing.T) { } } -func TestIsAdditionalCountry(t *testing.T) { - var b bool - list := []string{"FR", "DE", "GR"} - - clientInfo := network.GeoIPRecord{ - CountryCode: "FR", - } - - b = IsAdditionalCountry(clientInfo, list) - if b { - t.Fatal("Expected false, got true") - } - - clientInfo = network.GeoIPRecord{ - CountryCode: "GR", - } - - b = IsAdditionalCountry(clientInfo, list) - if !b { - t.Fatal("Expected true, got false") - } -} - -func TestIsPrimaryCountry(t *testing.T) { - var b bool - list := []string{"FR", "DE", "GR"} - - clientInfo := network.GeoIPRecord{ - CountryCode: "FR", - } - - b = IsPrimaryCountry(clientInfo, list) - if !b { - t.Fatal("Expected true, got false") - } - - clientInfo = network.GeoIPRecord{ - CountryCode: "GR", - } - - b = IsPrimaryCountry(clientInfo, list) - if b { - t.Fatal("Expected false, got true") - } -} - func TestIsStopped(t *testing.T) { stop := make(chan struct{}, 1) From a5c43bf5895aea8461827f45a393ddc82483ce5f Mon Sep 17 00:00:00 2001 From: Arnaud Rebillout Date: Thu, 29 May 2025 14:25:58 +0700 Subject: [PATCH 3/3] config: Remove copy of isInSlice Now that we import `utils` in `config` (cf. previous commits), we can use `IsInSlice` from `utils.go`, no need to duplicate it anymore. --- config/config.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/config/config.go b/config/config.go index 78e7e588..e264798d 100644 --- a/config/config.go +++ b/config/config.go @@ -163,7 +163,7 @@ func ReloadConfig() error { if c.WeightDistributionRange <= 0 { return fmt.Errorf("WeightDistributionRange must be > 0") } - if !isInSlice(c.OutputMode, []string{"auto", "json", "redirect"}) { + if !utils.IsInSlice(c.OutputMode, []string{"auto", "json", "redirect"}) { return fmt.Errorf("Config: outputMode can only be set to 'auto', 'json' or 'redirect'") } if c.Repository == "" { @@ -266,13 +266,3 @@ func testSentinelsEq(a, b []sentinels) bool { return true } - -//DUPLICATE -func isInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -}