From d1797715bc64f75c99d1e926032bfde819539d47 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Wed, 17 Jun 2026 12:08:41 -0400 Subject: [PATCH 1/6] feat: prepend default scheme to schemeless proxied URLs Requesting a URL without an http(s):// prefix (e.g. /example.com/page) previously failed with 'unsupported protocol scheme ""'. The proxy now prepends a configured default scheme to such requests so they Just Work. The default is 'https' and is configurable via the DEFAULT_SCHEME env var or the --default-scheme / -s CLI flag. Only 'http' and 'https' are accepted; any other value causes a startup error. Behavior for URLs that already include a scheme is unchanged, and the existing referer-based relative-path reconstruction still takes precedence when the referer points at a proxied URL. --- README.md | 1 + cmd/main.go | 16 ++++ handlers/default_scheme_test.go | 133 ++++++++++++++++++++++++++++++++ handlers/proxy.go | 78 ++++++++++++++++--- 4 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 handlers/default_scheme_test.go diff --git a/README.md b/README.md index cc4c72e..d4218ce 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ http://localhost:8080/ruleset | `ALLOWED_DOMAINS` | Comma separated list of allowed domains. Empty = no limitations | `` | | `ALLOWED_DOMAINS_RULESET` | Allow Domains from Ruleset. false = no limitations | `false` | | `FLARESOLVERR_HOST` | URL for the FlareSolverr service for Cloudflare bypass (optional) | `http://localhost:8191` | +| `DEFAULT_SCHEME` | Scheme prepended when a requested URL has no scheme (`example.com/page` → `://example.com/page`). Must be `http` or `https`. Also configurable via the `--default-scheme` / `-s` CLI flag. | `https` | `ALLOWED_DOMAINS` and `ALLOWED_DOMAINS_RULESET` are joined together. If both are empty, no limitations are applied. | `BASE_PATH` | Base path for the proxy, useful if you want to run the proxy on a subpath (e.g. http://localhost:8080/proxy/) | `` | diff --git a/cmd/main.go b/cmd/main.go index 54cab3a..e242bc9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,6 +61,17 @@ func main() { Help: "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.", }) + defaultSchemeEnv := os.Getenv("DEFAULT_SCHEME") + if defaultSchemeEnv == "" { + defaultSchemeEnv = "https" + } + + defaultScheme := parser.String("s", "default-scheme", &argparse.Options{ + Required: false, + Default: defaultSchemeEnv, + Help: "Scheme to prepend when a proxied URL has no scheme. Must be \"http\" or \"https\".", + }) + err := parser.Parse(os.Args) if err != nil { fmt.Print(parser.Usage(err)) @@ -91,6 +102,11 @@ func main() { *prefork = true } + if err := handlers.SetDefaultScheme(*defaultScheme); err != nil { + fmt.Println(err) + os.Exit(1) + } + // BASE_PATH allows running ladder under a sub-path, e.g. /mypath basePath := os.Getenv("BASE_PATH") if basePath != "" { diff --git a/handlers/default_scheme_test.go b/handlers/default_scheme_test.go new file mode 100644 index 0000000..deee0da --- /dev/null +++ b/handlers/default_scheme_test.go @@ -0,0 +1,133 @@ +package handlers + +import ( + "net/http/httptest" + "testing" + + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" +) + +func TestSetDefaultScheme(t *testing.T) { + original := DefaultScheme() + t.Cleanup(func() { _ = SetDefaultScheme(original) }) + + assert.NoError(t, SetDefaultScheme("http")) + assert.Equal(t, "http", DefaultScheme()) + + assert.NoError(t, SetDefaultScheme("HTTPS")) + assert.Equal(t, "https", DefaultScheme()) + + assert.NoError(t, SetDefaultScheme(" https ")) + assert.Equal(t, "https", DefaultScheme()) + + for _, bad := range []string{"ftp", "ws", "", "javascript", "file"} { + err := SetDefaultScheme(bad) + assert.Error(t, err, "expected error for %q", bad) + } + // Last successful value should still be in place after rejected attempts. + assert.Equal(t, "https", DefaultScheme()) +} + +func TestEnsureScheme(t *testing.T) { + original := DefaultScheme() + t.Cleanup(func() { _ = SetDefaultScheme(original) }) + + require := func(scheme string) { + t.Helper() + if err := SetDefaultScheme(scheme); err != nil { + t.Fatalf("SetDefaultScheme(%q) failed: %v", scheme, err) + } + } + + require("https") + cases := []struct { + name string + in string + want string + }{ + {"already https", "https://example.com/page", "https://example.com/page"}, + {"already http", "http://example.com", "http://example.com"}, + {"schemeless host", "example.com/page", "https://example.com/page"}, + {"schemeless host with query", "example.com/page?q=1", "https://example.com/page?q=1"}, + {"schemeless host with port", "example.com:8443/page", "https://example.com:8443/page"}, + {"localhost with port", "localhost:8080/api", "https://localhost:8080/api"}, + {"truly relative path", "/images/foo.jpg", "/images/foo.jpg"}, + {"single segment no dot", "about", "about"}, + {"empty", "", ""}, + {"protocol-relative looks like host", "example.com", "https://example.com"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ensureScheme(tc.in)) + }) + } + + require("http") + assert.Equal(t, "http://example.com/page", ensureScheme("example.com/page"), + "schemeless URL should pick up the configured non-default scheme") + assert.Equal(t, "https://example.com/page", ensureScheme("https://example.com/page"), + "explicit scheme must never be overridden") +} + +func TestExtractUrlPrependsDefaultScheme(t *testing.T) { + original := DefaultScheme() + t.Cleanup(func() { _ = SetDefaultScheme(original) }) + require := func(scheme string) { + t.Helper() + if err := SetDefaultScheme(scheme); err != nil { + t.Fatalf("SetDefaultScheme(%q) failed: %v", scheme, err) + } + } + + run := func(t *testing.T, target string, headers map[string]string) string { + t.Helper() + app := fiber.New() + var captured string + app.Get("/*", func(c *fiber.Ctx) error { + u, err := extractUrl(c) + if err != nil { + return c.Status(500).SendString(err.Error()) + } + captured = u + return c.SendString("ok") + }) + req := httptest.NewRequest("GET", target, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := app.Test(req) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + return captured + } + + require("https") + t.Run("schemeless URL with no referer gets https", func(t *testing.T) { + got := run(t, "/example.com/page", nil) + assert.Equal(t, "https://example.com/page", got) + }) + + t.Run("https URL is unchanged", func(t *testing.T) { + got := run(t, "/https://example.com/page", nil) + assert.Equal(t, "https://example.com/page", got) + }) + + t.Run("http URL is unchanged", func(t *testing.T) { + got := run(t, "/http://example.com/page", nil) + assert.Equal(t, "http://example.com/page", got) + }) + + t.Run("relative path with proxied referer still reconstructs", func(t *testing.T) { + got := run(t, "/images/foo.jpg", map[string]string{ + "Referer": "http://localhost:8080/https://realsite.com/article", + }) + assert.Equal(t, "https://realsite.com/images/foo.jpg", got) + }) + + require("http") + t.Run("configured http scheme is used", func(t *testing.T) { + got := run(t, "/example.com/page", nil) + assert.Equal(t, "http://example.com/page", got) + }) +} diff --git a/handlers/proxy.go b/handlers/proxy.go index 56f2b0a..f2d8094 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -59,8 +59,50 @@ var ( allowedDomains = []string{} defaultTimeout = 15 // in seconds basePath = normalizeBasePath(os.Getenv("BASE_PATH")) + defaultScheme = "https" ) +// SetDefaultScheme configures the scheme prepended to proxied URLs that are +// supplied without one (e.g. "example.com/page"). Only "http" and "https" +// are accepted. +func SetDefaultScheme(scheme string) error { + scheme = strings.ToLower(strings.TrimSpace(scheme)) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("invalid default scheme %q: must be \"http\" or \"https\"", scheme) + } + defaultScheme = scheme + return nil +} + +// DefaultScheme returns the currently configured default URL scheme. +func DefaultScheme() string { + return defaultScheme +} + +// ensureScheme prepends the configured default scheme to a URL that looks +// like an absolute URL but is missing one (e.g. "example.com/page" becomes +// "https://example.com/page"). URLs that already carry a scheme and paths +// that don't look host-like are returned unchanged. +func ensureScheme(raw string) string { + if raw == "" { + return raw + } + if i := strings.Index(raw, "://"); i > 0 && i < 12 { + return raw + } + if strings.HasPrefix(raw, "/") { + return raw + } + first := raw + if idx := strings.IndexAny(raw, "/?#"); idx >= 0 { + first = raw[:idx] + } + if !strings.ContainsAny(first, ".:") { + return raw + } + return defaultScheme + "://" + raw +} + func normalizeBasePath(p string) string { if p == "" { return "" @@ -79,6 +121,11 @@ func init() { if timeoutStr := os.Getenv("HTTP_TIMEOUT"); timeoutStr != "" { defaultTimeout, _ = strconv.Atoi(timeoutStr) } + if scheme := os.Getenv("DEFAULT_SCHEME"); scheme != "" { + if err := SetDefaultScheme(scheme); err != nil { + log.Fatal(err) + } + } } // extracts a URL from the request ctx. If the URL in the request @@ -113,19 +160,29 @@ func extractUrl(c *fiber.Ctx) (string, error) { return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err) } - // reconstruct the full URL using the referer's scheme, host, and the relative path / queries - fullUrl := &url.URL{ - Scheme: realUrl.Scheme, - Host: realUrl.Host, - Path: urlQuery.Path, - RawQuery: urlQuery.RawQuery, - } + // If the referer points to a proxied site we can reconstruct the full + // URL from it. Otherwise (no referer, or referer not a proxied URL), + // the user supplied a schemeless absolute URL — fall back to the + // configured default scheme. + if realUrl.Scheme != "" && realUrl.Host != "" { + fullUrl := &url.URL{ + Scheme: realUrl.Scheme, + Host: realUrl.Host, + Path: urlQuery.Path, + RawQuery: urlQuery.RawQuery, + } - if os.Getenv("LOG_URLS") == "true" { - log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String()) + if os.Getenv("LOG_URLS") == "true" { + log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String()) + } + return fullUrl.String(), nil } - return fullUrl.String(), nil + prepended := ensureScheme(reqUrl) + if prepended != reqUrl && os.Getenv("LOG_URLS") == "true" { + log.Printf("prepended default scheme: '%s' -> '%s'", reqUrl, prepended) + } + return prepended, nil } // default behavior: @@ -243,6 +300,7 @@ func modifyURL(uri string, rule ruleset.Rule) (string, error) { } func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) { + urlpath = ensureScheme(urlpath) urlQuery := "?" if len(queries) > 0 { for k, v := range queries { From 1b44d56b13207c2943f2820aea746081aad59429 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Wed, 17 Jun 2026 12:27:27 -0400 Subject: [PATCH 2/6] test: pin the unsupported-protocol-scheme bug repro and add end-to-end fetchSite check --- handlers/default_scheme_test.go | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/handlers/default_scheme_test.go b/handlers/default_scheme_test.go index deee0da..d19afaf 100644 --- a/handlers/default_scheme_test.go +++ b/handlers/default_scheme_test.go @@ -1,7 +1,9 @@ package handlers import ( + "net/http" "net/http/httptest" + "strings" "testing" "github.com/gofiber/fiber/v2" @@ -131,3 +133,45 @@ func TestExtractUrlPrependsDefaultScheme(t *testing.T) { assert.Equal(t, "http://example.com/page", got) }) } + +// TestUnsupportedProtocolSchemeBugRepro pins down the exact stdlib error the +// user reported. http.Client returns `unsupported protocol scheme ""` when +// handed a schemeless host-style URL — no network needed, the transport +// rejects it before dialing. If Go ever changes this wording the test will +// flag it so we can update the comment/docs. +func TestUnsupportedProtocolSchemeBugRepro(t *testing.T) { + _, err := http.Get("example.com/page") + if err == nil { + t.Fatalf("expected stdlib http.Get to fail for schemeless URL") + } + if !strings.Contains(err.Error(), `unsupported protocol scheme ""`) { + t.Fatalf("expected `unsupported protocol scheme \"\"` in error, got: %v", err) + } +} + +// TestFetchSiteHandlesSchemelessURL is the end-to-end fix verification: a +// schemeless URL flows through fetchSite to a real (httptest) upstream and +// returns the body without the unsupported-protocol error. +func TestFetchSiteHandlesSchemelessURL(t *testing.T) { + originalScheme := DefaultScheme() + t.Cleanup(func() { _ = SetDefaultScheme(originalScheme) }) + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("hello from upstream")) + })) + t.Cleanup(upstream.Close) + + // httptest.NewServer speaks plain http; configure the default accordingly. + if err := SetDefaultScheme("http"); err != nil { + t.Fatalf("SetDefaultScheme: %v", err) + } + + schemelessHost := strings.TrimPrefix(upstream.URL, "http://") + + body, _, resp, err := fetchSite(schemelessHost, map[string]string{}) + assert.NoError(t, err, "schemeless URL should now succeed via the default-scheme fallback") + if assert.NotNil(t, resp) { + assert.Equal(t, http.StatusOK, resp.StatusCode) + } + assert.Contains(t, body, "hello from upstream") +} From efc9c56d7636ab265de817a083e184eac429ae62 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Wed, 17 Jun 2026 12:30:11 -0400 Subject: [PATCH 3/6] chore: trim verbose comments --- handlers/default_scheme_test.go | 11 +---------- handlers/proxy.go | 15 +++------------ 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/handlers/default_scheme_test.go b/handlers/default_scheme_test.go index d19afaf..811ddca 100644 --- a/handlers/default_scheme_test.go +++ b/handlers/default_scheme_test.go @@ -134,11 +134,6 @@ func TestExtractUrlPrependsDefaultScheme(t *testing.T) { }) } -// TestUnsupportedProtocolSchemeBugRepro pins down the exact stdlib error the -// user reported. http.Client returns `unsupported protocol scheme ""` when -// handed a schemeless host-style URL — no network needed, the transport -// rejects it before dialing. If Go ever changes this wording the test will -// flag it so we can update the comment/docs. func TestUnsupportedProtocolSchemeBugRepro(t *testing.T) { _, err := http.Get("example.com/page") if err == nil { @@ -149,9 +144,6 @@ func TestUnsupportedProtocolSchemeBugRepro(t *testing.T) { } } -// TestFetchSiteHandlesSchemelessURL is the end-to-end fix verification: a -// schemeless URL flows through fetchSite to a real (httptest) upstream and -// returns the body without the unsupported-protocol error. func TestFetchSiteHandlesSchemelessURL(t *testing.T) { originalScheme := DefaultScheme() t.Cleanup(func() { _ = SetDefaultScheme(originalScheme) }) @@ -161,7 +153,6 @@ func TestFetchSiteHandlesSchemelessURL(t *testing.T) { })) t.Cleanup(upstream.Close) - // httptest.NewServer speaks plain http; configure the default accordingly. if err := SetDefaultScheme("http"); err != nil { t.Fatalf("SetDefaultScheme: %v", err) } @@ -169,7 +160,7 @@ func TestFetchSiteHandlesSchemelessURL(t *testing.T) { schemelessHost := strings.TrimPrefix(upstream.URL, "http://") body, _, resp, err := fetchSite(schemelessHost, map[string]string{}) - assert.NoError(t, err, "schemeless URL should now succeed via the default-scheme fallback") + assert.NoError(t, err) if assert.NotNil(t, resp) { assert.Equal(t, http.StatusOK, resp.StatusCode) } diff --git a/handlers/proxy.go b/handlers/proxy.go index f2d8094..e0cfbf5 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -62,9 +62,8 @@ var ( defaultScheme = "https" ) -// SetDefaultScheme configures the scheme prepended to proxied URLs that are -// supplied without one (e.g. "example.com/page"). Only "http" and "https" -// are accepted. +// SetDefaultScheme sets the scheme prepended to schemeless proxied URLs. +// Only "http" and "https" are accepted. func SetDefaultScheme(scheme string) error { scheme = strings.ToLower(strings.TrimSpace(scheme)) if scheme != "http" && scheme != "https" { @@ -74,15 +73,11 @@ func SetDefaultScheme(scheme string) error { return nil } -// DefaultScheme returns the currently configured default URL scheme. +// DefaultScheme returns the configured default URL scheme. func DefaultScheme() string { return defaultScheme } -// ensureScheme prepends the configured default scheme to a URL that looks -// like an absolute URL but is missing one (e.g. "example.com/page" becomes -// "https://example.com/page"). URLs that already carry a scheme and paths -// that don't look host-like are returned unchanged. func ensureScheme(raw string) string { if raw == "" { return raw @@ -160,10 +155,6 @@ func extractUrl(c *fiber.Ctx) (string, error) { return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err) } - // If the referer points to a proxied site we can reconstruct the full - // URL from it. Otherwise (no referer, or referer not a proxied URL), - // the user supplied a schemeless absolute URL — fall back to the - // configured default scheme. if realUrl.Scheme != "" && realUrl.Host != "" { fullUrl := &url.URL{ Scheme: realUrl.Scheme, From f2daa6ed6170555baf9f12614e0ca3ec04219d87 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Wed, 17 Jun 2026 13:43:40 -0400 Subject: [PATCH 4/6] refactor: simplify ensureScheme, normalize before fetchSite, remove init log.Fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ensureScheme now uses a '://' substring check instead of magic-length and dot/colon heuristics, so single-segment hosts (localhost/api) and schemeless host:port URLs (example.com:8443/x) now work. - Strip a stray leading '://' so '://example.com' no longer produces 'https://://example.com'. - Move normalization out of fetchSite and into the handler boundaries (extractUrl, api, raw) — single normalization point per entry. - Drop the init() block that called log.Fatal on invalid DEFAULT_SCHEME. Validation now happens once in cmd/main.go where it's recoverable, so utility commands like --merge-rulesets aren't killed by an unrelated env var. --- handlers/api.go | 1 + handlers/default_scheme_test.go | 24 ++++++++++++++++-------- handlers/proxy.go | 21 ++++----------------- handlers/raw.go | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/handlers/api.go b/handlers/api.go index f88e79e..8e984bc 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -33,6 +33,7 @@ func Api(c *fiber.Ctx) error { url = c.Params("*") } + url = ensureScheme(url) body, req, resp, err := fetchSite(url, queries) if err != nil { log.Println("ERROR:", err) diff --git a/handlers/default_scheme_test.go b/handlers/default_scheme_test.go index 811ddca..9856931 100644 --- a/handlers/default_scheme_test.go +++ b/handlers/default_scheme_test.go @@ -1,6 +1,7 @@ package handlers import ( + "io" "net/http" "net/http/httptest" "strings" @@ -54,10 +55,13 @@ func TestEnsureScheme(t *testing.T) { {"schemeless host with query", "example.com/page?q=1", "https://example.com/page?q=1"}, {"schemeless host with port", "example.com:8443/page", "https://example.com:8443/page"}, {"localhost with port", "localhost:8080/api", "https://localhost:8080/api"}, + {"single-segment host", "localhost/api", "https://localhost/api"}, {"truly relative path", "/images/foo.jpg", "/images/foo.jpg"}, - {"single segment no dot", "about", "about"}, + {"bare hostname", "example.com", "https://example.com"}, + {"stray leading scheme separator", "://example.com", "https://example.com"}, + {"non-network scheme treated as schemeless", "mailto:foo@bar.com", "https://mailto:foo@bar.com"}, + {"ftp scheme left alone", "ftp://example.com/file", "ftp://example.com/file"}, {"empty", "", ""}, - {"protocol-relative looks like host", "example.com", "https://example.com"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -144,7 +148,7 @@ func TestUnsupportedProtocolSchemeBugRepro(t *testing.T) { } } -func TestFetchSiteHandlesSchemelessURL(t *testing.T) { +func TestRawHandlerHandlesSchemelessURL(t *testing.T) { originalScheme := DefaultScheme() t.Cleanup(func() { _ = SetDefaultScheme(originalScheme) }) @@ -159,10 +163,14 @@ func TestFetchSiteHandlesSchemelessURL(t *testing.T) { schemelessHost := strings.TrimPrefix(upstream.URL, "http://") - body, _, resp, err := fetchSite(schemelessHost, map[string]string{}) + app := fiber.New() + app.Get("/raw/*", Raw) + req := httptest.NewRequest("GET", "/raw/"+schemelessHost, nil) + resp, err := app.Test(req) assert.NoError(t, err) - if assert.NotNil(t, resp) { - assert.Equal(t, http.StatusOK, resp.StatusCode) - } - assert.Contains(t, body, "hello from upstream") + assert.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Contains(t, string(body), "hello from upstream") } diff --git a/handlers/proxy.go b/handlers/proxy.go index e0cfbf5..933a9d9 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -79,20 +79,13 @@ func DefaultScheme() string { } func ensureScheme(raw string) string { - if raw == "" { + if raw == "" || strings.HasPrefix(raw, "/") { return raw } - if i := strings.Index(raw, "://"); i > 0 && i < 12 { - return raw - } - if strings.HasPrefix(raw, "/") { - return raw - } - first := raw - if idx := strings.IndexAny(raw, "/?#"); idx >= 0 { - first = raw[:idx] + if strings.HasPrefix(raw, "://") { + raw = raw[3:] } - if !strings.ContainsAny(first, ".:") { + if strings.Contains(raw, "://") { return raw } return defaultScheme + "://" + raw @@ -116,11 +109,6 @@ func init() { if timeoutStr := os.Getenv("HTTP_TIMEOUT"); timeoutStr != "" { defaultTimeout, _ = strconv.Atoi(timeoutStr) } - if scheme := os.Getenv("DEFAULT_SCHEME"); scheme != "" { - if err := SetDefaultScheme(scheme); err != nil { - log.Fatal(err) - } - } } // extracts a URL from the request ctx. If the URL in the request @@ -291,7 +279,6 @@ func modifyURL(uri string, rule ruleset.Rule) (string, error) { } func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) { - urlpath = ensureScheme(urlpath) urlQuery := "?" if len(queries) > 0 { for k, v := range queries { diff --git a/handlers/raw.go b/handlers/raw.go index 7baa0e5..94b5fa8 100644 --- a/handlers/raw.go +++ b/handlers/raw.go @@ -8,7 +8,7 @@ import ( func Raw(c *fiber.Ctx) error { // Get the url from the URL - urlQuery := c.Params("*") + urlQuery := ensureScheme(c.Params("*")) queries := c.Queries() body, _, _, err := fetchSite(urlQuery, queries) From 18dcd67c9784df2c382ae4f9a6b945ee49c6ba17 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Thu, 25 Jun 2026 13:23:18 -0400 Subject: [PATCH 5/6] fix: actually invoke applyRules so regexRules + injections take effect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The applyRules function was defined but never called from any code path, making the entire ruleset infrastructure (regexRules, injections) dead code. The only ruleset fields that took effect were Headers, URLMods, and the incidental side effects of rewriteHtml (which renames `src="/..."` to `script="..."` on relative-URL script tags). Invoke applyRules in fetchSite, BEFORE rewriteHtml. Order matters: - regexRules patterns like `]*src="..."` need to see original src attributes (rewriteHtml renames them and breaks subsequent matches). - injections add content to , position-independent — but easier to follow if all ruleset work happens before URL munging. --- handlers/proxy.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/handlers/proxy.go b/handlers/proxy.go index 933a9d9..3ce6332 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -377,8 +377,12 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request resp.Header.Del("Content-Security-Policy") } - // log.Print("rule", rule) TODO: Add a debug mode to print the rule - body := rewriteHtml(bodyB, u, rule) + // Apply ruleset modifications (regexRules + injections) BEFORE URL rewriting: + // rewriteHtml renames `src="/..."` to `script="..."` on relative-URL script tags, + // which would otherwise prevent our `]*src="..."` regex patterns from + // matching. applyRules also adds injections to before any prefix munging. + body := applyRules(string(bodyB), rule) + body = rewriteHtml([]byte(body), u, rule) return body, req, resp, nil } From 58ee7e0397ad8bcfaca6a7c6aad696376a507e99 Mon Sep 17 00:00:00 2001 From: Lemuel Barango Date: Thu, 25 Jun 2026 13:23:19 -0400 Subject: [PATCH 6/6] test: regression coverage for applyRules invocation + ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three tests in handlers/apply_rules_test.go: - TestApplyRulesUnit — applyRules itself accepts regexRules + injections and applies both. Always passed; documents the surface area. - TestFetchSiteInvokesApplyRules — regression test for the dead-code bug. Spins up an httptest.NewServer, configures a rule with regexRules and injections, calls fetchSite, asserts both effects landed. Fails without the applyRules invocation in fetchSite. - TestApplyRulesOrderingBeforeRewriteHtml — codifies the ordering: applyRules must run BEFORE rewriteHtml because rewriteHtml renames src= attributes on relative-URL script tags, which would otherwise prevent src-based regex patterns from matching. Fails if the call order is swapped. Verified: with the fix in proxy.go reverted, the second and third tests fail with clear error messages pointing at the ruleset effects that didn't land. --- handlers/apply_rules_test.go | 127 +++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 handlers/apply_rules_test.go diff --git a/handlers/apply_rules_test.go b/handlers/apply_rules_test.go new file mode 100644 index 0000000..ea4407c --- /dev/null +++ b/handlers/apply_rules_test.go @@ -0,0 +1,127 @@ +package handlers + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "ladder/pkg/ruleset" + + "github.com/stretchr/testify/assert" +) + +// TestApplyRulesUnit exercises applyRules directly: a regex strip and a head +// injection should both land on the input body. +func TestApplyRulesUnit(t *testing.T) { + // Seed rulesSet so applyRules doesn't early-return when len(rulesSet)==0. + prev := rulesSet + rulesSet = ruleset.RuleSet{{Domain: "example.test"}} + t.Cleanup(func() { rulesSet = prev }) + + input := `

hello

` + + rule := ruleset.Rule{ + Domain: "example.test", + RegexRules: []ruleset.Regex{ + {Match: `]*>`, Replace: ""}, + }, + } + rule.Injections = append(rule.Injections, struct { + Position string `yaml:"position,omitempty"` + Append string `yaml:"append,omitempty"` + Prepend string `yaml:"prepend,omitempty"` + Replace string `yaml:"replace,omitempty"` + }{Position: "head", Append: ``}) + + out := applyRules(input, rule) + + assert.NotContains(t, out, `

body content

` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, upstreamHTML) + })) + t.Cleanup(server.Close) + + u, err := url.Parse(server.URL) + assert.NoError(t, err) + + rule := ruleset.Rule{ + Domain: u.Host, + RegexRules: []ruleset.Regex{ + // Strict pattern requiring src= — only matches if applyRules runs + // BEFORE rewriteHtml's src→script rename. Confirms ordering too. + {Match: `]*src="https://example\.test/paywall\.js"[^>]*>`, Replace: ""}, + }, + } + rule.Injections = append(rule.Injections, struct { + Position string `yaml:"position,omitempty"` + Append string `yaml:"append,omitempty"` + Prepend string `yaml:"prepend,omitempty"` + Replace string `yaml:"replace,omitempty"` + }{Position: "head", Append: ``}) + + prev := rulesSet + rulesSet = ruleset.RuleSet{rule} + t.Cleanup(func() { rulesSet = prev }) + + body, _, _, err := fetchSite(server.URL, map[string]string{}) + assert.NoError(t, err) + + assert.NotContains(t, body, `paywall.js`, "regexRules in fetchSite should strip the upstream script") + assert.Contains(t, body, `