everywall · lemuelbarango · Jun 17, 2026 · Jun 17, 2026 · Jun 17, 2026 · Jun 17, 2026
diff --git a/README.md b/README.md
@@ -129,6 +129,7 @@ http://localhost:8080/ruleset
 | `ALLOWED_DOMAINS` | Comma separated list of allowed domains. Empty = no limitations | `` |
 | `ALLOWED_DOMAINS_RULESET` | Allow Domains from Ruleset. false = no limitations | `false` |
 | `FLARESOLVERR_HOST` | URL for the FlareSolverr service for Cloudflare bypass (optional) | `http://localhost:8191` |
+| `DEFAULT_SCHEME` | Scheme prepended when a requested URL has no scheme (`example.com/page` → `<scheme>://example.com/page`). Must be `http` or `https`. Also configurable via the `--default-scheme` / `-s` CLI flag. | `https` |
 
 `ALLOWED_DOMAINS` and `ALLOWED_DOMAINS_RULESET` are joined together. If both are empty, no limitations are applied.
 | `BASE_PATH` | Base path for the proxy, useful if you want to run the proxy on a subpath (e.g. http://localhost:8080/proxy/) | `` |

diff --git a/cmd/main.go b/cmd/main.go
@@ -61,6 +61,17 @@ func main() {
 		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
 	})
 
+	defaultSchemeEnv := os.Getenv("DEFAULT_SCHEME")
+	if defaultSchemeEnv == "" {
+		defaultSchemeEnv = "https"
+	}
+
+	defaultScheme := parser.String("s", "default-scheme", &argparse.Options{
+		Required: false,
+		Default:  defaultSchemeEnv,
+		Help:     "Scheme to prepend when a proxied URL has no scheme. Must be \"http\" or \"https\".",
+	})
+
 	err := parser.Parse(os.Args)
 	if err != nil {
 		fmt.Print(parser.Usage(err))
@@ -91,6 +102,11 @@ func main() {
 		*prefork = true
 	}
 
+	if err := handlers.SetDefaultScheme(*defaultScheme); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
 	// BASE_PATH allows running ladder under a sub-path, e.g. /mypath
 	basePath := os.Getenv("BASE_PATH")
 	if basePath != "" {

diff --git a/handlers/api.go b/handlers/api.go
@@ -33,6 +33,7 @@ func Api(c *fiber.Ctx) error {
 		url = c.Params("*")
 	}
 
+	url = ensureScheme(url)
 	body, req, resp, err := fetchSite(url, queries)
 	if err != nil {
 		log.Println("ERROR:", err)

diff --git a/handlers/apply_rules_test.go b/handlers/apply_rules_test.go
@@ -0,0 +1,127 @@
+package handlers
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"ladder/pkg/ruleset"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestApplyRulesUnit exercises applyRules directly: a regex strip and a head
+// injection should both land on the input body.
+func TestApplyRulesUnit(t *testing.T) {
+	// Seed rulesSet so applyRules doesn't early-return when len(rulesSet)==0.
+	prev := rulesSet
+	rulesSet = ruleset.RuleSet{{Domain: "example.test"}}
+	t.Cleanup(func() { rulesSet = prev })
+
+	input := `<html><head></head><body><script src="/paywall.js"></script><p>hello</p></body></html>`
+
+	rule := ruleset.Rule{
+		Domain: "example.test",
+		RegexRules: []ruleset.Regex{
+			{Match: `<script[^>]*></script>`, Replace: ""},
+		},
+	}
+	rule.Injections = append(rule.Injections, struct {
+		Position string `yaml:"position,omitempty"`
+		Append   string `yaml:"append,omitempty"`
+		Prepend  string `yaml:"prepend,omitempty"`
+		Replace  string `yaml:"replace,omitempty"`
+	}{Position: "head", Append: `<style>.paywall{display:none}</style>`})
+
+	out := applyRules(input, rule)
+
+	assert.NotContains(t, out, `<script src="/paywall.js">`, "regexRules should strip the script tag")
+	assert.Contains(t, out, `<style>.paywall{display:none}</style>`, "injections should append to <head>")
+	assert.Contains(t, out, `<p>hello</p>`, "non-matching content should pass through unchanged")
+}
+
+// TestFetchSiteInvokesApplyRules is the regression test for the dead-code bug
+// where applyRules was defined but never called from fetchSite. The test
+// configures a rule with both regexRules and injections, then verifies they
+// take effect on a response served by an httptest.NewServer.
+//
+// Without the fix this test fails: the script tag passes through unmangled
+// (or only mangled by rewriteHtml's src→script rename, which won't match the
+// strict regex below) and the injection is absent from <head>.
+func TestFetchSiteInvokesApplyRules(t *testing.T) {
+	const upstreamHTML = `<html><head><title>t</title></head><body><script src="https://example.test/paywall.js"></script><p>body content</p></body></html>`
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, upstreamHTML)
+	}))
+	t.Cleanup(server.Close)
+
+	u, err := url.Parse(server.URL)
+	assert.NoError(t, err)
+
+	rule := ruleset.Rule{
+		Domain: u.Host,
+		RegexRules: []ruleset.Regex{
+			// Strict pattern requiring src= — only matches if applyRules runs
+			// BEFORE rewriteHtml's src→script rename. Confirms ordering too.
+			{Match: `<script[^>]*src="https://example\.test/paywall\.js"[^>]*></script>`, Replace: ""},
+		},
+	}
+	rule.Injections = append(rule.Injections, struct {
+		Position string `yaml:"position,omitempty"`
+		Append   string `yaml:"append,omitempty"`
+		Prepend  string `yaml:"prepend,omitempty"`
+		Replace  string `yaml:"replace,omitempty"`
+	}{Position: "head", Append: `<style id="bpc-marker">body{filter:none}</style>`})
+
+	prev := rulesSet
+	rulesSet = ruleset.RuleSet{rule}
+	t.Cleanup(func() { rulesSet = prev })
+
+	body, _, _, err := fetchSite(server.URL, map[string]string{})
+	assert.NoError(t, err)
+
+	assert.NotContains(t, body, `paywall.js`, "regexRules in fetchSite should strip the upstream script")
+	assert.Contains(t, body, `<style id="bpc-marker">`, "injections in fetchSite should append to <head>")
+}
+
+// TestApplyRulesOrderingBeforeRewriteHtml documents the ordering requirement:
+// applyRules must run BEFORE rewriteHtml because rewriteHtml renames `src="/..."`
+// to `script="..."` on relative-URL script tags, which would otherwise prevent
+// `<script[^>]*src="..."` regex patterns from matching.
+func TestApplyRulesOrderingBeforeRewriteHtml(t *testing.T) {
+	const upstreamHTML = `<html><head></head><body><script src="/relative-paywall.js"></script></body></html>`
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, upstreamHTML)
+	}))
+	t.Cleanup(server.Close)
+
+	u, _ := url.Parse(server.URL)
+
+	rule := ruleset.Rule{
+		Domain: u.Host,
+		RegexRules: []ruleset.Regex{
+			{Match: `<script[^>]*src="/relative-paywall\.js"[^>]*></script>`, Replace: ""},
+		},
+	}
+
+	prev := rulesSet
+	rulesSet = ruleset.RuleSet{rule}
+	t.Cleanup(func() { rulesSet = prev })
+
+	body, _, _, err := fetchSite(server.URL, map[string]string{})
+	assert.NoError(t, err)
+
+	assert.NotContains(t, body, `relative-paywall.js`, "applyRules must see the original src= attribute before rewriteHtml renames it")
+}
+
+// Tiny sanity util so callers can spot test failures faster when reading output.
+var _ = io.EOF
+var _ = strings.TrimSpace
diff --git a/handlers/default_scheme_test.go b/handlers/default_scheme_test.go
@@ -0,0 +1,176 @@
+package handlers
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSetDefaultScheme(t *testing.T) {
+	original := DefaultScheme()
+	t.Cleanup(func() { _ = SetDefaultScheme(original) })
+
+	assert.NoError(t, SetDefaultScheme("http"))
+	assert.Equal(t, "http", DefaultScheme())
+
+	assert.NoError(t, SetDefaultScheme("HTTPS"))
+	assert.Equal(t, "https", DefaultScheme())
+
+	assert.NoError(t, SetDefaultScheme("  https  "))
+	assert.Equal(t, "https", DefaultScheme())
+
+	for _, bad := range []string{"ftp", "ws", "", "javascript", "file"} {
+		err := SetDefaultScheme(bad)
+		assert.Error(t, err, "expected error for %q", bad)
+	}
+	// Last successful value should still be in place after rejected attempts.
+	assert.Equal(t, "https", DefaultScheme())
+}
+
+func TestEnsureScheme(t *testing.T) {
+	original := DefaultScheme()
+	t.Cleanup(func() { _ = SetDefaultScheme(original) })
+
+	require := func(scheme string) {
+		t.Helper()
+		if err := SetDefaultScheme(scheme); err != nil {
+			t.Fatalf("SetDefaultScheme(%q) failed: %v", scheme, err)
+		}
+	}
+
+	require("https")
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"already https", "https://example.com/page", "https://example.com/page"},
+		{"already http", "http://example.com", "http://example.com"},
+		{"schemeless host", "example.com/page", "https://example.com/page"},
+		{"schemeless host with query", "example.com/page?q=1", "https://example.com/page?q=1"},
+		{"schemeless host with port", "example.com:8443/page", "https://example.com:8443/page"},
+		{"localhost with port", "localhost:8080/api", "https://localhost:8080/api"},
+		{"single-segment host", "localhost/api", "https://localhost/api"},
+		{"truly relative path", "/images/foo.jpg", "/images/foo.jpg"},
+		{"bare hostname", "example.com", "https://example.com"},
+		{"stray leading scheme separator", "://example.com", "https://example.com"},
+		{"non-network scheme treated as schemeless", "mailto:foo@bar.com", "https://mailto:foo@bar.com"},
+		{"ftp scheme left alone", "ftp://example.com/file", "ftp://example.com/file"},
+		{"empty", "", ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, ensureScheme(tc.in))
+		})
+	}
+
+	require("http")
+	assert.Equal(t, "http://example.com/page", ensureScheme("example.com/page"),
+		"schemeless URL should pick up the configured non-default scheme")
+	assert.Equal(t, "https://example.com/page", ensureScheme("https://example.com/page"),
+		"explicit scheme must never be overridden")
+}
+
+func TestExtractUrlPrependsDefaultScheme(t *testing.T) {
+	original := DefaultScheme()
+	t.Cleanup(func() { _ = SetDefaultScheme(original) })
+	require := func(scheme string) {
+		t.Helper()
+		if err := SetDefaultScheme(scheme); err != nil {
+			t.Fatalf("SetDefaultScheme(%q) failed: %v", scheme, err)
+		}
+	}
+
+	run := func(t *testing.T, target string, headers map[string]string) string {
+		t.Helper()
+		app := fiber.New()
+		var captured string
+		app.Get("/*", func(c *fiber.Ctx) error {
+			u, err := extractUrl(c)
+			if err != nil {
+				return c.Status(500).SendString(err.Error())
+			}
+			captured = u
+			return c.SendString("ok")
+		})
+		req := httptest.NewRequest("GET", target, nil)
+		for k, v := range headers {
+			req.Header.Set(k, v)
+		}
+		resp, err := app.Test(req)
+		assert.NoError(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		return captured
+	}
+
+	require("https")
+	t.Run("schemeless URL with no referer gets https", func(t *testing.T) {
+		got := run(t, "/example.com/page", nil)
+		assert.Equal(t, "https://example.com/page", got)
+	})
+
+	t.Run("https URL is unchanged", func(t *testing.T) {
+		got := run(t, "/https://example.com/page", nil)
+		assert.Equal(t, "https://example.com/page", got)
+	})
+
+	t.Run("http URL is unchanged", func(t *testing.T) {
+		got := run(t, "/http://example.com/page", nil)
+		assert.Equal(t, "http://example.com/page", got)
+	})
+
+	t.Run("relative path with proxied referer still reconstructs", func(t *testing.T) {
+		got := run(t, "/images/foo.jpg", map[string]string{
+			"Referer": "http://localhost:8080/https://realsite.com/article",
+		})
+		assert.Equal(t, "https://realsite.com/images/foo.jpg", got)
+	})
+
+	require("http")
+	t.Run("configured http scheme is used", func(t *testing.T) {
+		got := run(t, "/example.com/page", nil)
+		assert.Equal(t, "http://example.com/page", got)
+	})
+}
+
+func TestUnsupportedProtocolSchemeBugRepro(t *testing.T) {
+	_, err := http.Get("example.com/page")
+	if err == nil {
+		t.Fatalf("expected stdlib http.Get to fail for schemeless URL")
+	}
+	if !strings.Contains(err.Error(), `unsupported protocol scheme ""`) {
+		t.Fatalf("expected `unsupported protocol scheme \"\"` in error, got: %v", err)
+	}
+}
+
+func TestRawHandlerHandlesSchemelessURL(t *testing.T) {
+	originalScheme := DefaultScheme()
+	t.Cleanup(func() { _ = SetDefaultScheme(originalScheme) })
+
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("hello from upstream"))
+	}))
+	t.Cleanup(upstream.Close)
+
+	if err := SetDefaultScheme("http"); err != nil {
+		t.Fatalf("SetDefaultScheme: %v", err)
+	}
+
+	schemelessHost := strings.TrimPrefix(upstream.URL, "http://")
+
+	app := fiber.New()
+	app.Get("/raw/*", Raw)
+	req := httptest.NewRequest("GET", "/raw/"+schemelessHost, nil)
+	resp, err := app.Test(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Contains(t, string(body), "hello from upstream")
+}