Release/v0.0.24

README.md

@@ -36,6 +36,7 @@ sequenceDiagram
 - [x] Remove/inject custom code (HTML, CSS, JavaScript) into the page
 - [x] Apply domain based ruleset/code to modify response / requested URL
 - [x] Keep site browsable
+- [x] TLS Fingerprint emulation (JA3, JA4R, HTTP2)
 - [x] API
 - [x] Fetch RAW HTML
 - [x] Custom User Agent
@@ -178,6 +179,10 @@ There is a basic ruleset available in a separate repository [ruleset.yaml](https
       prepend: | 
         <h2>Subtitle</h2>
 - domain: demo.com
+  tlsFingerprint:
+    ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0"
+    ja4r: "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0000,0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601"
+    http2: "1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s"
   headers:
     content-security-policy: script-src 'self';
     user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36

cmd/main.go

@@ -124,7 +124,7 @@ func main() {
 		URL:  basePath + "/favicon.ico",
 	}))
 
-	if os.Getenv("NOLOGS") != "true" {
+	if os.Getenv("NOLOGS") != "true" && os.Getenv("LOG_URLS") != "false" {
 		app.Use(func(c *fiber.Ctx) error {
 			log.Println(c.Method(), c.Path())
 

go.mod

@@ -13,17 +13,39 @@ require (
 require github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 
 require (
+	github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.30 // indirect
+	github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 // indirect
 	github.com/andybalholm/brotli v1.2.1 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gaukas/clienthellod v0.4.2 // indirect
+	github.com/gaukas/godicttls v0.0.4 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.22 // indirect
+	github.com/onsi/ginkgo/v2 v2.23.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/quic-go/qpack v0.5.1 // indirect
+	github.com/quic-go/quic-go v0.53.0 // indirect
+	github.com/refraction-networking/uquic v0.0.6 // indirect
+	github.com/refraction-networking/utls v1.8.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.70.0 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
+	go.uber.org/mock v0.5.2 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
+	golang.org/x/mod v0.33.0 // indirect
 	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/tools v0.42.0 // indirect
+	h12.io/socks v1.0.3 // indirect
 )

handlers/cycletls_client.go

@@ -0,0 +1,39 @@
+package handlers
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strconv"
+
+	"github.com/Danny-Dasilva/CycleTLS/cycletls"
+)
+
+// fetchWithCycleTLS performs an HTTP GET using CycleTLS to spoof JA3, JA4r and/or
+// HTTP/2 fingerprints. It returns a synthetic *http.Response that is fully compatible
+// with the rest of the proxy pipeline (rewriteHtml, applyRules, etc.).
+func fetchWithCycleTLS(targetURL string, opts cycletls.Options) (*http.Response, []byte, error) {
+	client := cycletls.Init()
+	defer client.Close()
+
+	clResp, err := client.Do(targetURL, opts, "GET")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Build a synthetic *http.Response so callers don't need to know about CycleTLS.
+	bodyBytes := []byte(clResp.Body)
+
+	syntheticResp := &http.Response{
+		StatusCode: clResp.Status,
+		Status:     strconv.Itoa(clResp.Status) + " " + http.StatusText(clResp.Status),
+		Header:     make(http.Header),
+		Body:       io.NopCloser(bytes.NewReader(bodyBytes)),
+	}
+
+	for k, v := range clResp.Headers {
+		syntheticResp.Header.Set(k, v)
+	}
+
+	return syntheticResp, bodyBytes, nil
+}

handlers/form.html

@@ -42,6 +42,11 @@ <h1 class="text-center text-3xl sm:text-4xl font-extrabold text-slate-900 tracki
             if (url.indexOf('http') === -1) {
                 url = 'https://' + url;
             }
+            try {
+                url = encodeURIComponent(decodeURIComponent(url));
+            } catch (e) {
+                url = encodeURIComponent(url);
+            }
             window.location.href = window.location.pathname.replace(/\/$/, '') + '/' + url;
             return false;
         });

handlers/proxy.go

@@ -16,6 +16,7 @@ import (
 
 	"ladder/pkg/ruleset"
 
+	cycletls "github.com/Danny-Dasilva/CycleTLS/cycletls"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/gofiber/fiber/v2"
 )
@@ -102,25 +103,27 @@ func extractUrl(c *fiber.Ctx) (string, error) {
 	// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
 	if isRelativePath {
 		// Parse the referer URL from the request header.
-		refererUrl, err := url.Parse(c.Get("referer"))
-		if err != nil {
-			return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
+		referer := c.Get("referer")
+		if referer == "" {
+			return "", fmt.Errorf("relative URL '%s' requires a Referer header to reconstruct the target URL; navigate to a full URL (e.g. https://example.com) or use the form first", reqUrl)
 		}
-
-		// Extract the real url from referer path
-		realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
+		refererUrl, err := url.Parse(referer)
 		if err != nil {
-			return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
+			return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
 		}
 
 		// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
 		fullUrl := &url.URL{
-			Scheme:   realUrl.Scheme,
-			Host:     realUrl.Host,
+			Scheme:   refererUrl.Scheme,
+			Host:     refererUrl.Host,
 			Path:     urlQuery.Path,
 			RawQuery: urlQuery.RawQuery,
 		}
 
+		if fullUrl.Host == "" {
+			return "", fmt.Errorf("relative URL '%s' has a referer '%s' with no host; cannot reconstruct target URL", reqUrl, referer)
+		}
+
 		if os.Getenv("LOG_URLS") == "true" {
 			log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
 		}
@@ -272,35 +275,29 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		return "", nil, nil, err
 	}
 
-	// Fetch the site
-	client := &http.Client{
-		Timeout: time.Second * time.Duration(defaultTimeout),
-	}
-	req, _ := http.NewRequest("GET", url, nil)
-
+	// Resolve headers shared by both code paths.
+	userAgent := UserAgent
 	if rule.Headers.UserAgent != "" {
-		req.Header.Set("User-Agent", rule.Headers.UserAgent)
-	} else {
-		req.Header.Set("User-Agent", UserAgent)
+		userAgent = rule.Headers.UserAgent
 	}
 
-	if rule.Headers.XForwardedFor != "" {
-		if rule.Headers.XForwardedFor != "none" {
-			req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor)
-		}
-	} else {
-		req.Header.Set("X-Forwarded-For", ForwardedFor)
+	referer := u.String()
+	if rule.Headers.Referer != "" && rule.Headers.Referer != "none" {
+		referer = rule.Headers.Referer
+	} else if rule.Headers.Referer == "none" {
+		referer = ""
 	}
 
-	if rule.Headers.Referer != "" {
-		if rule.Headers.Referer != "none" {
-			req.Header.Set("Referer", rule.Headers.Referer)
+	xForwardedFor := ForwardedFor
+	if rule.Headers.XForwardedFor != "" {
+		if rule.Headers.XForwardedFor == "none" {
+			xForwardedFor = ""
+		} else {
+			xForwardedFor = rule.Headers.XForwardedFor
 		}
-	} else {
-		req.Header.Set("Referer", u.String())
 	}
 
-	// Handle FlareSolverr integration
+	// Handle FlareSolverr integration (shared by both paths).
 	cookieValue := rule.Headers.Cookie
 	debug := os.Getenv("LOG_URLS") == "true"
 
@@ -319,23 +316,124 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		}
 	}
 
-	if cookieValue != "" {
-		req.Header.Set("Cookie", cookieValue)
-	}
+	fp := rule.TLSFingerprint
+	useCycleTLS := fp.Ja3 != "" || fp.Ja4r != "" || fp.HTTP2 != ""
 
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", nil, nil, err
-	}
-	defer resp.Body.Close()
+	var resp *http.Response
+	var bodyB []byte
+	var req *http.Request
 
-	bodyB, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", nil, nil, err
+	if useCycleTLS {
+		// ------------------------------------------------------------------
+		// CycleTLS path: spoofs JA3 / JA4r / HTTP2 TLS+HTTP fingerprints.
+		// ------------------------------------------------------------------
+
+		extraHeaders := map[string]string{}
+		/*
+			if xForwardedFor != "" {
+				extraHeaders["X-Forwarded-For"] = xForwardedFor
+			}
+			if referer != "" {
+				extraHeaders["Referer"] = referer
+			}
+			if cookieValue != "" {
+				extraHeaders["Cookie"] = cookieValue
+			}
+		*/
+		extraHeaders["Accept-Encoding"] = "gzip, deflate"
+		extraHeaders["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+		extraHeaders["Accept-Language"] = "en-US,en;q=0.9"
+		//extraHeaders["Cache-Control"] = "no-cache"
+		//extraHeaders["Pragma"] = "no-cache"
+		//extraHeaders["Upgrade-Insecure-Requests"] = "1"
+
+		cycleTLSCookies := []cycletls.Cookie{}
+		if cookieValue != "" {
+			// split cookie from ruleset into individual cookies and add to CycleTLS options
+			cookieParts := strings.Split(cookieValue, ";")
+			for _, part := range cookieParts {
+				part = strings.TrimSpace(part)
+				if part == "" {
+					continue
+				}
+				kv := strings.SplitN(part, "=", 2)
+				if len(kv) != 2 {
+					continue
+				}
+				cycleTLSCookies = append(cycleTLSCookies, cycletls.Cookie{
+					Name:  kv[0],
+					Value: kv[1],
+				})
+			}
+		}
+
+		ctOpts := cycletls.Options{
+			UserAgent:        userAgent,
+			Ja3:              fp.Ja3,
+			Ja4r:             fp.Ja4r,
+			HTTP2Fingerprint: fp.HTTP2,
+			Timeout:          defaultTimeout,
+			Headers:          extraHeaders,
+			Cookies:          cycleTLSCookies,
+		}
+
+		if debug {
+			log.Printf("CycleTLS fetch %s (ja3=%q ja4r=%q http2=%q)", url, fp.Ja3, fp.Ja4r, fp.HTTP2)
+		}
+
+		var cycleTLSErr error
+		resp, bodyB, cycleTLSErr = fetchWithCycleTLS(url, ctOpts)
+		if cycleTLSErr != nil {
+			return "", nil, nil, cycleTLSErr
+		}
+
+		// Build a synthetic *http.Request for callers that inspect it (e.g. api.go).
+		req, _ = http.NewRequest("GET", url, nil)
+		req.Header.Set("User-Agent", userAgent)
+		if xForwardedFor != "" {
+			req.Header.Set("X-Forwarded-For", xForwardedFor)
+		}
+		if referer != "" {
+			req.Header.Set("Referer", referer)
+		}
+		if cookieValue != "" {
+			req.Header.Set("Cookie", cookieValue)
+		}
+	} else {
+		// ------------------------------------------------------------------
+		// Standard net/http path.
+		// ------------------------------------------------------------------
+		client := &http.Client{
+			Timeout: time.Second * time.Duration(defaultTimeout),
+		}
+		req, _ = http.NewRequest("GET", url, nil)
+
+		req.Header.Set("User-Agent", userAgent)
+		if xForwardedFor != "" {
+			req.Header.Set("X-Forwarded-For", xForwardedFor)
+		}
+		if referer != "" {
+			req.Header.Set("Referer", referer)
+		}
+		if cookieValue != "" {
+			req.Header.Set("Cookie", cookieValue)
+		}
+
+		var doErr error
+		resp, doErr = client.Do(req)
+		if doErr != nil {
+			return "", nil, nil, doErr
+		}
+		defer resp.Body.Close()
+
+		var readErr error
+		bodyB, readErr = io.ReadAll(resp.Body)
+		if readErr != nil {
+			return "", nil, nil, readErr
+		}
 	}
 
 	if rule.Headers.CSP != "" {
-		// log.Println(rule.Headers.CSP)
 		resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
 	} else {
 		resp.Header.Del("Content-Security-Policy")